refactor: extract object store into dedicated crate with filesystem backend (#7996)

* refactor: extract object store code into windmill-object-store crate with filesystem backend

Consolidate all object_store-dependent code from windmill-common into a new
windmill-object-store crate. Add a filesystem-backed object store implementation
using LocalFileSystem for dev/testing without cloud credentials. Includes 30
comprehensive tests covering render_endpoint, lfs_to_object_store_resource,
duckdb_connection_settings, error mapping, and filesystem-backed integration tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* all

* all

* all

* all

* fix: fix raw_app hardcoded path, add missing ObjectStoreResource import, and add tests

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor: move S3ModeFormat to windmill-types, make windmill-parser-sql optional, restore debug logs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* all

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

backend/windmill-api/Cargo.toml

@@ -10,15 +10,15 @@ path = "src/lib.rs"
 
 [features]
 default = []
-private = ["windmill-audit/private", "windmill-common/private", "windmill-sandbox/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"]
-enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-sandbox/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"]
+private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"]
+enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"]
 stripe = []
 inline_preview = ["dep:windmill-worker", "windmill-api-configs/inline_preview"]
 agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"]
 enterprise_saml = ["dep:samael", "dep:libxml"]
 benchmark = []
 embedding = ["windmill-api-embeddings/embedding"]
-parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-sandbox/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"]
+parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus"]
 openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"]
 tantivy = ["dep:windmill-indexer"]
@@ -58,7 +58,8 @@ windmill-trigger.workspace = true
 windmill-store.workspace = true
 windmill-queue.workspace = true
 windmill-common = { workspace = true, default-features = false }
-windmill-sandbox.workspace = true
+windmill-types.workspace = true
+windmill-object-store.workspace = true
 windmill-audit.workspace = true
 windmill-parser.workspace = true
 windmill-parser-sql.workspace = true
@@ -130,7 +131,6 @@ async-recursion.workspace = true
 rsa = { workspace = true, optional = true}
 uuid.workspace = true
 datafusion = { workspace = true, optional = true}
-object_store = { workspace = true, optional = true}
 openidconnect = { workspace = true, optional = true}
 jsonwebtoken = { workspace = true }
 matchit = { workspace = true, optional = true }

backend/windmill-api/build.rs

@@ -0,0 +1,11 @@
+fn main() {
+    // rust-embed requires the embedded folder to exist at compile time.
+    // When building without a prior frontend build, create the directory
+    // so the derive macro doesn't panic.
+    let dir = std::env::var("FRONTEND_BUILD_DIR")
+        .unwrap_or_else(|_| "../../frontend/build/".to_string());
+    let path = std::path::Path::new(&dir);
+    if !path.exists() {
+        std::fs::create_dir_all(path).ok();
+    }
+}

backend/windmill-api/src/apps.rs

@@ -39,7 +39,7 @@ use itertools::Itertools;
 use lazy_static::lazy_static;
 use magic_crypt::MagicCryptTrait;
 #[cfg(feature = "parquet")]
-use object_store::{Attribute, Attributes};
+use windmill_object_store::object_store_reexports::{Attribute, Attributes};
 #[cfg(feature = "parquet")]
 use regex::Regex;
 use serde::{Deserialize, Serialize};
@@ -78,9 +78,10 @@ use hmac::Mac;
 use windmill_common::{
     jwt,
     oauth2::HmacSha256,
-    s3_helpers::{build_object_store_client, S3Object, S3Permission},
     variables::get_workspace_key,
 };
+#[cfg(feature = "parquet")]
+use windmill_types::s3::{S3Object, S3Permission};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -406,7 +407,7 @@ async fn get_raw_app_data(
     Extension(db): Extension<DB>,
 ) -> Result<Response> {
     #[cfg(all(feature = "enterprise", feature = "parquet"))]
-    let object_store = windmill_common::s3_helpers::get_object_store().await;
+    let object_store = windmill_object_store::get_object_store().await;
 
     // tracing::info!("secret_with_ext: {}", secret_with_ext);
     let mut splitted = secret_with_ext.split('.');
@@ -442,10 +443,12 @@ async fn get_raw_app_data(
     if let Some(os) = object_store {
         let path = format!("/app_bundles/{}/{}.{}", w_id, id, file_type);
         let stream = os
-            .get(&object_store::path::Path::from(path))
-            .await?
+            .get(&windmill_object_store::object_store_reexports::Path::from(path))
+            .await
+            .map_err(windmill_object_store::object_store_error_to_error)?
             .bytes()
-            .await?;
+            .await
+            .map_err(windmill_object_store::object_store_error_to_error)?;
         tracing::info!("stream: {}", stream.len());
         body = Some(Body::from(stream));
     }
@@ -949,13 +952,13 @@ async fn store_raw_app_file<'a>(
 ) -> Result<()> {
     #[cfg(all(feature = "enterprise", feature = "parquet"))]
     {
-        let object_store = windmill_common::s3_helpers::get_object_store().await;
+        let object_store = windmill_object_store::get_object_store().await;
 
         let path: String = format!("/app_bundles/{}/{}.{}", w_id, id, file_type);
 
         if let Some(os) = object_store {
             if let Err(e) = os
-                .put(&object_store::path::Path::from(path.clone()), data.into())
+                .put(&windmill_object_store::object_store_reexports::Path::from(path.clone()), data.into())
                 .await
             {
                 tracing::error!("Failed to put snapshot to s3 at {path}: {:?}", e);
@@ -2480,7 +2483,7 @@ async fn upload_s3_file_from_app(
     let s3_resource = s3_resource_opt.ok_or(Error::internal_err(
         "No files storage resource defined at the workspace level".to_string(),
     ))?;
-    let s3_client = build_object_store_client(&s3_resource).await?;
+    let s3_client = windmill_object_store::build_object_store_client(&s3_resource).await?;
 
     let options = Attributes::from_iter(vec![
         (
@@ -2537,7 +2540,7 @@ async fn delete_s3_file_from_app(
         ..
     } = jwt::decode_with_internal_secret::<S3DeleteTokenClaims>(&query.delete_token).await?;
 
-    let path = object_store::path::Path::parse(file_key.as_str())
+    let path = windmill_object_store::object_store_reexports::Path::parse(file_key.as_str())
         .map_err(|e| Error::internal_err(format!("Error parsing file key: {}", e)))?;
 
     if workspace != w_id {
@@ -2581,7 +2584,7 @@ async fn delete_s3_file_from_app(
         ))?
     };
 
-    let s3_client = build_object_store_client(&s3_resource).await?;
+    let s3_client = windmill_object_store::build_object_store_client(&s3_resource).await?;
 
     s3_client.delete(&path).await.map_err(|err| {
         tracing::error!("Error deleting file: {:?}", err);

backend/windmill-api/src/args.rs

@@ -91,8 +91,8 @@ impl RawWebhookArgs {
             get_random_file_name, get_workspace_s3_resource, upload_file_internal,
         };
         use futures::TryStreamExt;
-        use object_store::{Attribute, Attributes};
-        use windmill_common::s3_helpers::build_object_store_client;
+        use windmill_object_store::object_store_reexports::{Attribute, Attributes};
+        use windmill_object_store::build_object_store_client;
 
         let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, w_id, None).await?;
 

backend/windmill-api/src/job_helpers_oss.rs

@@ -7,18 +7,20 @@ use axum::Router;
 #[cfg(not(feature = "private"))]
 use uuid::Uuid;
 #[cfg(not(feature = "private"))]
-use windmill_common::s3_helpers::StorageResourceType;
+use windmill_types::s3::StorageResourceType;
 
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use crate::db::{ApiAuthed, OptJobAuthed, DB};
 #[cfg(all(feature = "parquet", not(feature = "private")))]
-use object_store::{ObjectStore, PutMultipartOpts, PutResult};
+use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult};
+#[cfg(not(feature = "private"))]
+use windmill_object_store::ObjectStoreResource;
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use std::sync::Arc;
 #[cfg(not(feature = "private"))]
 use windmill_common::error;
 #[cfg(all(feature = "parquet", not(feature = "private")))]
-use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource};
+use windmill_common::db::UserDB;
 
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use bytes::Bytes;
@@ -155,11 +157,11 @@ pub async fn get_workspace_s3_resource_and_check_paths<'c>(
     _authed_api: Option<&ApiAuthed>,
     _w_id: &str,
     _storage: Option<String>,
-    _paths: &[(&str, windmill_common::s3_helpers::S3Permission)],
+    _paths: &[(&str, windmill_types::s3::S3Permission)],
     _job_id: Option<uuid::Uuid>,
 ) -> windmill_common::error::Result<(
     Option<bool>,
-    Option<windmill_common::s3_helpers::ObjectStoreResource>,
+    Option<windmill_types::s3::ObjectStoreResource>,
 )> {
     Err(windmill_common::error::Error::internal_err(
         "Not implemented in Windmill's Open Source repository".to_string(),

backend/windmill-api/src/jobs.rs

@@ -42,7 +42,8 @@ use windmill_common::runnable_settings::{
 };
 #[cfg(feature = "inline_preview")]
 use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams};
-use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat};
+use windmill_types::s3::BundleFormat;
+use windmill_object_store::upload_artifact_to_store;
 use windmill_common::scripts::ScriptRunnableSettingsInline;
 use windmill_common::triggers::TriggerMetadata;
 use windmill_common::utils::{RunnableKind, WarnAfterExt};
@@ -1384,40 +1385,15 @@ async fn get_logs_from_store(
     logs: &str,
     log_file_index: &Option<Vec<String>>,
 ) -> Option<error::Result<Body>> {
-    if log_offset > 0 {
-        if let Some(file_index) = log_file_index.clone() {
-            tracing::debug!("Getting logs from store: {file_index:?}");
-            if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
-                tracing::debug!("object store client present, streaming from there");
-
-                let logs = logs.to_string();
-                let stream = async_stream::stream! {
-                    yield Ok(bytes::Bytes::from(
-                        r#"to remove ansi colors, use: | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'
-                "#
-                        .to_string(),
-                    ));
-                    for file_p in file_index.clone() {
-                        let file_p_2 = file_p.clone();
-                        let file = os.get(&object_store::path::Path::from(file_p)).await;
-                        if let Ok(file) = file {
-                            if let Ok(bytes) = file.bytes().await {
-                                yield Ok(bytes::Bytes::from(bytes)) as object_store::Result<bytes::Bytes>;
-                            }
-                        } else {
-                            tracing::debug!("error getting file from store: {file_p_2}: {}", file.err().unwrap());
-                        }
-                    }
-
-                    yield Ok(bytes::Bytes::from(logs))
-                };
-                return Some(Ok(Body::from_stream(stream)));
-            } else {
-                tracing::debug!("object store client not present, cannot stream logs from store");
-            }
-        }
-    }
-    return None;
+    use futures::StreamExt;
+    let stream = windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await?;
+    let header = bytes::Bytes::from(
+        r#"to remove ansi colors, use: | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'
+"#
+        .to_string(),
+    );
+    let prefixed_stream = futures::stream::once(async { Ok(header) }).chain(stream);
+    Some(Ok(Body::from_stream(prefixed_stream)))
 }
 
 async fn get_logs_from_disk(
@@ -4840,7 +4816,7 @@ async fn run_bundle_preview_script(
 
             uploaded = true;
 
-            let path = windmill_common::s3_helpers::bundle(&w_id, &id);
+            let path = windmill_object_store::bundle(&w_id, &id);
             upload_artifact_to_store(
                 &path,
                 data,
@@ -5794,9 +5770,9 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
     }
 
     #[cfg(all(feature = "enterprise", feature = "parquet"))]
-    if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
+    if let Some(os) = windmill_object_store::get_object_store().await {
         let file = os
-            .get(&object_store::path::Path::from(format!("logs/{file_p}")))
+            .get(&windmill_object_store::object_store_reexports::Path::from(format!("logs/{file_p}")))
             .await;
         if let Ok(file) = file {
             if let Ok(bytes) = file.bytes().await {

backend/windmill-api/src/service_logs.rs

@@ -98,11 +98,11 @@ async fn get_log_file(
     require_devops_role(&db, &email).await?;
     let path = path.to_path();
     #[cfg(feature = "parquet")]
-    let s3_client = windmill_common::s3_helpers::get_object_store().await;
+    let s3_client = windmill_object_store::get_object_store().await;
     #[cfg(feature = "parquet")]
     if let Some(s3_client) = s3_client {
         let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);
-        let file = s3_client.get(&object_store::path::Path::from(path)).await;
+        let file = s3_client.get(&windmill_object_store::object_store_reexports::Path::from(path)).await;
         match file {
             Ok(file) => {
                 let bytes = file.bytes().await;

backend/windmill-api/src/triggers/http/handler.rs

@@ -35,7 +35,7 @@ use windmill_trigger::TriggerMode;
 #[cfg(feature = "parquet")]
 use {
     crate::job_helpers_oss::get_workspace_s3_resource,
-    windmill_common::s3_helpers::build_object_store_client,
+    windmill_object_store::build_object_store_client,
 };
 
 async fn conditional_cors_middleware(
@@ -365,13 +365,13 @@ async fn route_job(
             } else {
                 config.s3.clone()
             };
-            let path = object_store::path::Path::from(path);
+            let path = windmill_object_store::object_store_reexports::Path::from(path);
             let s3_object = s3_client.get(&path).await;
 
             let s3_object = match s3_object {
-                Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => {
+                Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => {
                     // fallback to index.html if the file is not found
-                    let path = object_store::path::Path::from(format!(
+                    let path = windmill_object_store::object_store_reexports::Path::from(format!(
                         "{}/index.html",
                         config.s3.trim_end_matches('/')
                     ));
@@ -410,7 +410,7 @@ async fn route_job(
                 "content-type",
                 s3_object
                     .attributes
-                    .get(&object_store::Attribute::ContentType)
+                    .get(&windmill_object_store::object_store_reexports::Attribute::ContentType)
                     .map(|s| s.parse().ok())
                     .flatten()
                     .unwrap_or("application/octet-stream".parse().unwrap()),
@@ -422,7 +422,7 @@ async fn route_job(
                         || {
                             s3_object
                                 .attributes
-                                .get(&object_store::Attribute::ContentDisposition)
+                                .get(&windmill_object_store::object_store_reexports::Attribute::ContentDisposition)
                                 .map(|s| s.parse().ok())
                                 .flatten()
                                 .unwrap_or("inline".parse().unwrap())