From a3b6fa4eb3bbd3f43e59bf6439027a548b564dd2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 18 Feb 2026 13:24:45 +0100 Subject: [PATCH] refactor: extract object store into dedicated crate with filesystem backend (#7996) * refactor: extract object store code into windmill-object-store crate with filesystem backend Consolidate all object_store-dependent code from windmill-common into a new windmill-object-store crate. Add a filesystem-backed object store implementation using LocalFileSystem for dev/testing without cloud credentials. Includes 30 comprehensive tests covering render_endpoint, lfs_to_object_store_resource, duckdb_connection_settings, error mapping, and filesystem-backed integration tests. Co-Authored-By: Claude Opus 4.6 * all * all * all * all * fix: fix raw_app hardcoded path, add missing ObjectStoreResource import, and add tests Co-Authored-By: Claude Opus 4.6 * refactor: move S3ModeFormat to windmill-types, make windmill-parser-sql optional, restore debug logs Co-Authored-By: Claude Opus 4.6 * all --------- Co-authored-by: Claude Opus 4.6 --- backend/Cargo.lock | 77 +- backend/Cargo.toml | 15 +- backend/ee-repo-ref.txt | 2 +- .../parsers/windmill-parser-sql/Cargo.toml | 1 + .../parsers/windmill-parser-sql/src/lib.rs | 14 +- backend/src/main.rs | 2 +- backend/src/monitor.rs | 12 +- backend/substitute_ee_code.sh | 7 +- backend/windmill-api-scripts/Cargo.toml | 1 + backend/windmill-api-scripts/src/scripts.rs | 4 +- backend/windmill-api-settings/Cargo.toml | 4 +- backend/windmill-api-settings/src/lib.rs | 10 +- backend/windmill-api-workspaces/Cargo.toml | 1 + .../windmill-api-workspaces/src/workspaces.rs | 2 +- backend/windmill-api/Cargo.toml | 10 +- backend/windmill-api/build.rs | 11 + backend/windmill-api/src/apps.rs | 25 +- backend/windmill-api/src/args.rs | 4 +- backend/windmill-api/src/job_helpers_oss.rs | 12 +- backend/windmill-api/src/jobs.rs | 52 +- backend/windmill-api/src/service_logs.rs | 4 +- .../windmill-api/src/triggers/http/handler.rs | 12 +- backend/windmill-common/Cargo.toml | 3 +- backend/windmill-common/src/ai_types.rs | 2 +- backend/windmill-common/src/error.rs | 39 - backend/windmill-common/src/jobs.rs | 36 - backend/windmill-common/src/lib.rs | 5 - backend/windmill-common/src/worker.rs | 133 +- backend/windmill-indexer/Cargo.toml | 6 +- backend/windmill-object-store/Cargo.toml | 60 + .../src/job_s3_helpers_oss.rs | 14 +- .../src/lib.rs} | 1828 ++++++++++------- backend/windmill-trigger-http/Cargo.toml | 1 + backend/windmill-trigger-http/src/lib.rs | 2 +- backend/windmill-types/Cargo.toml | 1 + backend/windmill-types/src/lib.rs | 1 + backend/windmill-types/src/s3.rs | 479 +++++ backend/windmill-worker/Cargo.toml | 12 +- .../windmill-worker/src/ai/image_handler.rs | 3 +- .../windmill-worker/src/ai/query_builder.rs | 3 +- backend/windmill-worker/src/ai/types.rs | 2 +- .../windmill-worker/src/bigquery_executor.rs | 2 +- backend/windmill-worker/src/bun_executor.rs | 13 +- backend/windmill-worker/src/common.rs | 28 +- .../windmill-worker/src/csharp_executor.rs | 6 +- .../windmill-worker/src/duckdb_executor.rs | 5 +- backend/windmill-worker/src/global_cache.rs | 148 +- backend/windmill-worker/src/go_executor.rs | 5 +- backend/windmill-worker/src/java_executor.rs | 5 +- backend/windmill-worker/src/mssql_executor.rs | 2 +- backend/windmill-worker/src/mysql_executor.rs | 2 +- .../windmill-worker/src/oracledb_executor.rs | 2 +- backend/windmill-worker/src/pg_executor.rs | 2 +- .../windmill-worker/src/python_executor.rs | 6 +- backend/windmill-worker/src/rust_executor.rs | 5 +- .../windmill-worker/src/snowflake_executor.rs | 2 +- .../src/universal_pkg_installer.rs | 8 +- backend/write_latest_ee_ref.sh | 4 +- 58 files changed, 1968 insertions(+), 1189 deletions(-) create mode 100644 backend/windmill-api/build.rs create mode 100644 backend/windmill-object-store/Cargo.toml rename backend/{windmill-common => windmill-object-store}/src/job_s3_helpers_oss.rs (69%) rename backend/{windmill-common/src/s3_helpers.rs => windmill-object-store/src/lib.rs} (54%) create mode 100644 backend/windmill-types/src/s3.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6b97bf50ad..e27fdc1d92 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15741,7 +15741,6 @@ dependencies = [ "gethostname", "git-version", "lazy_static", - "object_store", "once_cell", "prometheus", "rand 0.9.0", @@ -15777,10 +15776,10 @@ dependencies = [ "windmill-dep-map", "windmill-git-sync", "windmill-indexer", + "windmill-object-store", "windmill-operator", "windmill-queue", "windmill-runtime-nativets", - "windmill-sandbox", "windmill-test-utils", "windmill-worker", "windows-service", @@ -15847,7 +15846,6 @@ dependencies = [ "matchit 0.7.3", "mime_guess", "native-tls", - "object_store", "openidconnect", "openssl", "pin-project", @@ -15917,13 +15915,13 @@ dependencies = [ "windmill-mcp", "windmill-native-triggers", "windmill-oauth", + "windmill-object-store", "windmill-parser", "windmill-parser-py", "windmill-parser-py-imports", "windmill-parser-sql", "windmill-parser-ts", "windmill-queue", - "windmill-sandbox", "windmill-store", "windmill-trigger", "windmill-trigger-email", @@ -15935,6 +15933,7 @@ dependencies = [ "windmill-trigger-postgres", "windmill-trigger-sqs", "windmill-trigger-websocket", + "windmill-types", "windmill-worker", ] @@ -16275,6 +16274,7 @@ dependencies = [ "windmill-common", "windmill-dep-map", "windmill-git-sync", + "windmill-object-store", "windmill-parser-py", "windmill-parser-ts", "windmill-queue", @@ -16291,7 +16291,6 @@ dependencies = [ "chrono", "futures", "lazy_static", - "object_store", "regex", "rsa", "serde", @@ -16305,6 +16304,7 @@ dependencies = [ "windmill-alerting", "windmill-api-auth", "windmill-common", + "windmill-object-store", ] [[package]] @@ -16383,6 +16383,7 @@ dependencies = [ "windmill-dep-map", "windmill-git-sync", "windmill-queue", + "windmill-types", ] [[package]] @@ -16462,7 +16463,6 @@ dependencies = [ "magic-crypt", "mail-send", "native-tls", - "object_store", "once_cell", "openidconnect", "opentelemetry 0.27.1", @@ -16564,7 +16564,6 @@ dependencies = [ "flume", "futures", "lazy_static", - "object_store", "serde", "serde_json", "sqlx", @@ -16574,6 +16573,7 @@ dependencies = [ "tracing", "uuid", "windmill-common", + "windmill-object-store", ] [[package]] @@ -16685,6 +16685,40 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-object-store" +version = "1.638.3" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-sts", + "aws-smithy-types-convert", + "axum 0.7.9", + "bytes", + "chrono", + "datafusion", + "futures", + "globset", + "lazy_static", + "object_store", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "tempfile", + "tokio", + "tokio-stream", + "tracing", + "uuid", + "windmill-common", + "windmill-parser-sql", + "windmill-types", +] + [[package]] name = "windmill-operator" version = "1.638.4" @@ -16874,6 +16908,7 @@ dependencies = [ "serde_json", "sqlparser 0.59.0", "windmill-parser", + "windmill-types", ] [[package]] @@ -16981,26 +17016,6 @@ dependencies = [ "windmill-queue", ] -[[package]] -name = "windmill-sandbox" -version = "1.638.3" -dependencies = [ - "bytes", - "chrono", - "flate2", - "object_store", - "serde", - "serde_json", - "sha2 0.10.9", - "sqlx", - "tar", - "tempfile", - "tokio", - "tracing", - "uuid", - "windmill-common", -] - [[package]] name = "windmill-sql-datatype-parser-wasm" version = "1.638.4" @@ -17182,6 +17197,7 @@ dependencies = [ "windmill-common", "windmill-git-sync", "windmill-trigger", + "windmill-types", ] [[package]] @@ -17346,6 +17362,7 @@ name = "windmill-types" version = "1.638.4" dependencies = [ "anyhow", + "bitflags 2.9.4", "chrono", "hex", "itertools 0.14.0", @@ -17383,7 +17400,6 @@ dependencies = [ "derive_more 1.0.0", "dotenv", "eventsource-stream", - "flate2", "flume", "futures", "gcp_auth", @@ -17403,7 +17419,6 @@ dependencies = [ "mysql_async", "native-tls", "nix 0.27.1", - "object_store", "once_cell", "opentelemetry 0.27.1", "opentelemetry-proto 0.29.0", @@ -17425,7 +17440,6 @@ dependencies = [ "sha2 0.10.9", "sqlx", "tar", - "tempfile", "tiberius", "tokio", "tokio-postgres 0.7.13", @@ -17444,6 +17458,7 @@ dependencies = [ "windmill-jseval", "windmill-macros", "windmill-mcp", + "windmill-object-store", "windmill-parser", "windmill-parser-bash", "windmill-parser-csharp", @@ -17461,7 +17476,7 @@ dependencies = [ "windmill-parser-yaml", "windmill-queue", "windmill-runtime-nativets", - "windmill-sandbox", + "windmill-types", "yaml-rust", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f25d245ce3..90abdd5a6b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true [workspace] resolver = "2" members = [ + "./windmill-object-store", "./windmill-api", "./windmill-api-scripts", "./windmill-api-flows", @@ -71,7 +72,6 @@ members = [ "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", "./windmill-test-utils", "./windmill-api-integration-tests", - "./windmill-sandbox", ] exclude = ["./windmill-duckdb-ffi-internal"] @@ -97,19 +97,19 @@ lto = "thin" [features] default = [] -private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-sandbox/private", "windmill-test-utils/private"] +private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] -enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-sandbox/enterprise"] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise"] local_reports = ["windmill-common/local_reports"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] stripe = ["windmill-api/stripe"] benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] embedding = ["windmill-api/embedding"] -parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-sandbox/parquet", "dep:object_store"] +parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker/parquet"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"] flow_testing = ["windmill-worker/flow_testing"] quickjs = ["windmill-worker/quickjs", "windmill-api/quickjs"] -openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect"] +openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect", "windmill-object-store/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] @@ -200,12 +200,12 @@ tokio-stream.workspace = true dotenv.workspace = true windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } +windmill-object-store.workspace = true windmill-git-sync.workspace = true windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } windmill-api-settings.workspace = true windmill-worker.workspace = true -windmill-sandbox.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } windmill-operator = { workspace = true, optional = true } @@ -230,7 +230,6 @@ serde_derive.workspace = true serde_yml.workspace = true serde.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } -object_store = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } constant_time_eq = { workspace = true, optional = true } rustls.workspace = true @@ -270,6 +269,7 @@ windmill-worker = { path = "./windmill-worker" } windmill-dep-map = { path = "./windmill-dep-map" } windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } +windmill-object-store = { path = "./windmill-object-store" } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } windmill-autoscaling = { path = "./windmill-autoscaling" } @@ -330,7 +330,6 @@ windmill-jseval = { path = "./windmill-jseval" } windmill-runtime-nativets = { path = "./windmill-runtime-nativets" } windmill-api-client = { path = "./windmill-api-client" } windmill-test-utils = { path = "./windmill-test-utils" } -windmill-sandbox = { path = "./windmill-sandbox" } reqwest-retry = "^0" reqwest-middleware = { version = "^0", features = ["json"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 2ffcfc7f02..2c3170d5f3 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -592848d59ca2304926fb2bd85d000668a7f46a77 +0eccae6a9a9ecde09816cd4d88ca9ab305659e4c diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index 41533a6014..be92d1133b 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -16,6 +16,7 @@ regex.workspace = true [dependencies] windmill-parser.workspace = true +windmill-types.workspace = true anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index fa2be84430..e99fbd2589 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -141,19 +141,7 @@ pub fn parse_db_resource(code: &str) -> Option { cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap()) } -#[derive(Clone, Copy, Debug)] -pub enum S3ModeFormat { - Json, - Csv, - Parquet, -} -pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { - match format { - S3ModeFormat::Json => "json", - S3ModeFormat::Csv => "csv", - S3ModeFormat::Parquet => "parquet", - } -} +pub use windmill_types::s3::{s3_mode_extension, S3ModeFormat}; pub struct S3ModeArgs { pub prefix: Option, pub storage: Option, diff --git a/backend/src/main.rs b/backend/src/main.rs index 3304f0c9df..c4440b7f34 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -108,7 +108,7 @@ use crate::monitor::{ }; #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::reload_object_store_setting; +use windmill_object_store::reload_object_store_setting; const DEFAULT_NUM_WORKERS: usize = 1; const DEFAULT_PORT: u16 = 8000; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 2e70267605..f3849a46b1 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -40,7 +40,7 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; #[cfg(feature = "oauth2")] use windmill_common::global_settings::OAUTH_SETTING; #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::reload_object_store_setting; +use windmill_object_store::reload_object_store_setting; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::APP_WORKSPACED_ROUTE, @@ -95,7 +95,7 @@ use windmill_worker::{ }; #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::ObjectStoreReload; +use windmill_object_store::ObjectStoreReload; #[cfg(feature = "enterprise")] use crate::ee_oss::verify_license_key; @@ -712,7 +712,7 @@ async fn send_log_file_to_object_store( } #[cfg(feature = "parquet")] - let s3_client = windmill_common::s3_helpers::get_object_store().await; + let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE) @@ -725,7 +725,7 @@ async fn send_log_file_to_object_store( tracing::error!("Error reading log file: {:?}", e); return; } - let path = object_store::path::Path::from_url_path(format!( + let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( "{}{hostname}/{highest_file}", windmill_common::tracing_init::LOGS_SERVICE )); @@ -1174,7 +1174,7 @@ async fn delete_log_files_from_disk_and_store( _s3_prefix: &str, ) { #[cfg(feature = "parquet")] - let os = windmill_common::s3_helpers::get_object_store().await; + let os = windmill_object_store::get_object_store().await; #[cfg(not(feature = "parquet"))] let os: Option<()> = None; @@ -1204,7 +1204,7 @@ async fn delete_log_files_from_disk_and_store( #[cfg(feature = "parquet")] if _should_del_from_store { if let Some(os) = _os2 { - let p = object_store::path::Path::from(format!("{}{}", _s3_prefix, path)); + let p = windmill_object_store::object_store_reexports::Path::from(format!("{}{}", _s3_prefix, path)); if let Err(e) = os.delete(&p).await { tracing::error!("Failed to delete from object store {}: {e}", p.to_string()) } else { diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index 4f1a870b1a..e98f44529e 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -53,8 +53,13 @@ if [[ $EE_CODE_DIR == /* ]]; then else EE_CODE_DIR="${root_dirpath}/${EE_CODE_DIR}" fi -echo "EE code directory = ${EE_CODE_DIR} | Revert = ${REVERT}" +# Fallback to ~/windmill-ee-private if the default location doesn't exist +if [ ! -d "${EE_CODE_DIR}" ] && [ "${EE_CODE_DIR}" == "${root_dirpath}/../windmill-ee-private/" ]; then + EE_CODE_DIR="${HOME}/windmill-ee-private" +fi + +echo "EE code directory = ${EE_CODE_DIR} | Revert = ${REVERT}" if [ ! -d "${EE_CODE_DIR}" ]; then echo "Windmill EE repo not found, please clone it next to this repository (or use the --dir option) and try again" diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index 407348ee72..d905882b09 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -16,6 +16,7 @@ python = ["dep:windmill-parser-py"] [dependencies] windmill-common = { workspace = true, default-features = false } +windmill-object-store.workspace = true windmill-api-auth.workspace = true windmill-queue.workspace = true windmill-audit.workspace = true diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 65df3e0628..c264d95f06 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -51,11 +51,11 @@ use windmill_common::{ runnable_settings::{ min_version_supports_runnable_settings_v0, RunnableSettings, RunnableSettingsTrait, }, - s3_helpers::upload_artifact_to_store, scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, utils::{paginate_without_limits, WarnAfterExt}, worker::CLOUD_HOSTED, }; +use windmill_object_store::upload_artifact_to_store; use windmill_common::{ db::UserDB, @@ -497,7 +497,7 @@ async fn create_snapshot_script( uploaded = true; - let path = windmill_common::s3_helpers::bundle(&w_id, &hash); + let path = windmill_object_store::bundle(&w_id, &hash); upload_artifact_to_store( &path, data, diff --git a/backend/windmill-api-settings/Cargo.toml b/backend/windmill-api-settings/Cargo.toml index c2efd408b2..2323cddde7 100644 --- a/backend/windmill-api-settings/Cargo.toml +++ b/backend/windmill-api-settings/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] enterprise = [] private = ["windmill-common/private"] -parquet = ["dep:object_store", "windmill-common/parquet"] +parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] license = ["dep:rsa"] [dependencies] @@ -36,4 +36,4 @@ uuid.workspace = true base64.workspace = true sha2.workspace = true rsa = { workspace = true, optional = true } -object_store = { workspace = true, optional = true } +windmill-object-store.workspace = true diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a36ca3a467..5478231102 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -152,10 +152,10 @@ pub async fn test_email( } #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::ObjectSettings; +use windmill_object_store::ObjectSettings; #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::build_object_store_from_settings; +use windmill_object_store::build_object_store_from_settings; #[cfg(feature = "parquet")] pub async fn test_s3_bucket( @@ -170,7 +170,7 @@ pub async fn test_s3_bucket( .await? .store; - let mut list = client.list(Some(&object_store::path::Path::from("".to_string()))); + let mut list = client.list(Some(&windmill_object_store::object_store_reexports::Path::from("".to_string()))); let first_file = list.next().await; if first_file.is_some() { if let Err(e) = first_file.as_ref().unwrap() { @@ -182,13 +182,13 @@ pub async fn test_s3_bucket( tracing::info!("No files in blob storage"); } - let path = object_store::path::Path::from(format!( + let path = windmill_object_store::object_store_reexports::Path::from(format!( "/test-s3-bucket-{uuid}", uuid = uuid::Uuid::new_v4() )); tracing::info!("Testing blob storage at path: {path}"); client - .put(&path, object_store::PutPayload::from_static(b"hello")) + .put(&path, windmill_object_store::object_store_reexports::PutPayload::from_static(b"hello")) .await .map_err(|e| anyhow::anyhow!("error writing file to {path}: {e:#}"))?; let content = client diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 1caf13e7f6..b698426d53 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -17,6 +17,7 @@ no_auth = ["windmill-api-auth/no_auth"] [dependencies] windmill-common = { workspace = true, default-features = false } +windmill-types.workspace = true windmill-api-auth.workspace = true windmill-api-users.workspace = true windmill-audit.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c6b756bd29..aa1cd45d42 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -30,7 +30,7 @@ use uuid::Uuid; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; -use windmill_common::s3_helpers::LargeFileStorage; +use windmill_types::s3::LargeFileStorage; use windmill_common::users::username_to_permissioned_as; use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index b2f51f880d..66b002043e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,15 +10,15 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-sandbox/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-sandbox/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] stripe = [] inline_preview = ["dep:windmill-worker", "windmill-api-configs/inline_preview"] agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-sandbox/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] tantivy = ["dep:windmill-indexer"] @@ -58,7 +58,8 @@ windmill-trigger.workspace = true windmill-store.workspace = true windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } -windmill-sandbox.workspace = true +windmill-types.workspace = true +windmill-object-store.workspace = true windmill-audit.workspace = true windmill-parser.workspace = true windmill-parser-sql.workspace = true @@ -130,7 +131,6 @@ async-recursion.workspace = true rsa = { workspace = true, optional = true} uuid.workspace = true datafusion = { workspace = true, optional = true} -object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} jsonwebtoken = { workspace = true } matchit = { workspace = true, optional = true } diff --git a/backend/windmill-api/build.rs b/backend/windmill-api/build.rs new file mode 100644 index 0000000000..768df207fb --- /dev/null +++ b/backend/windmill-api/build.rs @@ -0,0 +1,11 @@ +fn main() { + // rust-embed requires the embedded folder to exist at compile time. + // When building without a prior frontend build, create the directory + // so the derive macro doesn't panic. + let dir = std::env::var("FRONTEND_BUILD_DIR") + .unwrap_or_else(|_| "../../frontend/build/".to_string()); + let path = std::path::Path::new(&dir); + if !path.exists() { + std::fs::create_dir_all(path).ok(); + } +} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 948dcc5ace..8874e092d8 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -39,7 +39,7 @@ use itertools::Itertools; use lazy_static::lazy_static; use magic_crypt::MagicCryptTrait; #[cfg(feature = "parquet")] -use object_store::{Attribute, Attributes}; +use windmill_object_store::object_store_reexports::{Attribute, Attributes}; #[cfg(feature = "parquet")] use regex::Regex; use serde::{Deserialize, Serialize}; @@ -78,9 +78,10 @@ use hmac::Mac; use windmill_common::{ jwt, oauth2::HmacSha256, - s3_helpers::{build_object_store_client, S3Object, S3Permission}, variables::get_workspace_key, }; +#[cfg(feature = "parquet")] +use windmill_types::s3::{S3Object, S3Permission}; pub fn workspaced_service() -> Router { Router::new() @@ -406,7 +407,7 @@ async fn get_raw_app_data( Extension(db): Extension, ) -> Result { #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::get_object_store().await; + let object_store = windmill_object_store::get_object_store().await; // tracing::info!("secret_with_ext: {}", secret_with_ext); let mut splitted = secret_with_ext.split('.'); @@ -442,10 +443,12 @@ async fn get_raw_app_data( if let Some(os) = object_store { let path = format!("/app_bundles/{}/{}.{}", w_id, id, file_type); let stream = os - .get(&object_store::path::Path::from(path)) - .await? + .get(&windmill_object_store::object_store_reexports::Path::from(path)) + .await + .map_err(windmill_object_store::object_store_error_to_error)? .bytes() - .await?; + .await + .map_err(windmill_object_store::object_store_error_to_error)?; tracing::info!("stream: {}", stream.len()); body = Some(Body::from(stream)); } @@ -949,13 +952,13 @@ async fn store_raw_app_file<'a>( ) -> Result<()> { #[cfg(all(feature = "enterprise", feature = "parquet"))] { - let object_store = windmill_common::s3_helpers::get_object_store().await; + let object_store = windmill_object_store::get_object_store().await; let path: String = format!("/app_bundles/{}/{}.{}", w_id, id, file_type); if let Some(os) = object_store { if let Err(e) = os - .put(&object_store::path::Path::from(path.clone()), data.into()) + .put(&windmill_object_store::object_store_reexports::Path::from(path.clone()), data.into()) .await { tracing::error!("Failed to put snapshot to s3 at {path}: {:?}", e); @@ -2480,7 +2483,7 @@ async fn upload_s3_file_from_app( let s3_resource = s3_resource_opt.ok_or(Error::internal_err( "No files storage resource defined at the workspace level".to_string(), ))?; - let s3_client = build_object_store_client(&s3_resource).await?; + let s3_client = windmill_object_store::build_object_store_client(&s3_resource).await?; let options = Attributes::from_iter(vec![ ( @@ -2537,7 +2540,7 @@ async fn delete_s3_file_from_app( .. } = jwt::decode_with_internal_secret::(&query.delete_token).await?; - let path = object_store::path::Path::parse(file_key.as_str()) + let path = windmill_object_store::object_store_reexports::Path::parse(file_key.as_str()) .map_err(|e| Error::internal_err(format!("Error parsing file key: {}", e)))?; if workspace != w_id { @@ -2581,7 +2584,7 @@ async fn delete_s3_file_from_app( ))? }; - let s3_client = build_object_store_client(&s3_resource).await?; + let s3_client = windmill_object_store::build_object_store_client(&s3_resource).await?; s3_client.delete(&path).await.map_err(|err| { tracing::error!("Error deleting file: {:?}", err); diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index bcbfba03d0..c71fff0acb 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -91,8 +91,8 @@ impl RawWebhookArgs { get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; use futures::TryStreamExt; - use object_store::{Attribute, Attributes}; - use windmill_common::s3_helpers::build_object_store_client; + use windmill_object_store::object_store_reexports::{Attribute, Attributes}; + use windmill_object_store::build_object_store_client; let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, w_id, None).await?; diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 6f0f6d851d..01f5d67bd3 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -7,18 +7,20 @@ use axum::Router; #[cfg(not(feature = "private"))] use uuid::Uuid; #[cfg(not(feature = "private"))] -use windmill_common::s3_helpers::StorageResourceType; +use windmill_types::s3::StorageResourceType; #[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, OptJobAuthed, DB}; #[cfg(all(feature = "parquet", not(feature = "private")))] -use object_store::{ObjectStore, PutMultipartOpts, PutResult}; +use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; +#[cfg(not(feature = "private"))] +use windmill_object_store::ObjectStoreResource; #[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; #[cfg(not(feature = "private"))] use windmill_common::error; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource}; +use windmill_common::db::UserDB; #[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; @@ -155,11 +157,11 @@ pub async fn get_workspace_s3_resource_and_check_paths<'c>( _authed_api: Option<&ApiAuthed>, _w_id: &str, _storage: Option, - _paths: &[(&str, windmill_common::s3_helpers::S3Permission)], + _paths: &[(&str, windmill_types::s3::S3Permission)], _job_id: Option, ) -> windmill_common::error::Result<( Option, - Option, + Option, )> { Err(windmill_common::error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 8bc597621e..960ad29d71 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -42,7 +42,8 @@ use windmill_common::runnable_settings::{ }; #[cfg(feature = "inline_preview")] use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams}; -use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; +use windmill_types::s3::BundleFormat; +use windmill_object_store::upload_artifact_to_store; use windmill_common::scripts::ScriptRunnableSettingsInline; use windmill_common::triggers::TriggerMetadata; use windmill_common::utils::{RunnableKind, WarnAfterExt}; @@ -1384,40 +1385,15 @@ async fn get_logs_from_store( logs: &str, log_file_index: &Option>, ) -> Option> { - if log_offset > 0 { - if let Some(file_index) = log_file_index.clone() { - tracing::debug!("Getting logs from store: {file_index:?}"); - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - tracing::debug!("object store client present, streaming from there"); - - let logs = logs.to_string(); - let stream = async_stream::stream! { - yield Ok(bytes::Bytes::from( - r#"to remove ansi colors, use: | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' - "# - .to_string(), - )); - for file_p in file_index.clone() { - let file_p_2 = file_p.clone(); - let file = os.get(&object_store::path::Path::from(file_p)).await; - if let Ok(file) = file { - if let Ok(bytes) = file.bytes().await { - yield Ok(bytes::Bytes::from(bytes)) as object_store::Result; - } - } else { - tracing::debug!("error getting file from store: {file_p_2}: {}", file.err().unwrap()); - } - } - - yield Ok(bytes::Bytes::from(logs)) - }; - return Some(Ok(Body::from_stream(stream))); - } else { - tracing::debug!("object store client not present, cannot stream logs from store"); - } - } - } - return None; + use futures::StreamExt; + let stream = windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await?; + let header = bytes::Bytes::from( + r#"to remove ansi colors, use: | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' +"# + .to_string(), + ); + let prefixed_stream = futures::stream::once(async { Ok(header) }).chain(stream); + Some(Ok(Body::from_stream(prefixed_stream))) } async fn get_logs_from_disk( @@ -4840,7 +4816,7 @@ async fn run_bundle_preview_script( uploaded = true; - let path = windmill_common::s3_helpers::bundle(&w_id, &id); + let path = windmill_object_store::bundle(&w_id, &id); upload_artifact_to_store( &path, data, @@ -5794,9 +5770,9 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + if let Some(os) = windmill_object_store::get_object_store().await { let file = os - .get(&object_store::path::Path::from(format!("logs/{file_p}"))) + .get(&windmill_object_store::object_store_reexports::Path::from(format!("logs/{file_p}"))) .await; if let Ok(file) = file { if let Ok(bytes) = file.bytes().await { diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 0b3ca56af7..20d2b58d0e 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -98,11 +98,11 @@ async fn get_log_file( require_devops_role(&db, &email).await?; let path = path.to_path(); #[cfg(feature = "parquet")] - let s3_client = windmill_common::s3_helpers::get_object_store().await; + let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client.get(&object_store::path::Path::from(path)).await; + let file = s3_client.get(&windmill_object_store::object_store_reexports::Path::from(path)).await; match file { Ok(file) => { let bytes = file.bytes().await; diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index 540a1c148b..f3ec05348a 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -35,7 +35,7 @@ use windmill_trigger::TriggerMode; #[cfg(feature = "parquet")] use { crate::job_helpers_oss::get_workspace_s3_resource, - windmill_common::s3_helpers::build_object_store_client, + windmill_object_store::build_object_store_client, }; async fn conditional_cors_middleware( @@ -365,13 +365,13 @@ async fn route_job( } else { config.s3.clone() }; - let path = object_store::path::Path::from(path); + let path = windmill_object_store::object_store_reexports::Path::from(path); let s3_object = s3_client.get(&path).await; let s3_object = match s3_object { - Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => { + Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => { // fallback to index.html if the file is not found - let path = object_store::path::Path::from(format!( + let path = windmill_object_store::object_store_reexports::Path::from(format!( "{}/index.html", config.s3.trim_end_matches('/') )); @@ -410,7 +410,7 @@ async fn route_job( "content-type", s3_object .attributes - .get(&object_store::Attribute::ContentType) + .get(&windmill_object_store::object_store_reexports::Attribute::ContentType) .map(|s| s.parse().ok()) .flatten() .unwrap_or("application/octet-stream".parse().unwrap()), @@ -422,7 +422,7 @@ async fn route_job( || { s3_object .attributes - .get(&object_store::Attribute::ContentDisposition) + .get(&windmill_object_store::object_store_reexports::Attribute::ContentDisposition) .map(|s| s.parse().ok()) .flatten() .unwrap_or("inline".parse().unwrap()) diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index d7e79cfcd4..636061571e 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -14,7 +14,7 @@ jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] prometheus = ["dep:prometheus"] benchmark = [] -parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion", "dep:aws-config", "dep:aws-credential-types"] +parquet = [] aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic", "dep:opentelemetry"] @@ -64,7 +64,6 @@ regex.workspace = true git-version.workspace = true cron.workspace = true magic-crypt.workspace = true -object_store = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } aws-sdk-sts = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/ai_types.rs b/backend/windmill-common/src/ai_types.rs index 2d72190f5b..707fab0c2a 100644 --- a/backend/windmill-common/src/ai_types.rs +++ b/backend/windmill-common/src/ai_types.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use crate::flow_status::AgentAction; -use crate::s3_helpers::S3Object; +use windmill_types::s3::S3Object; // ============================================================================ // Shared Types for OpenAI-compatible message format diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index c5e4414fce..eaf7d857c1 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -313,42 +313,3 @@ where Self(err.into()) } } -#[cfg(feature = "parquet")] -impl From for Error { - fn from(err: object_store::Error) -> Self { - use object_store::Error::*; - match err { - Generic { store, source } => Error::Generic( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Generic {} error: {}", store, source), - ), - NotFound { path, source } => Error::NotFound(format!("{}: {}", path, source)), - InvalidPath { source } => Error::BadRequest(format!("Invalid path: {}", source)), - JoinError { source } => Error::InternalErr(format!("Join error: {}", source)), - NotSupported { source } => { - Error::BadRequest(format!("Operation not supported: {}", source)) - } - AlreadyExists { path, source } => { - Error::BadRequest(format!("Object at {} already exists: {}", path, source)) - } - Precondition { path, source } => { - Error::BadRequest(format!("Precondition failed at {}: {}", path, source)) - } - NotModified { path, source } => { - Error::ExecutionErr(format!("Not modified at {}: {}", path, source)) - } - NotImplemented => Error::BadRequest("Operation not yet implemented.".to_string()), - PermissionDenied { path, source } => { - Error::PermissionDenied(format!("Permission denied at {}: {}", path, source)) - } - Unauthenticated { path, source } => { - Error::NotAuthorized(format!("Unauthenticated for {}: {}", path, source)) - } - UnknownConfigurationKey { store, key } => Error::BadConfig(format!( - "Invalid config key '{}' for store '{}'", - key, store - )), - _ => Error::InternalErr(format!("Object store error: {}", err)), - } - } -} diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 9362f7ff02..12004b945d 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -250,42 +250,6 @@ pub async fn get_logs_from_disk( return None; } -#[cfg(all(feature = "enterprise", feature = "parquet"))] -pub async fn get_logs_from_store( - log_offset: i32, - logs: &str, - log_file_index: &Option>, -) -> Option>> { - use crate::s3_helpers::get_object_store; - - if log_offset > 0 { - if let Some(file_index) = log_file_index.clone() { - if let Some(os) = get_object_store().await { - let logs = logs.to_string(); - let stream = async_stream::stream! { - for file_p in file_index.clone() { - let file_p_2 = file_p.clone(); - let file = os.get(&object_store::path::Path::from(file_p)).await; - if let Ok(file) = file { - if let Ok(bytes) = file.bytes().await { - yield Ok(bytes::Bytes::from(bytes)) as object_store::Result; - } - } else { - tracing::debug!("error getting file from store: {file_p_2}: {}", file.err().unwrap()); - } - } - - yield Ok(bytes::Bytes::from(logs)) - }; - return Some(stream); - } else { - tracing::debug!("object store client not present, cannot stream logs from store"); - } - } - } - return None; -} - lazy_static::lazy_static! { pub static ref TAGS_ARE_SENSITIVE: bool = std::env::var("TAGS_ARE_SENSITIVE").map( |v| v.parse().unwrap() diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 805bbb2c82..ee9c25a3e5 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -57,10 +57,6 @@ pub mod global_settings; pub mod indexer; pub mod instance_config; pub mod job_metrics; -#[cfg(all(feature = "parquet", feature = "private"))] -pub mod job_s3_helpers_ee; -#[cfg(feature = "parquet")] -pub mod job_s3_helpers_oss; pub mod min_version; pub mod notify_events; pub mod runtime_assets; @@ -83,7 +79,6 @@ pub mod otel_oss; pub mod queue; pub mod result_stream; pub mod runnable_settings; -pub mod s3_helpers; pub mod schedule; pub mod schema; pub mod scripts; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 0ffb9a9272..f80c78f244 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -902,137 +902,6 @@ pub fn copy_dir_recursively(src: &Path, dst: &Path) -> error::Result<()> { Ok(()) } -pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) { - if tokio::fs::metadata(&bin_path).await.is_ok() { - (true, format!("loaded from local cache: {}\n", bin_path)) - } else { - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = crate::s3_helpers::get_object_store().await { - let started = std::time::Instant::now(); - use crate::s3_helpers::attempt_fetch_bytes; - - if let Ok(mut x) = attempt_fetch_bytes(os, _remote_path).await { - if is_dir { - if let Err(e) = extract_tar(x, bin_path).await { - tracing::error!("could not write tar archive locally: {e:?}"); - return ( - false, - "error writing tar archive from object store".to_string(), - ); - } - } else { - if let Err(e) = write_binary_file(bin_path, &mut x) { - tracing::error!("could not write bundle/bin file locally: {e:?}"); - return ( - false, - "error writing bundle/bin file from object store".to_string(), - ); - } - } - tracing::info!("loaded from object store {}", bin_path); - return ( - true, - format!( - "loaded bin/bundle from object store {} in {}ms", - bin_path, - started.elapsed().as_millis() - ), - ); - } - } - let _ = is_dir; - (false, "".to_string()) - } -} - -pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { - if tokio::fs::metadata(&bin_path).await.is_ok() { - return true; - } else { - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = crate::s3_helpers::get_object_store().await { - return os - .get(&object_store::path::Path::from(_remote_path)) - .await - .is_ok(); - } - return false; - } -} - -pub async fn save_cache( - local_cache_path: &str, - _remote_cache_path: &str, - origin: &str, - is_dir: bool, -) -> crate::error::Result { - let mut _cached_to_s3 = false; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = crate::s3_helpers::get_object_store().await { - use object_store::path::Path; - let file_to_cache = if is_dir { - let tar_path = format!( - "{ROOT_CACHE_DIR}/tar/{}_tar.tar", - local_cache_path - .split("/") - .last() - .unwrap_or(&uuid::Uuid::new_v4().to_string()) - ); - let tar_file = std::fs::File::create(&tar_path)?; - let mut tar = tar::Builder::new(tar_file); - tar.append_dir_all(".", &origin)?; - let tar_metadata = tokio::fs::metadata(&tar_path).await; - if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 { - tracing::info!("Failed to tar cache: {origin}"); - return Err(error::Error::ExecutionErr(format!( - "Failed to tar cache: {origin}" - ))); - } - tar_path - } else { - origin.to_owned() - }; - - if let Err(e) = os - .put( - &Path::from(_remote_cache_path), - std::fs::read(&file_to_cache)?.into(), - ) - .await - { - tracing::error!( - "Failed to put go bin to object store: {_remote_cache_path}. Error: {:?}", - e - ); - } else { - _cached_to_s3 = true; - if is_dir { - tokio::fs::remove_dir_all(&file_to_cache).await?; - } - } - } - - // if !*CLOUD_HOSTED { - if true { - if is_dir { - copy_dir_recursively(&PathBuf::from(origin), &PathBuf::from(local_cache_path))?; - } else { - std::fs::copy(origin, local_cache_path)?; - } - Ok(format!( - "\nwrote cached binary: {} (backed by EE distributed object store: {_cached_to_s3})\n", - local_cache_path - )) - } else if _cached_to_s3 { - Ok(format!( - "wrote cached binary to object store {}\n", - local_cache_path - )) - } else { - Ok("".to_string()) - } -} - #[cfg(all(feature = "enterprise", feature = "parquet"))] pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { use std::time::Instant; @@ -1059,7 +928,7 @@ pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { Ok(()) } #[cfg(all(feature = "enterprise", feature = "parquet"))] -fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { +pub fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { use std::fs::File; use std::io::Write; diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index d3a1687b00..acf90ab57a 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -10,12 +10,13 @@ path = "src/lib.rs" [features] default = [] -parquet = ["dep:object_store", "windmill-common/parquet"] +parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] private = ["windmill-common/private"] -enterprise = ["windmill-common/enterprise"] +enterprise = ["windmill-common/enterprise", "windmill-object-store/enterprise"] [dependencies] windmill-common.workspace = true +windmill-object-store.workspace = true tantivy.workspace = true tokio.workspace = true sqlx.workspace = true @@ -28,7 +29,6 @@ uuid.workspace = true futures.workspace = true tempfile.workspace = true bytes.workspace = true -object_store = { workspace = true, optional = true} astral-tokio-tar.workspace = true lazy_static.workspace = true const_format.workspace = true diff --git a/backend/windmill-object-store/Cargo.toml b/backend/windmill-object-store/Cargo.toml new file mode 100644 index 0000000000..3602007d8c --- /dev/null +++ b/backend/windmill-object-store/Cargo.toml @@ -0,0 +1,60 @@ +[package] +name = "windmill-object-store" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[features] +default = [] +private = [] +enterprise = [] +parquet = [ + "windmill-common/parquet", + "dep:object_store", + "dep:aws-sdk-sts", + "dep:aws-smithy-types-convert", + "dep:datafusion", + "dep:aws-config", + "dep:aws-credential-types", + "dep:windmill-parser-sql", +] +openidconnect = ["windmill-common/openidconnect"] + +[lib] +name = "windmill_object_store" +path = "src/lib.rs" + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-types.workspace = true +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tracing.workspace = true +reqwest.workspace = true +uuid.workspace = true +chrono.workspace = true +bytes.workspace = true +lazy_static.workspace = true +quick_cache.workspace = true +async-trait.workspace = true +futures.workspace = true +async-stream.workspace = true +windmill-parser-sql = { workspace = true, optional = true } +globset.workspace = true +axum.workspace = true + +object_store = { workspace = true, optional = true } +aws-sdk-sts = { workspace = true, optional = true } +aws-smithy-types-convert = { workspace = true, optional = true } +datafusion = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-credential-types = { workspace = true, optional = true } + +[dev-dependencies] +tempfile.workspace = true +tokio = { workspace = true, features = ["rt", "macros"] } +object_store.workspace = true diff --git a/backend/windmill-common/src/job_s3_helpers_oss.rs b/backend/windmill-object-store/src/job_s3_helpers_oss.rs similarity index 69% rename from backend/windmill-common/src/job_s3_helpers_oss.rs rename to backend/windmill-object-store/src/job_s3_helpers_oss.rs index b7ebd57f3e..2d16bb3267 100644 --- a/backend/windmill-common/src/job_s3_helpers_oss.rs +++ b/backend/windmill-object-store/src/job_s3_helpers_oss.rs @@ -3,21 +3,21 @@ pub use crate::job_s3_helpers_ee::*; #[cfg(not(feature = "private"))] -use crate::s3_helpers::{ObjectStoreResource, StorageResourceType}; +use windmill_types::s3::{ObjectStoreResource, StorageResourceType}; #[cfg(not(feature = "private"))] pub async fn get_s3_resource_internal<'c>( _resource_type: StorageResourceType, _s3_resource_value_raw: serde_json::Value, _gen_token: TokenGenerator<'c>, - _db: &crate::DB, -) -> crate::error::Result { + _db: &windmill_common::DB, +) -> windmill_common::error::Result { todo!() } #[cfg(not(feature = "private"))] pub enum TokenGenerator<'c> { - AsClient(&'c crate::client::AuthedClient), + AsClient(&'c windmill_common::client::AuthedClient), AsServerInstance(), } @@ -26,7 +26,7 @@ impl<'c> TokenGenerator<'c> { pub async fn gen_token( &self, _audience: &str, - _db: Option<&crate::DB>, + _db: Option<&windmill_common::DB>, ) -> anyhow::Result { todo!() } @@ -34,9 +34,9 @@ impl<'c> TokenGenerator<'c> { #[cfg(all(feature = "parquet", not(feature = "private")))] pub(crate) async fn generate_s3_aws_oidc_resource<'c>( - _clone: crate::s3_helpers::S3AwsOidcResource, + _clone: windmill_types::s3::S3AwsOidcResource, _token_generator: TokenGenerator<'c>, _init_private_key: Option<&sqlx::Pool>, -) -> crate::error::Result { +) -> windmill_common::error::Result { todo!() } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-object-store/src/lib.rs similarity index 54% rename from backend/windmill-common/src/s3_helpers.rs rename to backend/windmill-object-store/src/lib.rs index 9294e4c4c4..1449412a83 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -1,38 +1,21 @@ -use crate::error::{self}; +pub use windmill_types::s3::*; + +use std::collections::HashMap; + +use quick_cache::sync::Cache; +use windmill_common::error::{self}; + +#[cfg(feature = "parquet")] +use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region}; #[cfg(feature = "parquet")] use aws_sdk_sts::config::ProvideCredentials; #[cfg(feature = "parquet")] use axum::async_trait; -use chrono::{DateTime, Utc}; -#[cfg(feature = "parquet")] -use object_store::aws::AwsCredential; -#[cfg(feature = "parquet")] -use object_store::azure::MicrosoftAzureBuilder; -#[cfg(feature = "parquet")] -use object_store::gcp::GoogleCloudStorageBuilder; -#[cfg(feature = "parquet")] -use object_store::ObjectStore; -#[cfg(feature = "parquet")] -use object_store::{aws::AmazonS3Builder, ClientOptions}; -use quick_cache::sync::Cache; -#[cfg(feature = "parquet")] -use reqwest::header::HeaderMap; -use serde::de::Visitor; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::fmt; -#[cfg(feature = "parquet")] -use std::sync::{Arc, Mutex}; - -#[cfg(feature = "parquet")] -use tokio::sync::RwLock; - -#[cfg(feature = "parquet")] -use crate::error::to_anyhow; -#[cfg(feature = "parquet")] -use crate::utils::rd_string; #[cfg(feature = "parquet")] use bytes::Bytes; #[cfg(feature = "parquet")] +use chrono::{DateTime, Utc}; +#[cfg(feature = "parquet")] use datafusion::arrow::array::{RecordBatch, RecordBatchWriter}; #[cfg(feature = "parquet")] use datafusion::arrow::error::ArrowError; @@ -45,93 +28,87 @@ use datafusion::parquet::arrow::ArrowWriter; #[cfg(feature = "parquet")] use futures::TryStreamExt; #[cfg(feature = "parquet")] +use object_store::aws::AwsCredential; +#[cfg(feature = "parquet")] +use object_store::azure::MicrosoftAzureBuilder; +#[cfg(feature = "parquet")] +use object_store::gcp::GoogleCloudStorageBuilder; +#[cfg(feature = "parquet")] +use object_store::CredentialProvider; +#[cfg(feature = "parquet")] +use object_store::ObjectStore; +#[cfg(feature = "parquet")] +use object_store::{aws::AmazonS3Builder, ClientOptions}; +#[cfg(feature = "parquet")] +use reqwest::header::HeaderMap; +#[cfg(feature = "parquet")] use std::io::Write; #[cfg(feature = "parquet")] +use std::sync::{Arc, Mutex}; +#[cfg(feature = "parquet")] +use tokio::sync::RwLock; +#[cfg(feature = "parquet")] use tokio::task; #[cfg(feature = "parquet")] -use windmill_parser_sql::S3ModeFormat; +use windmill_common::error::to_anyhow; +#[cfg(feature = "parquet")] +use windmill_common::utils::rd_string; +#[cfg(all(feature = "parquet", feature = "private"))] +pub mod job_s3_helpers_ee; +#[cfg(feature = "parquet")] +pub mod job_s3_helpers_oss; -use std::collections::HashMap; - -lazy_static::lazy_static! { - static ref S3_BUCKET_RESTRICTIONS: Option>> = { - parse_bucket_restrictions() +// Re-export object_store types so consumers don't need a direct object_store dep +#[cfg(feature = "parquet")] +pub mod object_store_reexports { + pub use object_store::local::LocalFileSystem; + pub use object_store::memory::InMemory; + pub use object_store::path::Path; + pub use object_store::{ + Attribute, Attributes, Error as ObjectStoreError, GetResult, ObjectStore, + PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, WriteMultipart, }; } -/// Parses the S3_BUCKETS_WORKSPACE_RESTRICTIONS environment variable -/// Format: bucket_a:workspace1,workspace2;bucket_b:workspace3,workspace4 -fn parse_bucket_restrictions() -> Option>> { - let env_var = std::env::var("S3_BUCKETS_WORKSPACE_RESTRICTIONS").ok()?; - - if env_var.trim().is_empty() { - return None; - } - - let mut restrictions = HashMap::new(); - - for bucket_rule in env_var.split(';') { - let bucket_rule = bucket_rule.trim(); - if bucket_rule.is_empty() { - continue; +#[cfg(feature = "parquet")] +pub fn object_store_error_to_error(err: object_store::Error) -> error::Error { + use object_store::Error::*; + match err { + Generic { store, source } => error::Error::Generic( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + format!("Generic {} error: {}", store, source), + ), + NotFound { path, source } => error::Error::NotFound(format!("{}: {}", path, source)), + InvalidPath { source } => error::Error::BadRequest(format!("Invalid path: {}", source)), + JoinError { source } => error::Error::InternalErr(format!("Join error: {}", source)), + NotSupported { source } => { + error::Error::BadRequest(format!("Operation not supported: {}", source)) } - - let parts: Vec<&str> = bucket_rule.splitn(2, ':').collect(); - if parts.len() != 2 { - tracing::warn!( - "Invalid bucket restriction format: '{}'. Expected 'bucket:workspace1,workspace2'", - bucket_rule - ); - continue; + AlreadyExists { path, source } => { + error::Error::BadRequest(format!("Object at {} already exists: {}", path, source)) } - - let bucket_name = parts[0].trim().to_string(); - let workspaces: Vec = parts[1] - .split(',') - .map(|w| w.trim().to_string()) - .filter(|w| !w.is_empty()) - .collect(); - - if workspaces.is_empty() { - tracing::warn!( - "No workspaces specified for bucket '{}', skipping restriction", - bucket_name - ); - continue; + Precondition { path, source } => { + error::Error::BadRequest(format!("Precondition failed at {}: {}", path, source)) } - - restrictions.insert(bucket_name, workspaces); - } - - if restrictions.is_empty() { - None - } else { - tracing::info!( - "S3 bucket restrictions loaded for {} buckets", - restrictions.len() - ); - Some(restrictions) + NotModified { path, source } => { + error::Error::ExecutionErr(format!("Not modified at {}: {}", path, source)) + } + NotImplemented => error::Error::BadRequest("Operation not yet implemented.".to_string()), + PermissionDenied { path, source } => { + error::Error::PermissionDenied(format!("Permission denied at {}: {}", path, source)) + } + Unauthenticated { path, source } => { + error::Error::NotAuthorized(format!("Unauthenticated for {}: {}", path, source)) + } + UnknownConfigurationKey { store, key } => error::Error::BadConfig(format!( + "Invalid config key '{}' for store '{}'", + key, store + )), + _ => error::Error::InternalErr(format!("Object store error: {}", err)), } } -/// Checks if a workspace is allowed to access a given bucket based on restrictions -/// Returns Ok(()) if access is allowed, Err if restricted -pub fn check_bucket_workspace_restriction( - bucket_name: &str, - workspace_id: &str, -) -> error::Result<()> { - if let Some(ref restrictions) = *S3_BUCKET_RESTRICTIONS { - if let Some(allowed_workspaces) = restrictions.get(bucket_name) { - if !allowed_workspaces.contains(&workspace_id.to_string()) { - return Err(error::Error::NotAuthorized(format!( - "Workspace '{}' is not authorized to access bucket '{}'", - workspace_id, bucket_name - ))); - } - } - } - Ok(()) -} +// --- Object store builder infrastructure (moved from windmill-common/src/s3_helpers.rs) --- #[cfg(feature = "parquet")] #[derive(Clone)] @@ -179,14 +156,6 @@ impl From> for ExpirableObjectStore { } } -// #[cfg(feature = "parquet")] - -// impl ExpirableObjectStore { -// pub fn new(store: Arc, expiration: Option>) -> Self { -// Self { store, expiration } -// } -// } - #[cfg(feature = "parquet")] lazy_static::lazy_static! { pub static ref OBJECT_STORE_SETTINGS: Arc>> = Arc::new(RwLock::new(None)); @@ -225,17 +194,15 @@ pub async fn get_object_store() -> Option> { #[cfg(feature = "parquet")] pub enum ObjectStoreReload { - //if the jwks endpoints are not up yet, we should retry later soon Later, Never, } #[cfg(feature = "parquet")] -pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload { - use crate::{ +pub async fn reload_object_store_setting(db: &windmill_common::DB) -> ObjectStoreReload { + use windmill_common::{ ee_oss::{get_license_plan, LicensePlan}, global_settings::{load_value_from_global_settings, OBJECT_STORE_CONFIG_SETTING}, - s3_helpers::ObjectSettings, }; let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CONFIG_SETTING).await; @@ -300,279 +267,6 @@ pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload { return ObjectStoreReload::Never; } -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "type")] -pub enum LargeFileStorage { - S3Storage(S3Storage), - AzureBlobStorage(AzureBlobStorage), - S3AwsOidc(S3Storage), - AzureWorkloadIdentity(AzureBlobStorage), - GoogleCloudStorage(GoogleCloudStorage), - FilesystemStorage(FilesystemSettings), -} - -impl LargeFileStorage { - pub fn get_s3_resource_path(&self) -> &str { - match self { - LargeFileStorage::S3Storage(s3_lfs) => &s3_lfs.s3_resource_path, - LargeFileStorage::S3AwsOidc(s3_lfs) => &s3_lfs.s3_resource_path, - LargeFileStorage::AzureBlobStorage(az_lfs) => &az_lfs.azure_blob_resource_path, - LargeFileStorage::AzureWorkloadIdentity(az_lfs) => &az_lfs.azure_blob_resource_path, - LargeFileStorage::GoogleCloudStorage(gcs_lfs) => &gcs_lfs.gcs_resource_path, - LargeFileStorage::FilesystemStorage(fs) => &fs.root, - } - } - pub fn is_public_resource(&self) -> bool { - match self { - LargeFileStorage::S3Storage(lfs) => lfs.public_resource, - LargeFileStorage::S3AwsOidc(lfs) => lfs.public_resource, - LargeFileStorage::AzureBlobStorage(lfs) => lfs.public_resource, - LargeFileStorage::AzureWorkloadIdentity(lfs) => lfs.public_resource, - LargeFileStorage::GoogleCloudStorage(glfs) => glfs.public_resource, - LargeFileStorage::FilesystemStorage(_) => Some(false), - } - .unwrap_or(false) - } - pub fn get_advanced_permissions(&self) -> Option<&Vec> { - match self { - LargeFileStorage::S3Storage(lfs) => lfs.advanced_permissions.as_ref(), - LargeFileStorage::S3AwsOidc(lfs) => lfs.advanced_permissions.as_ref(), - LargeFileStorage::AzureBlobStorage(lfs) => lfs.advanced_permissions.as_ref(), - LargeFileStorage::AzureWorkloadIdentity(lfs) => lfs.advanced_permissions.as_ref(), - LargeFileStorage::GoogleCloudStorage(glfs) => glfs.advanced_permissions.as_ref(), - LargeFileStorage::FilesystemStorage(_) => None, - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct S3Storage { - pub s3_resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub public_resource: Option, - pub advanced_permissions: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct AzureBlobStorage { - pub azure_blob_resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub public_resource: Option, - pub advanced_permissions: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct GoogleCloudStorage { - pub gcs_resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub public_resource: Option, - pub advanced_permissions: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] -pub struct S3PermissionRule { - pub pattern: String, - pub allow: S3Permission, // read, write, delete, list -} -bitflags::bitflags! { - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub struct S3Permission: u8 { - const READ = 0b0001; - const WRITE = 0b0010; - const DELETE = 0b0100; - const LIST = 0b1000; - } -} - -impl Serialize for S3Permission { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut perms = Vec::new(); - if self.contains(S3Permission::READ) { - perms.push("read"); - } - if self.contains(S3Permission::WRITE) { - perms.push("write"); - } - if self.contains(S3Permission::DELETE) { - perms.push("delete"); - } - if self.contains(S3Permission::LIST) { - perms.push("list"); - } - let perms = perms.join(","); - perms.serialize(serializer) - } -} -impl<'de> Deserialize<'de> for S3Permission { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct PermVisitor; - impl<'de> Visitor<'de> for PermVisitor { - type Value = S3Permission; - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("comma separated list of permissions: read, write, delete, list") - } - fn visit_str(self, v: &str) -> Result { - let mut perms = S3Permission::empty(); - for value in v.split(',') { - perms |= match value { - "read" => S3Permission::READ, - "write" => S3Permission::WRITE, - "delete" => S3Permission::DELETE, - "list" => S3Permission::LIST, - _ => S3Permission::empty(), // ignore unknown permissions - }; - } - Ok(perms) - } - } - - deserializer.deserialize_str(PermVisitor) - } -} - -#[derive(Clone, Debug)] -pub enum ObjectStoreResource { - S3(S3Resource), - Azure(AzureBlobResource), - Gcs(GcsResource), -} - -impl ObjectStoreResource { - pub fn expiration(&self) -> Option> { - match self { - ObjectStoreResource::S3(s3_resource) => s3_resource.expiration, - _ => None, - } - } -} - -#[derive(Deserialize, Debug)] -pub enum StorageResourceType { - S3, - AzureBlob, - S3AwsOidc, - AzureWorkloadIdentity, - GoogleCloudStorage, -} - -#[derive(Debug, Deserialize, Serialize, Clone)] -pub struct S3Resource { - pub bucket: String, - pub region: String, - #[serde(rename = "endPoint")] - pub endpoint: String, - #[serde(rename = "useSSL")] - pub use_ssl: bool, - #[serde(rename = "accessKey")] - pub access_key: Option, - #[serde(rename = "secretKey")] - pub secret_key: Option, - #[serde(rename = "pathStyle")] - pub path_style: Option, - pub token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub expiration: Option>, - pub port: Option, -} - -impl S3Resource { - pub fn endpoint_with_region_fallback(&self, region_fallback: Option) -> String { - if self.endpoint.is_empty() { - let final_region = if self.region.is_empty() { - region_fallback.unwrap_or_else(|| "us-east-1".to_string()) - } else { - self.region.clone() - }; - format!("s3.{}.amazonaws.com", final_region) - } else { - self.endpoint.clone() - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone)] -pub struct AzureBlobResource { - pub endpoint: Option, - #[serde(rename = "useSSL")] - pub use_ssl: Option, - #[serde(rename = "accountName")] - pub account_name: String, - #[serde(rename = "tenantId")] - pub tenant_id: Option, - #[serde(rename = "clientId")] - pub client_id: Option, - #[serde(rename = "containerName")] - pub container_name: String, - #[serde(rename = "accessKey")] - pub access_key: Option, - #[serde(rename = "federatedTokenFile")] - pub federated_token_file: Option, -} - -fn as_string<'de, D>(deserializer: D) -> Result -where - D: serde::de::Deserializer<'de>, -{ - let v: serde_json::Value = Deserialize::deserialize(deserializer)?; - serde_json::to_string(&v).map_err(serde::de::Error::custom) -} - -#[derive(Debug, Deserialize, Clone)] -pub struct GcsResource { - pub bucket: String, - #[serde(rename = "serviceAccountKey")] - #[serde(deserialize_with = "as_string")] - pub service_account_key: String, -} - -#[derive(Debug, Deserialize, Serialize, Clone, Hash)] -pub struct S3AwsOidcResource { - #[serde(rename = "bucket")] - pub bucket: String, - pub region: Option, - #[serde(rename = "roleArn")] - pub role_arn: String, - pub audience: Option, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Default)] -pub struct S3Object { - pub s3: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub storage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub filename: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub presigned: Option, -} - -#[cfg(feature = "parquet")] -pub async fn get_etag_or_empty( - object_store_resource: &ObjectStoreResource, - s3_object: S3Object, -) -> Option { - let object_store_client = build_object_store_client(object_store_resource).await; - if object_store_client.is_err() { - return None; - } - - let object_key = object_store::path::Path::from(s3_object.s3); - - return object_store_client - .unwrap() - .head(&object_key) - .await - .ok() - .map(|meta| meta.e_tag) - .flatten(); -} - pub fn render_endpoint( raw_endpoint: String, use_ssl: bool, @@ -612,178 +306,10 @@ pub async fn build_object_store_client( build_azure_blob_client(&azure_blob_resource_ref) } ObjectStoreResource::Gcs(gcs_resource_ref) => build_gcs_client(&gcs_resource_ref).await, + ObjectStoreResource::Filesystem(fs) => build_filesystem_client(&fs.root_path), } } -/// Build an object store client from the workspace's `large_file_storage` setting. -/// Supports FilesystemStorage directly; for S3/Azure/GCS, resolves the resource -/// value from the DB (admin-only, no permission checks). -#[cfg(feature = "parquet")] -pub async fn get_workspace_object_store( - db: &crate::DB, - w_id: &str, -) -> error::Result> { - let raw_lfs = sqlx::query_scalar!( - "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", - w_id - ) - .fetch_optional(db) - .await? - .flatten() - .ok_or_else(|| { - error::Error::NotFound(format!( - "No large file storage configured for workspace {w_id}" - )) - })?; - - let lfs: LargeFileStorage = serde_json::from_value(raw_lfs).map_err(|e| { - error::Error::internal_err(format!("Failed to parse workspace storage config: {e}")) - })?; - - match lfs { - LargeFileStorage::FilesystemStorage(fs_settings) => { - let root = expand_home(&fs_settings.root); - std::fs::create_dir_all(&root).map_err(|e| { - error::Error::internal_err(format!( - "Failed to create filesystem storage root {}: {e}", - root.display() - )) - })?; - let store = - object_store::local::LocalFileSystem::new_with_prefix(&root).map_err(|e| { - error::Error::internal_err(format!( - "Failed to create filesystem object store at {}: {e}", - root.display() - )) - })?; - Ok(Arc::new(store) as Arc) - } - LargeFileStorage::S3Storage(_) | LargeFileStorage::S3AwsOidc(_) => { - let resource_path = lfs.get_s3_resource_path().trim_start_matches("$res:"); - let resource_value = get_resource_value(db, w_id, resource_path).await?; - let s3_resource: S3Resource = serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Failed to parse S3 resource: {e}")) - })?; - build_s3_client(&s3_resource).await - } - LargeFileStorage::AzureBlobStorage(_) | LargeFileStorage::AzureWorkloadIdentity(_) => { - let resource_path = lfs.get_s3_resource_path().trim_start_matches("$res:"); - let resource_value = get_resource_value(db, w_id, resource_path).await?; - let azure_resource: AzureBlobResource = - serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Failed to parse Azure resource: {e}")) - })?; - build_azure_blob_client(&azure_resource) - } - LargeFileStorage::GoogleCloudStorage(_) => { - let resource_path = lfs.get_s3_resource_path().trim_start_matches("$res:"); - let resource_value = get_resource_value(db, w_id, resource_path).await?; - let gcs_resource: GcsResource = - serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Failed to parse GCS resource: {e}")) - })?; - build_gcs_client(&gcs_resource).await - } - } -} - -#[cfg(feature = "parquet")] -fn expand_home(path: &str) -> std::path::PathBuf { - if path.starts_with("~/") { - if let Some(home) = std::env::var_os("HOME") { - let mut p = std::path::PathBuf::from(home); - p.push(&path[2..]); - return p; - } - } - std::path::PathBuf::from(path) -} - -#[cfg(feature = "parquet")] -async fn get_resource_value( - db: &crate::DB, - w_id: &str, - resource_path: &str, -) -> error::Result { - sqlx::query_scalar!( - "SELECT value FROM resource WHERE workspace_id = $1 AND path = $2", - w_id, - resource_path - ) - .fetch_optional(db) - .await? - .flatten() - .ok_or_else(|| { - error::Error::NotFound(format!( - "Resource {resource_path} not found in workspace {w_id}" - )) - }) -} - -#[derive(PartialEq)] -pub enum BundleFormat { - Esm, - Cjs, -} - -impl BundleFormat { - pub fn from_string(s: &str) -> Option { - match s { - "esm" => Some(Self::Esm), - "cjs" => Some(Self::Cjs), - _ => None, - } - } -} - -pub const DEFAULT_STORAGE: &str = "_default_"; - -pub async fn upload_artifact_to_store( - path: &str, - data: bytes::Bytes, - standalone_dir: &str, -) -> error::Result<()> { - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = crate::s3_helpers::get_object_store().await; - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let object_store: Option<()> = None; - Ok( - if &crate::utils::MODE_AND_ADDONS.mode == &crate::utils::Mode::Standalone - && object_store.is_none() - { - let path = format!("{}/{}", standalone_dir, path); - tracing::info!("Writing file to path {path}"); - - let split_path = path.split("/").collect::>(); - std::fs::create_dir_all(split_path[..split_path.len() - 1].join("/"))?; - - crate::worker::write_file_bytes(&path, &data)?; - } else { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - return Err(error::Error::ExecutionErr( - "codebase is an EE feature".to_string(), - )); - } - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = object_store { - if let Err(e) = os - .put(&object_store::path::Path::from(path), data.into()) - .await - { - tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); - return Err(error::Error::ExecutionErr(format!( - "Failed to put {path} to s3" - ))); - } - } else { - return Err(error::Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); - } - }, - ) -} - #[cfg(feature = "parquet")] pub async fn attempt_fetch_bytes( client: Arc, @@ -826,11 +352,6 @@ pub async fn attempt_fetch_bytes( return Ok(bytes); } -#[cfg(feature = "parquet")] -use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region}; -#[cfg(feature = "parquet")] -use object_store::CredentialProvider; - #[cfg(feature = "parquet")] pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result> { let static_creds = s3_resource_ref.access_key.as_ref().is_some_and(|x| x != "") @@ -863,7 +384,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result error::Result https://github.com/apache/arrow-rs-object-store/issues/419 let store = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store_builder.build())) .map_err(|panic_info| { tracing::error!( @@ -1007,43 +527,21 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result Option<&String> { - match self { - ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(), - ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name), - ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket), - ObjectSettings::Gcs(gcs_settings) => Some(&gcs_settings.bucket), - ObjectSettings::Filesystem(fs_settings) => Some(&fs_settings.root), - } - } +#[cfg(feature = "parquet")] +pub fn build_filesystem_client(root_path: &str) -> error::Result> { + let store = object_store::local::LocalFileSystem::new_with_prefix(root_path).map_err(|e| { + error::Error::internal_err(format!( + "Error building filesystem object store: {:?}", + e + )) + })?; + Ok(Arc::new(store)) } #[cfg(feature = "parquet")] pub async fn build_object_store_from_settings( settings: ObjectSettings, - init_private_key: Option<&crate::DB>, + init_private_key: Option<&windmill_common::DB>, ) -> error::Result { match settings { ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings) @@ -1075,51 +573,12 @@ pub async fn build_object_store_from_settings( .await .map(|x| ExpirableObjectStore::from(x)) } - ObjectSettings::Filesystem(fs_settings) => { - let root_str = if fs_settings.root.starts_with("~/") { - if let Some(home) = std::env::var_os("HOME") { - let mut p = std::path::PathBuf::from(home); - p.push(&fs_settings.root[2..]); - p - } else { - std::path::PathBuf::from(&fs_settings.root) - } - } else { - std::path::PathBuf::from(&fs_settings.root) - }; - let root = root_str; - std::fs::create_dir_all(&root).map_err(|e| { - error::Error::internal_err(format!( - "Failed to create filesystem object store root {}: {e}", - root.display() - )) - })?; - let store = object_store::local::LocalFileSystem::new_with_prefix(&root).map_err( - |e| { - error::Error::internal_err(format!( - "Failed to create filesystem object store at {}: {e}", - root.display() - )) - }, - )?; - Ok(ExpirableObjectStore::from(Arc::new(store) as Arc)) + ObjectSettings::Filesystem(fs) => { + build_filesystem_client(&fs.root_path).map(|x| ExpirableObjectStore::from(x)) } } } -#[derive(Debug, Deserialize, Serialize, Clone)] -pub struct S3Settings { - pub bucket: Option, - pub region: Option, - pub access_key: Option, - pub secret_key: Option, - pub endpoint: Option, - pub allow_http: Option, // default to true - pub path_style: Option, - pub store_logs: Option, - pub port: Option, -} - #[cfg(feature = "parquet")] fn none_if_empty(s: Option) -> Option { if s.is_none() || s.as_ref().unwrap().is_empty() { @@ -1179,16 +638,265 @@ impl CredentialProvider for AwsCredentialAdapter { } } +// --- End moved code --- + +lazy_static::lazy_static! { + static ref S3_BUCKET_RESTRICTIONS: Option>> = { + parse_bucket_restrictions() + }; +} + +fn parse_bucket_restrictions() -> Option>> { + let env_var = std::env::var("S3_BUCKETS_WORKSPACE_RESTRICTIONS").ok()?; + parse_bucket_restrictions_from_str(&env_var) +} + +fn parse_bucket_restrictions_from_str(input: &str) -> Option>> { + if input.trim().is_empty() { + return None; + } + + let mut restrictions = HashMap::new(); + + for bucket_rule in input.split(';') { + let bucket_rule = bucket_rule.trim(); + if bucket_rule.is_empty() { + continue; + } + + let parts: Vec<&str> = bucket_rule.splitn(2, ':').collect(); + if parts.len() != 2 { + tracing::warn!( + "Invalid bucket restriction format: '{}'. Expected 'bucket:workspace1,workspace2'", + bucket_rule + ); + continue; + } + + let bucket_name = parts[0].trim().to_string(); + let workspaces: Vec = parts[1] + .split(',') + .map(|w| w.trim().to_string()) + .filter(|w| !w.is_empty()) + .collect(); + + if workspaces.is_empty() { + tracing::warn!( + "No workspaces specified for bucket '{}', skipping restriction", + bucket_name + ); + continue; + } + + restrictions.insert(bucket_name, workspaces); + } + + if restrictions.is_empty() { + None + } else { + tracing::info!( + "S3 bucket restrictions loaded for {} buckets", + restrictions.len() + ); + Some(restrictions) + } +} + +pub fn check_bucket_workspace_restriction( + bucket_name: &str, + workspace_id: &str, +) -> error::Result<()> { + if let Some(ref restrictions) = *S3_BUCKET_RESTRICTIONS { + if let Some(allowed_workspaces) = restrictions.get(bucket_name) { + if !allowed_workspaces.contains(&workspace_id.to_string()) { + return Err(error::Error::NotAuthorized(format!( + "Workspace '{}' is not authorized to access bucket '{}'", + workspace_id, bucket_name + ))); + } + } + } + Ok(()) +} + +pub const DEFAULT_STORAGE: &str = "_default_"; + pub fn bundle(w_id: &str, hash: &str) -> String { format!("script_bundle/{}/{}", w_id, hash) } pub fn raw_app(w_id: &str, version: &i64) -> String { - format!("/home/rfiszel/raw_app/{}/{}", w_id, version) + format!("raw_app/{}/{}", w_id, version) +} + +pub async fn upload_artifact_to_store( + path: &str, + data: bytes::Bytes, + standalone_dir: &str, +) -> error::Result<()> { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let object_store = get_object_store().await; + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + let object_store: Option<()> = None; + Ok( + if &windmill_common::utils::MODE_AND_ADDONS.mode == &windmill_common::utils::Mode::Standalone + && object_store.is_none() + { + let path = format!("{}/{}", standalone_dir, path); + tracing::info!("Writing file to path {path}"); + + let split_path = path.split("/").collect::>(); + std::fs::create_dir_all(split_path[..split_path.len() - 1].join("/"))?; + + windmill_common::worker::write_file_bytes(&path, &data)?; + } else { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + return Err(error::Error::ExecutionErr( + "codebase is an EE feature".to_string(), + )); + } + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = object_store { + if let Err(e) = os + .put(&object_store::path::Path::from(path), data.into()) + .await + { + tracing::info!("Failed to put snapshot to s3 at {path}: {:?}", e); + return Err(error::Error::ExecutionErr(format!( + "Failed to put {path} to s3" + ))); + } + } else { + return Err(error::Error::BadConfig("Object store is required for snapshot script and is not configured for servers".to_string())); + } + }, + ) +} + +#[cfg(feature = "parquet")] +pub async fn get_etag_or_empty( + object_store_resource: &ObjectStoreResource, + s3_object: S3Object, +) -> Option { + let object_store_client = build_object_store_client(object_store_resource).await; + if object_store_client.is_err() { + return None; + } + + let object_key = object_store::path::Path::from(s3_object.s3); + + return object_store_client + .unwrap() + .head(&object_key) + .await + .ok() + .map(|meta| meta.e_tag) + .flatten(); +} + +pub fn lfs_to_object_store_resource( + lfs: &LargeFileStorage, + resource_value: serde_json::Value, +) -> error::Result { + match lfs { + LargeFileStorage::S3Storage(_) | LargeFileStorage::S3AwsOidc(_) => { + let s3_resource: S3Resource = serde_json::from_value(resource_value).map_err(|e| { + error::Error::internal_err(format!("Error parsing S3 resource: {e:?}")) + })?; + Ok(ObjectStoreResource::S3(s3_resource)) + } + LargeFileStorage::AzureBlobStorage(_) | LargeFileStorage::AzureWorkloadIdentity(_) => { + let azure_blob_resource: AzureBlobResource = serde_json::from_value(resource_value) + .map_err(|e| { + error::Error::internal_err(format!("Error parsing Azure Blob resource: {e:?}")) + })?; + Ok(ObjectStoreResource::Azure(azure_blob_resource)) + } + LargeFileStorage::GoogleCloudStorage(_) => { + let gcs_resource: GcsResource = + serde_json::from_value(resource_value).map_err(|e| { + error::Error::internal_err(format!("Error parsing GCS resource: {e:?}")) + })?; + Ok(ObjectStoreResource::Gcs(gcs_resource)) + } + LargeFileStorage::FilesystemStorage(fs) => Ok(ObjectStoreResource::Filesystem( + FilesystemSettings { + root_path: fs.root_path.clone(), + }, + )), + } +} + +pub fn format_duckdb_connection_settings( + object_store_resource: ObjectStoreResource, +) -> error::Result { + match object_store_resource { + ObjectStoreResource::S3(s3_resource) => duckdb_connection_settings_internal(s3_resource), + ObjectStoreResource::Azure(azure_resource) => { + let connection_string = format!( + "CREATE SECRET az_secret (TYPE AZURE, CONNECTION_STRING 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={};EndpointSuffix=core.windows.net');", + azure_resource.account_name, + azure_resource.access_key.unwrap_or_default() + ); + let response = DuckdbConnectionSettingsResponse { + connection_settings_str: connection_string, + azure_container_path: Some(format!("az://{}", azure_resource.container_name)), + s3_bucket: None, + }; + Ok(response) + } + ObjectStoreResource::Gcs(_) => { + return Err(error::Error::BadRequest( + "GCS is not supported in DuckDB".to_string(), + )); + } + ObjectStoreResource::Filesystem(_) => { + return Err(error::Error::BadRequest( + "Filesystem is not supported in DuckDB".to_string(), + )); + } + } +} + +pub fn duckdb_connection_settings_internal( + s3_resource: S3Resource, +) -> error::Result { + let mut duckdb_settings: String = String::new(); + + duckdb_settings.push_str("SET home_directory='./';\n"); + duckdb_settings.push_str("INSTALL 'httpfs';\n"); + if s3_resource.path_style.unwrap_or(true) { + duckdb_settings.push_str("SET s3_url_style='path';\n"); + } + duckdb_settings.push_str(format!("SET s3_region='{}';\n", s3_resource.region).as_str()); + duckdb_settings.push_str( + format!( + "SET s3_endpoint='{}';\n", + s3_resource.endpoint_with_region_fallback(None) + ) + .as_str(), + ); + if !s3_resource.use_ssl { + duckdb_settings.push_str("SET s3_use_ssl=0;\n"); + } + if let Some(access_key_id) = s3_resource.access_key { + duckdb_settings.push_str(format!("SET s3_access_key_id='{}';\n", access_key_id).as_str()); + } + if let Some(secret_access_key) = s3_resource.secret_key { + duckdb_settings + .push_str(format!("SET s3_secret_access_key='{}';\n", secret_access_key).as_str()); + } + + let response = DuckdbConnectionSettingsResponse { + connection_settings_str: duckdb_settings, + azure_container_path: None, + s3_bucket: Some(s3_resource.bucket), + }; + return Ok(response); } -// Originally used a Arc> -// But cannot call .close() on it because it moves the value and the object is not Sized #[cfg(feature = "parquet")] enum RecordBatchWriterEnum { Parquet(ArrowWriter), @@ -1241,7 +949,7 @@ impl Write for ChannelWriter { #[cfg(not(feature = "parquet"))] pub async fn convert_json_line_stream>( mut _stream: impl futures::TryStreamExt> + Unpin, - _output_format: windmill_parser_sql::S3ModeFormat, + _output_format: S3ModeFormat, ) -> anyhow::Result>> { Ok(async_stream::stream! { yield Err(anyhow::anyhow!("Parquet feature is not enabled. Cannot convert JSON line stream.")); @@ -1266,13 +974,11 @@ pub async fn convert_json_line_stream>( .to_str() .ok_or_else(|| anyhow::anyhow!("Invalid path"))?; - // Write the stream to a temporary file let mut file: tokio::fs::File = tokio::fs::File::create(&path).await.map_err(to_anyhow)?; while let Some(chunk) = stream.next().await { match chunk { Ok(chunk) => { - // Convert the chunk to bytes and write it to the file let b: bytes::Bytes = serde_json::to_string(&chunk)?.into(); file.write_all(&b).await?; file.write_all(b"\n").await?; @@ -1292,7 +998,7 @@ pub async fn convert_json_line_stream>( ctx.register_json( "my_table", path_str, - NdJsonReadOptions { ..Default::default() }, + NdJsonReadOptions::default(), ) .await .map_err(to_anyhow)?; @@ -1319,8 +1025,6 @@ pub async fn convert_json_line_stream>( } }))); - // This spawn is so that the data is sent in the background. Else the function would deadlock - // when hitting the mpsc channel limit task::spawn(async move { while let Some(batch_result) = datafusion_stream.next().await { let batch: RecordBatch = match batch_result { @@ -1335,10 +1039,7 @@ pub async fn convert_json_line_stream>( } }; let writer = writer.clone(); - // Writer calls blocking_send which would crash if called from the async context let write_result = task::spawn_blocking(move || { - // SAFETY: We await so the code is actually sequential, lock unwrap cannot panic - // Second unwrap is ok because we initialized the option with Some writer.lock().unwrap().as_mut().unwrap().write(&batch) }) .await; @@ -1368,122 +1069,721 @@ pub async fn convert_json_line_stream>( Ok(tokio_stream::wrappers::ReceiverStream::new(rx)) } -#[derive(Deserialize, Serialize, Clone, Debug)] -pub struct DuckdbConnectionSettingsResponse { - pub connection_settings_str: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub azure_container_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub s3_bucket: Option, -} - -#[derive(Deserialize, Serialize)] -pub struct DuckdbConnectionSettingsQueryV2 { - #[serde(skip_serializing_if = "Option::is_none")] - pub s3_resource_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub storage: Option, -} - -pub fn lfs_to_object_store_resource( - lfs: &LargeFileStorage, - resource_value: serde_json::Value, -) -> error::Result { - match lfs { - LargeFileStorage::S3Storage(_) | LargeFileStorage::S3AwsOidc(_) => { - let s3_resource: S3Resource = serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Error parsing S3 resource: {e:?}")) - })?; - Ok(ObjectStoreResource::S3(s3_resource)) - } - LargeFileStorage::AzureBlobStorage(_) | LargeFileStorage::AzureWorkloadIdentity(_) => { - let azure_blob_resource: AzureBlobResource = serde_json::from_value(resource_value) - .map_err(|e| { - error::Error::internal_err(format!("Error parsing Azure Blob resource: {e:?}")) - })?; - Ok(ObjectStoreResource::Azure(azure_blob_resource)) - } - LargeFileStorage::GoogleCloudStorage(_) => { - let gcs_resource: GcsResource = - serde_json::from_value(resource_value).map_err(|e| { - error::Error::internal_err(format!("Error parsing GCS resource: {e:?}")) - })?; - Ok(ObjectStoreResource::Gcs(gcs_resource)) - } - LargeFileStorage::FilesystemStorage(_) => Err(error::Error::internal_err( - "FilesystemStorage does not have an ObjectStoreResource".to_string(), - )), - } -} - -pub fn format_duckdb_connection_settings( - object_store_resource: ObjectStoreResource, -) -> error::Result { - match object_store_resource { - ObjectStoreResource::S3(s3_resource) => duckdb_connection_settings_internal(s3_resource), - ObjectStoreResource::Azure(azure_resource) => { - let connection_string = format!( - "CREATE SECRET az_secret (TYPE AZURE, CONNECTION_STRING 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={};EndpointSuffix=core.windows.net');", - azure_resource.account_name, - azure_resource.access_key.unwrap_or_default() - ); - let response = DuckdbConnectionSettingsResponse { - connection_settings_str: connection_string, - azure_container_path: Some(format!("az://{}", azure_resource.container_name)), - s3_bucket: None, - }; - Ok(response) - } - ObjectStoreResource::Gcs(_) => { - return Err(error::Error::BadRequest( - "GCS is not supported in DuckDB".to_string(), - )); - } - } -} - -pub fn duckdb_connection_settings_internal( - s3_resource: S3Resource, -) -> error::Result { - let mut duckdb_settings: String = String::new(); - - duckdb_settings.push_str("SET home_directory='./';\n"); // TODO: make this configurable maybe, or point to a temporary folder - duckdb_settings.push_str("INSTALL 'httpfs';\n"); - if s3_resource.path_style.unwrap_or(true) { - duckdb_settings.push_str("SET s3_url_style='path';\n"); - } - duckdb_settings.push_str(format!("SET s3_region='{}';\n", s3_resource.region).as_str()); - duckdb_settings.push_str( - format!( - "SET s3_endpoint='{}';\n", - s3_resource.endpoint_with_region_fallback(None) - ) - .as_str(), - ); - if !s3_resource.use_ssl { - duckdb_settings.push_str("SET s3_use_ssl=0;\n"); // default is true for DuckDB - } - if let Some(access_key_id) = s3_resource.access_key { - duckdb_settings.push_str(format!("SET s3_access_key_id='{}';\n", access_key_id).as_str()); - } - if let Some(secret_access_key) = s3_resource.secret_key { - duckdb_settings - .push_str(format!("SET s3_secret_access_key='{}';\n", secret_access_key).as_str()); - } - - let response = DuckdbConnectionSettingsResponse { - connection_settings_str: duckdb_settings, - azure_container_path: None, - s3_bucket: Some(s3_resource.bucket), - }; - return Ok(response); -} - -// DuckDB does not parse anything in case of S3 errors and just returns a generic error message. -// To display better error messages, we cache the errors in a Map -// -// We leverage the fact that workers have an internal server to insert the error message -// from the S3 Proxy, and read it directly in memory from the worker. lazy_static::lazy_static! { pub static ref S3_PROXY_LAST_ERRORS_CACHE: Cache = Cache::new(4); } + +#[cfg(feature = "parquet")] +pub async fn get_logs_from_store( + log_offset: i32, + logs: &str, + log_file_index: &Option>, +) -> Option>> { + if log_offset > 0 { + if let Some(file_index) = log_file_index.clone() { + if let Some(os) = get_object_store().await { + let logs = logs.to_string(); + let stream = async_stream::stream! { + for file_p in file_index { + let file = os.get(&object_store::path::Path::from(file_p.clone())).await; + match file { + Ok(file) => { + if let Ok(bytes) = file.bytes().await { + yield Ok(bytes::Bytes::from(bytes)); + } + } + Err(e) => { + tracing::debug!("error getting file from store: {file_p}: {e}"); + } + } + } + yield Ok(bytes::Bytes::from(logs)) + }; + return Some(stream); + } else { + tracing::debug!("object store client not present"); + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + // --- render_endpoint tests --- + + #[test] + fn test_render_endpoint_ssl_prefix() { + let result = render_endpoint( + "s3.amazonaws.com".to_string(), + true, + None, + Some(true), + "mybucket".to_string(), + ); + assert_eq!(result, "https://s3.amazonaws.com"); + } + + #[test] + fn test_render_endpoint_non_ssl_prefix() { + let result = render_endpoint( + "minio.local".to_string(), + false, + None, + Some(true), + "mybucket".to_string(), + ); + assert_eq!(result, "http://minio.local"); + } + + #[test] + fn test_render_endpoint_with_port() { + let result = render_endpoint( + "minio.local".to_string(), + false, + Some(9000), + Some(true), + "mybucket".to_string(), + ); + assert_eq!(result, "http://minio.local:9000"); + } + + #[test] + fn test_render_endpoint_virtual_hosted_style() { + let result = render_endpoint( + "s3.amazonaws.com".to_string(), + true, + None, + Some(false), + "mybucket".to_string(), + ); + assert_eq!(result, "https://mybucket.s3.amazonaws.com"); + } + + #[test] + fn test_render_endpoint_passthrough_with_scheme() { + let result = render_endpoint( + "https://custom.endpoint.com".to_string(), + false, // use_ssl is ignored when scheme already present + None, + Some(false), // path_style is also ignored + "mybucket".to_string(), + ); + assert_eq!(result, "https://custom.endpoint.com"); + } + + // --- lfs_to_object_store_resource tests --- + + #[test] + fn test_lfs_to_object_store_resource_s3() { + let lfs = LargeFileStorage::S3Storage(S3Storage { + s3_resource_path: "u/admin/s3_resource".to_string(), + public_resource: None, + advanced_permissions: None, + }); + let resource_json = serde_json::json!({ + "bucket": "my-bucket", + "region": "us-east-1", + "endPoint": "s3.amazonaws.com", + "useSSL": true, + "accessKey": "AKIA...", + "secretKey": "secret" + }); + let result = lfs_to_object_store_resource(&lfs, resource_json).unwrap(); + match result { + ObjectStoreResource::S3(s3) => { + assert_eq!(s3.bucket, "my-bucket"); + assert_eq!(s3.region, "us-east-1"); + assert_eq!(s3.endpoint, "s3.amazonaws.com"); + assert!(s3.use_ssl); + } + _ => panic!("Expected S3 resource"), + } + } + + #[test] + fn test_lfs_to_object_store_resource_azure() { + let lfs = LargeFileStorage::AzureBlobStorage(AzureBlobStorage { + azure_blob_resource_path: "u/admin/azure_resource".to_string(), + public_resource: None, + advanced_permissions: None, + }); + let resource_json = serde_json::json!({ + "accountName": "myaccount", + "containerName": "mycontainer" + }); + let result = lfs_to_object_store_resource(&lfs, resource_json).unwrap(); + match result { + ObjectStoreResource::Azure(az) => { + assert_eq!(az.account_name, "myaccount"); + assert_eq!(az.container_name, "mycontainer"); + } + _ => panic!("Expected Azure resource"), + } + } + + #[test] + fn test_lfs_to_object_store_resource_gcs() { + let lfs = LargeFileStorage::GoogleCloudStorage(GoogleCloudStorage { + gcs_resource_path: "u/admin/gcs_resource".to_string(), + public_resource: None, + advanced_permissions: None, + }); + let resource_json = serde_json::json!({ + "bucket": "gcs-bucket", + "serviceAccountKey": {"type": "service_account", "project_id": "test"} + }); + let result = lfs_to_object_store_resource(&lfs, resource_json).unwrap(); + match result { + ObjectStoreResource::Gcs(gcs) => { + assert_eq!(gcs.bucket, "gcs-bucket"); + } + _ => panic!("Expected GCS resource"), + } + } + + #[test] + fn test_lfs_to_object_store_resource_filesystem() { + let lfs = LargeFileStorage::FilesystemStorage(FilesystemStorage { + root_path: "/tmp/mydata".to_string(), + public_resource: None, + advanced_permissions: None, + }); + // resource_value is ignored for filesystem + let result = + lfs_to_object_store_resource(&lfs, serde_json::Value::Null).unwrap(); + match result { + ObjectStoreResource::Filesystem(fs) => { + assert_eq!(fs.root_path, "/tmp/mydata"); + } + _ => panic!("Expected Filesystem resource"), + } + } + + // --- duckdb_connection_settings tests --- + + #[test] + fn test_duckdb_connection_settings_s3_basic() { + let s3 = S3Resource { + bucket: "test-bucket".to_string(), + region: "eu-west-1".to_string(), + endpoint: "s3.eu-west-1.amazonaws.com".to_string(), + use_ssl: true, + access_key: Some("AKIA123".to_string()), + secret_key: Some("secret456".to_string()), + path_style: Some(true), + token: None, + expiration: None, + port: None, + }; + let result = duckdb_connection_settings_internal(s3).unwrap(); + assert!(result.connection_settings_str.contains("SET s3_region='eu-west-1'")); + assert!(result.connection_settings_str.contains("SET s3_access_key_id='AKIA123'")); + assert!(result.connection_settings_str.contains("SET s3_secret_access_key='secret456'")); + assert!(result.connection_settings_str.contains("SET s3_url_style='path'")); + assert!(!result.connection_settings_str.contains("SET s3_use_ssl=0")); + assert_eq!(result.s3_bucket, Some("test-bucket".to_string())); + assert!(result.azure_container_path.is_none()); + } + + #[test] + fn test_duckdb_connection_settings_s3_no_ssl() { + let s3 = S3Resource { + bucket: "bucket".to_string(), + region: "us-east-1".to_string(), + endpoint: "minio:9000".to_string(), + use_ssl: false, + access_key: None, + secret_key: None, + path_style: Some(false), + token: None, + expiration: None, + port: None, + }; + let result = duckdb_connection_settings_internal(s3).unwrap(); + assert!(result.connection_settings_str.contains("SET s3_use_ssl=0")); + assert!(!result.connection_settings_str.contains("SET s3_url_style='path'")); + } + + #[test] + fn test_duckdb_connection_settings_azure() { + let resource = ObjectStoreResource::Azure(AzureBlobResource { + endpoint: None, + use_ssl: None, + account_name: "myaccount".to_string(), + tenant_id: None, + client_id: None, + container_name: "mycontainer".to_string(), + access_key: Some("base64key==".to_string()), + federated_token_file: None, + }); + let result = format_duckdb_connection_settings(resource).unwrap(); + assert!(result.connection_settings_str.contains("AccountName=myaccount")); + assert!(result.connection_settings_str.contains("AccountKey=base64key==")); + assert_eq!( + result.azure_container_path, + Some("az://mycontainer".to_string()) + ); + assert!(result.s3_bucket.is_none()); + } + + #[test] + fn test_duckdb_connection_settings_gcs_unsupported() { + let resource = ObjectStoreResource::Gcs(GcsResource { + bucket: "bucket".to_string(), + service_account_key: "{}".to_string(), + }); + let result = format_duckdb_connection_settings(resource); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("GCS is not supported")); + } + + #[test] + fn test_duckdb_connection_settings_filesystem_unsupported() { + let resource = ObjectStoreResource::Filesystem(FilesystemSettings { + root_path: "/tmp/data".to_string(), + }); + let result = format_duckdb_connection_settings(resource); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Filesystem is not supported")); + } + + // --- object_store error mapping --- + + #[cfg(feature = "parquet")] + #[test] + fn test_object_store_error_mapping() { + use windmill_common::error::Error; + + // NotFound + let err = object_store::Error::NotFound { + path: "test/path".into(), + source: "missing".into(), + }; + let mapped = object_store_error_to_error(err); + assert!(matches!(mapped, Error::NotFound(_))); + + // PermissionDenied + let err = object_store::Error::PermissionDenied { + path: "secret".into(), + source: "forbidden".into(), + }; + let mapped = object_store_error_to_error(err); + assert!(matches!(mapped, Error::PermissionDenied(_))); + + // InvalidPath + let err = object_store::Error::InvalidPath { + source: object_store::path::Error::EmptySegment { path: "".into() }, + }; + let mapped = object_store_error_to_error(err); + assert!(matches!(mapped, Error::BadRequest(_))); + + // NotImplemented + let mapped = object_store_error_to_error(object_store::Error::NotImplemented); + assert!(matches!(mapped, Error::BadRequest(_))); + + // Unauthenticated + let err = object_store::Error::Unauthenticated { + path: "obj".into(), + source: "no creds".into(), + }; + let mapped = object_store_error_to_error(err); + assert!(matches!(mapped, Error::NotAuthorized(_))); + } + + // --- Filesystem-backed integration tests --- + + #[cfg(feature = "parquet")] + #[test] + fn test_build_filesystem_client() { + let dir = tempfile::tempdir().unwrap(); + let result = build_filesystem_client(dir.path().to_str().unwrap()); + assert!(result.is_ok()); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_filesystem_put_get_roundtrip() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let client = build_filesystem_client(dir.path().to_str().unwrap()).unwrap(); + let path = Path::from("test.txt"); + let data = bytes::Bytes::from("hello world"); + + client + .put(&path, PutPayload::from(data.clone())) + .await + .unwrap(); + let result = client.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(result, data); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_filesystem_nested_paths() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let client = build_filesystem_client(dir.path().to_str().unwrap()).unwrap(); + let path = Path::from("a/b/c.txt"); + let data = bytes::Bytes::from("nested content"); + + client + .put(&path, PutPayload::from(data.clone())) + .await + .unwrap(); + let result = client.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(result, data); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_filesystem_overwrite() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let client = build_filesystem_client(dir.path().to_str().unwrap()).unwrap(); + let path = Path::from("overwrite.txt"); + + client + .put(&path, PutPayload::from(bytes::Bytes::from("v1"))) + .await + .unwrap(); + client + .put(&path, PutPayload::from(bytes::Bytes::from("v2"))) + .await + .unwrap(); + let result = client.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(result, bytes::Bytes::from("v2")); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_attempt_fetch_bytes_success() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let client = build_filesystem_client(dir.path().to_str().unwrap()).unwrap(); + let data = bytes::Bytes::from("fetch me"); + client + .put(&Path::from("fetch.txt"), PutPayload::from(data.clone())) + .await + .unwrap(); + + let result = attempt_fetch_bytes(client, "fetch.txt").await.unwrap(); + assert_eq!(result, data); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_attempt_fetch_bytes_missing_key() { + let dir = tempfile::tempdir().unwrap(); + let client = build_filesystem_client(dir.path().to_str().unwrap()).unwrap(); + + let result = attempt_fetch_bytes(client, "nonexistent.txt").await; + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + windmill_common::error::Error::ExecutionErr(_) + )); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_attempt_fetch_bytes_empty_object() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let client = build_filesystem_client(dir.path().to_str().unwrap()).unwrap(); + client + .put( + &Path::from("empty.txt"), + PutPayload::from(bytes::Bytes::new()), + ) + .await + .unwrap(); + + let result = attempt_fetch_bytes(client, "empty.txt").await; + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("does not exist in bucket")); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_get_etag_or_empty() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let client = build_filesystem_client(root).unwrap(); + client + .put( + &Path::from("etag.txt"), + PutPayload::from(bytes::Bytes::from("content")), + ) + .await + .unwrap(); + + let resource = ObjectStoreResource::Filesystem(FilesystemSettings { + root_path: root.to_string(), + }); + let s3_obj = S3Object { + s3: "etag.txt".to_string(), + storage: None, + filename: None, + presigned: None, + }; + + let etag = get_etag_or_empty(&resource, s3_obj).await; + // LocalFileSystem should return an etag based on file metadata + assert!(etag.is_some()); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_get_etag_or_empty_missing() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + + let resource = ObjectStoreResource::Filesystem(FilesystemSettings { + root_path: root.to_string(), + }); + let s3_obj = S3Object { + s3: "nonexistent.txt".to_string(), + storage: None, + filename: None, + presigned: None, + }; + + let etag = get_etag_or_empty(&resource, s3_obj).await; + assert!(etag.is_none()); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_settings_to_client_end_to_end() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap().to_string(); + let settings = ObjectSettings::Filesystem(FilesystemSettings { + root_path: root, + }); + + let expirable = build_object_store_from_settings(settings, None) + .await + .unwrap(); + let data = bytes::Bytes::from("end to end via settings"); + expirable + .store + .put( + &Path::from("e2e.txt"), + PutPayload::from(data.clone()), + ) + .await + .unwrap(); + let result = expirable + .store + .get(&Path::from("e2e.txt")) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(result, data); + assert!(expirable.refresh.is_none()); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_resource_to_client_end_to_end() { + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap().to_string(); + let resource = ObjectStoreResource::Filesystem(FilesystemSettings { + root_path: root, + }); + + let client = build_object_store_client(&resource).await.unwrap(); + let data = bytes::Bytes::from("end to end via resource"); + client + .put(&Path::from("e2e.txt"), PutPayload::from(data.clone())) + .await + .unwrap(); + let result = client + .get(&Path::from("e2e.txt")) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(result, data); + } + + // --- bundle / raw_app path tests --- + + #[test] + fn test_bundle_path_format() { + assert_eq!(bundle("my_workspace", "abc123"), "script_bundle/my_workspace/abc123"); + } + + #[test] + fn test_raw_app_path_format() { + assert_eq!(raw_app("my_workspace", &42), "raw_app/my_workspace/42"); + } + + // --- parse_bucket_restrictions tests --- + + #[test] + fn test_parse_bucket_restrictions_single_bucket() { + let result = + parse_bucket_restrictions_from_str("my-bucket:workspace1,workspace2").unwrap(); + assert_eq!(result.len(), 1); + assert_eq!( + result.get("my-bucket").unwrap(), + &vec!["workspace1".to_string(), "workspace2".to_string()] + ); + } + + #[test] + fn test_parse_bucket_restrictions_multiple_buckets() { + let result = parse_bucket_restrictions_from_str( + "bucket-a:ws1,ws2;bucket-b:ws3", + ) + .unwrap(); + assert_eq!(result.len(), 2); + assert_eq!( + result.get("bucket-a").unwrap(), + &vec!["ws1".to_string(), "ws2".to_string()] + ); + assert_eq!( + result.get("bucket-b").unwrap(), + &vec!["ws3".to_string()] + ); + } + + #[test] + fn test_parse_bucket_restrictions_empty_string() { + assert!(parse_bucket_restrictions_from_str("").is_none()); + assert!(parse_bucket_restrictions_from_str(" ").is_none()); + } + + #[test] + fn test_parse_bucket_restrictions_invalid_format_skipped() { + // "no-colon" is invalid, only "valid:ws1" should be parsed + let result = + parse_bucket_restrictions_from_str("no-colon;valid:ws1").unwrap(); + assert_eq!(result.len(), 1); + assert!(result.contains_key("valid")); + } + + #[test] + fn test_parse_bucket_restrictions_trailing_semicolons() { + let result = + parse_bucket_restrictions_from_str(";bucket:ws1;;").unwrap(); + assert_eq!(result.len(), 1); + assert!(result.contains_key("bucket")); + } + + // --- build_filesystem_client error path --- + + #[cfg(feature = "parquet")] + #[test] + fn test_build_filesystem_client_nonexistent_path() { + // LocalFileSystem::new_with_prefix fails when the directory doesn't exist + let result = build_filesystem_client("/tmp/windmill_test_nonexistent_dir_12345_xyz"); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Error building filesystem object store")); + } + + // --- get_logs_from_store test --- + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_get_logs_from_store_with_filesystem() { + use futures::StreamExt; + use object_store::{path::Path, ObjectStore, PutPayload}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap().to_string(); + let client = build_filesystem_client(&root).unwrap(); + + // Write two log chunk files + client + .put( + &Path::from("logs/chunk1.log"), + PutPayload::from(bytes::Bytes::from("chunk1 content\n")), + ) + .await + .unwrap(); + client + .put( + &Path::from("logs/chunk2.log"), + PutPayload::from(bytes::Bytes::from("chunk2 content\n")), + ) + .await + .unwrap(); + + // Set the global OBJECT_STORE_SETTINGS to our filesystem store + { + let mut settings = OBJECT_STORE_SETTINGS.write().await; + *settings = Some(ExpirableObjectStore::from(client)); + } + + let file_index = Some(vec![ + "logs/chunk1.log".to_string(), + "logs/chunk2.log".to_string(), + ]); + let tail_logs = "tail logs here"; + + // log_offset > 0, file_index Some, object store set → should return a stream + let stream = get_logs_from_store(1, tail_logs, &file_index).await; + assert!(stream.is_some()); + + let chunks: Vec = stream + .unwrap() + .filter_map(|r| async { r.ok() }) + .collect() + .await; + assert_eq!(chunks.len(), 3); // chunk1 + chunk2 + tail_logs + assert_eq!(chunks[0], bytes::Bytes::from("chunk1 content\n")); + assert_eq!(chunks[1], bytes::Bytes::from("chunk2 content\n")); + assert_eq!(chunks[2], bytes::Bytes::from("tail logs here")); + + // Clean up global state + { + let mut settings = OBJECT_STORE_SETTINGS.write().await; + *settings = None; + } + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_get_logs_from_store_zero_offset() { + // log_offset == 0 → always returns None regardless of other params + let file_index = Some(vec!["some/file.log".to_string()]); + let result = get_logs_from_store(0, "logs", &file_index).await; + assert!(result.is_none()); + } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_get_logs_from_store_no_file_index() { + // file_index is None → returns None + let result = get_logs_from_store(1, "logs", &None).await; + assert!(result.is_none()); + } +} diff --git a/backend/windmill-trigger-http/Cargo.toml b/backend/windmill-trigger-http/Cargo.toml index 43420793cf..865b82a682 100644 --- a/backend/windmill-trigger-http/Cargo.toml +++ b/backend/windmill-trigger-http/Cargo.toml @@ -16,6 +16,7 @@ cloud = ["windmill-common/cloud"] [dependencies] windmill-api-auth.workspace = true windmill-common = { workspace = true, default-features = false } +windmill-types.workspace = true windmill-trigger.workspace = true windmill-audit.workspace = true windmill-git-sync.workspace = true diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs index 2c109ff183..69acfcc2af 100644 --- a/backend/windmill-trigger-http/src/lib.rs +++ b/backend/windmill-trigger-http/src/lib.rs @@ -7,11 +7,11 @@ use tokio::sync::{RwLock, RwLockReadGuard}; use windmill_common::{ error::{Error, Result}, flows::Retry, - s3_helpers::S3Object, utils::ExpiringCacheEntry, worker::CLOUD_HOSTED, DB, }; +use windmill_types::s3::S3Object; use windmill_api_auth::ApiAuthed; use windmill_trigger::TriggerMode; diff --git a/backend/windmill-types/Cargo.toml b/backend/windmill-types/Cargo.toml index 4eb18fe88e..9d25773e99 100644 --- a/backend/windmill-types/Cargo.toml +++ b/backend/windmill-types/Cargo.toml @@ -20,3 +20,4 @@ anyhow.workspace = true tracing.workspace = true itertools.workspace = true strum.workspace = true +bitflags.workspace = true diff --git a/backend/windmill-types/src/lib.rs b/backend/windmill-types/src/lib.rs index 8b7e68d75c..6e935c3ece 100644 --- a/backend/windmill-types/src/lib.rs +++ b/backend/windmill-types/src/lib.rs @@ -5,6 +5,7 @@ pub mod flows; pub mod jobs; pub mod more_serde; pub mod runnable_settings; +pub mod s3; pub mod schedule; pub mod scripts; pub mod triggers; diff --git a/backend/windmill-types/src/s3.rs b/backend/windmill-types/src/s3.rs new file mode 100644 index 0000000000..e3cbd1f9a3 --- /dev/null +++ b/backend/windmill-types/src/s3.rs @@ -0,0 +1,479 @@ +use chrono::{DateTime, Utc}; +use serde::de::Visitor; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct FilesystemSettings { + pub root_path: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FilesystemStorage { + pub root_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_resource: Option, + pub advanced_permissions: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum LargeFileStorage { + S3Storage(S3Storage), + AzureBlobStorage(AzureBlobStorage), + S3AwsOidc(S3Storage), + AzureWorkloadIdentity(AzureBlobStorage), + GoogleCloudStorage(GoogleCloudStorage), + FilesystemStorage(FilesystemStorage), +} + +impl LargeFileStorage { + pub fn get_s3_resource_path(&self) -> &str { + match self { + LargeFileStorage::S3Storage(s3_lfs) => &s3_lfs.s3_resource_path, + LargeFileStorage::S3AwsOidc(s3_lfs) => &s3_lfs.s3_resource_path, + LargeFileStorage::AzureBlobStorage(az_lfs) => &az_lfs.azure_blob_resource_path, + LargeFileStorage::AzureWorkloadIdentity(az_lfs) => &az_lfs.azure_blob_resource_path, + LargeFileStorage::GoogleCloudStorage(gcs_lfs) => &gcs_lfs.gcs_resource_path, + LargeFileStorage::FilesystemStorage(fs_lfs) => &fs_lfs.root_path, + } + } + pub fn is_public_resource(&self) -> bool { + match self { + LargeFileStorage::S3Storage(lfs) => lfs.public_resource, + LargeFileStorage::S3AwsOidc(lfs) => lfs.public_resource, + LargeFileStorage::AzureBlobStorage(lfs) => lfs.public_resource, + LargeFileStorage::AzureWorkloadIdentity(lfs) => lfs.public_resource, + LargeFileStorage::GoogleCloudStorage(glfs) => glfs.public_resource, + LargeFileStorage::FilesystemStorage(fs_lfs) => fs_lfs.public_resource, + } + .unwrap_or(false) + } + pub fn get_advanced_permissions(&self) -> Option<&Vec> { + match self { + LargeFileStorage::S3Storage(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::S3AwsOidc(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::AzureBlobStorage(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::AzureWorkloadIdentity(lfs) => lfs.advanced_permissions.as_ref(), + LargeFileStorage::GoogleCloudStorage(glfs) => glfs.advanced_permissions.as_ref(), + LargeFileStorage::FilesystemStorage(fs_lfs) => fs_lfs.advanced_permissions.as_ref(), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct S3Storage { + pub s3_resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_resource: Option, + pub advanced_permissions: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct AzureBlobStorage { + pub azure_blob_resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_resource: Option, + pub advanced_permissions: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GoogleCloudStorage { + pub gcs_resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_resource: Option, + pub advanced_permissions: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] +pub struct S3PermissionRule { + pub pattern: String, + pub allow: S3Permission, +} + +bitflags::bitflags! { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct S3Permission: u8 { + const READ = 0b0001; + const WRITE = 0b0010; + const DELETE = 0b0100; + const LIST = 0b1000; + } +} + +impl Serialize for S3Permission { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut perms = Vec::new(); + if self.contains(S3Permission::READ) { + perms.push("read"); + } + if self.contains(S3Permission::WRITE) { + perms.push("write"); + } + if self.contains(S3Permission::DELETE) { + perms.push("delete"); + } + if self.contains(S3Permission::LIST) { + perms.push("list"); + } + let perms = perms.join(","); + perms.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for S3Permission { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct PermVisitor; + impl<'de> Visitor<'de> for PermVisitor { + type Value = S3Permission; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("comma separated list of permissions: read, write, delete, list") + } + fn visit_str(self, v: &str) -> Result { + let mut perms = S3Permission::empty(); + for value in v.split(',') { + perms |= match value { + "read" => S3Permission::READ, + "write" => S3Permission::WRITE, + "delete" => S3Permission::DELETE, + "list" => S3Permission::LIST, + _ => S3Permission::empty(), + }; + } + Ok(perms) + } + } + + deserializer.deserialize_str(PermVisitor) + } +} + +#[derive(Clone, Debug)] +pub enum ObjectStoreResource { + S3(S3Resource), + Azure(AzureBlobResource), + Gcs(GcsResource), + Filesystem(FilesystemSettings), +} + +impl ObjectStoreResource { + pub fn expiration(&self) -> Option> { + match self { + ObjectStoreResource::S3(s3_resource) => s3_resource.expiration, + _ => None, + } + } +} + +#[derive(Deserialize, Debug)] +pub enum StorageResourceType { + S3, + AzureBlob, + S3AwsOidc, + AzureWorkloadIdentity, + GoogleCloudStorage, + Filesystem, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct S3Resource { + pub bucket: String, + pub region: String, + #[serde(rename = "endPoint")] + pub endpoint: String, + #[serde(rename = "useSSL")] + pub use_ssl: bool, + #[serde(rename = "accessKey")] + pub access_key: Option, + #[serde(rename = "secretKey")] + pub secret_key: Option, + #[serde(rename = "pathStyle")] + pub path_style: Option, + pub token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expiration: Option>, + pub port: Option, +} + +impl S3Resource { + pub fn endpoint_with_region_fallback(&self, region_fallback: Option) -> String { + if self.endpoint.is_empty() { + let final_region = if self.region.is_empty() { + region_fallback.unwrap_or_else(|| "us-east-1".to_string()) + } else { + self.region.clone() + }; + format!("s3.{}.amazonaws.com", final_region) + } else { + self.endpoint.clone() + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct AzureBlobResource { + pub endpoint: Option, + #[serde(rename = "useSSL")] + pub use_ssl: Option, + #[serde(rename = "accountName")] + pub account_name: String, + #[serde(rename = "tenantId")] + pub tenant_id: Option, + #[serde(rename = "clientId")] + pub client_id: Option, + #[serde(rename = "containerName")] + pub container_name: String, + #[serde(rename = "accessKey")] + pub access_key: Option, + #[serde(rename = "federatedTokenFile")] + pub federated_token_file: Option, +} + +fn as_string<'de, D>(deserializer: D) -> Result +where + D: serde::de::Deserializer<'de>, +{ + let v: serde_json::Value = Deserialize::deserialize(deserializer)?; + serde_json::to_string(&v).map_err(serde::de::Error::custom) +} + +#[derive(Debug, Deserialize, Clone)] +pub struct GcsResource { + pub bucket: String, + #[serde(rename = "serviceAccountKey")] + #[serde(deserialize_with = "as_string")] + pub service_account_key: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, Hash)] +pub struct S3AwsOidcResource { + #[serde(rename = "bucket")] + pub bucket: String, + pub region: Option, + #[serde(rename = "roleArn")] + pub role_arn: String, + pub audience: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct S3Object { + pub s3: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub storage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub filename: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub presigned: Option, +} + +#[derive(Debug, PartialEq)] +pub enum BundleFormat { + Esm, + Cjs, +} + +impl BundleFormat { + pub fn from_string(s: &str) -> Option { + match s { + "esm" => Some(Self::Esm), + "cjs" => Some(Self::Cjs), + _ => None, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "typ", content = "value")] +pub enum ObjectStoreSettings { + S3(S3Settings), +} + +#[derive(Debug, Deserialize, Clone)] +#[serde(tag = "type")] +pub enum ObjectSettings { + S3(S3Settings), + Azure(AzureBlobResource), + AwsOidc(S3AwsOidcResource), + Gcs(GcsResource), + Filesystem(FilesystemSettings), +} + +impl ObjectSettings { + pub fn get_bucket(&self) -> Option<&String> { + match self { + ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(), + ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name), + ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket), + ObjectSettings::Gcs(gcs_settings) => Some(&gcs_settings.bucket), + ObjectSettings::Filesystem(fs_settings) => Some(&fs_settings.root_path), + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct S3Settings { + pub bucket: Option, + pub region: Option, + pub access_key: Option, + pub secret_key: Option, + pub endpoint: Option, + pub allow_http: Option, + pub path_style: Option, + pub store_logs: Option, + pub port: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct DuckdbConnectionSettingsResponse { + pub connection_settings_str: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub azure_container_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub s3_bucket: Option, +} + +#[derive(Deserialize, Serialize)] +pub struct DuckdbConnectionSettingsQueryV2 { + #[serde(skip_serializing_if = "Option::is_none")] + pub s3_resource_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub storage: Option, +} + +#[derive(Clone, Copy, Debug)] +pub enum S3ModeFormat { + Json, + Csv, + Parquet, +} + +pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { + match format { + S3ModeFormat::Json => "json", + S3ModeFormat::Csv => "csv", + S3ModeFormat::Parquet => "parquet", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_s3_permission_serde_custom_format() { + let perm = S3Permission::READ | S3Permission::WRITE; + let serialized = serde_json::to_string(&perm).unwrap(); + assert_eq!(serialized, "\"read,write\""); + + let deserialized: S3Permission = serde_json::from_str("\"read,write\"").unwrap(); + assert_eq!(deserialized, S3Permission::READ | S3Permission::WRITE); + + // Unknown permissions are silently ignored + let deserialized: S3Permission = + serde_json::from_str("\"read,unknown,delete\"").unwrap(); + assert_eq!(deserialized, S3Permission::READ | S3Permission::DELETE); + + // All four permissions + let all: S3Permission = + serde_json::from_str("\"read,write,delete,list\"").unwrap(); + assert_eq!( + all, + S3Permission::READ | S3Permission::WRITE | S3Permission::DELETE | S3Permission::LIST + ); + + // Empty string gives empty permission set + let empty: S3Permission = serde_json::from_str("\"\"").unwrap(); + assert_eq!(empty, S3Permission::empty()); + } + + #[test] + fn test_s3_resource_endpoint_fallback() { + let resource = S3Resource { + bucket: "b".to_string(), + region: "".to_string(), + endpoint: "".to_string(), + use_ssl: true, + access_key: None, + secret_key: None, + path_style: None, + token: None, + expiration: None, + port: None, + }; + + // Both empty, no fallback → defaults to us-east-1 + assert_eq!( + resource.endpoint_with_region_fallback(None), + "s3.us-east-1.amazonaws.com" + ); + + // Both empty, fallback provided + assert_eq!( + resource.endpoint_with_region_fallback(Some("eu-west-1".to_string())), + "s3.eu-west-1.amazonaws.com" + ); + + // Region set, endpoint empty → use region + let with_region = S3Resource { + region: "ap-southeast-1".to_string(), + ..resource.clone() + }; + assert_eq!( + with_region.endpoint_with_region_fallback(Some("ignored".to_string())), + "s3.ap-southeast-1.amazonaws.com" + ); + + // Endpoint set → return as-is + let with_endpoint = S3Resource { + endpoint: "custom.s3.endpoint.com".to_string(), + ..resource.clone() + }; + assert_eq!( + with_endpoint.endpoint_with_region_fallback(Some("ignored".to_string())), + "custom.s3.endpoint.com" + ); + } + + #[test] + fn test_lfs_methods_filesystem() { + let rules = vec![S3PermissionRule { + pattern: "**/*.csv".to_string(), + allow: S3Permission::READ, + }]; + let lfs = LargeFileStorage::FilesystemStorage(FilesystemStorage { + root_path: "/data/workspace".to_string(), + public_resource: Some(true), + advanced_permissions: Some(rules.clone()), + }); + + assert_eq!(lfs.get_s3_resource_path(), "/data/workspace"); + assert!(lfs.is_public_resource()); + let perms = lfs.get_advanced_permissions().unwrap(); + assert_eq!(perms.len(), 1); + assert_eq!(perms[0].pattern, "**/*.csv"); + + // Default public_resource = false when None + let lfs_default = LargeFileStorage::FilesystemStorage(FilesystemStorage { + root_path: "/tmp".to_string(), + public_resource: None, + advanced_permissions: None, + }); + assert!(!lfs_default.is_public_resource()); + assert!(lfs_default.get_advanced_permissions().is_none()); + } + + #[test] + fn test_bundle_format_from_string() { + assert_eq!(BundleFormat::from_string("esm"), Some(BundleFormat::Esm)); + assert_eq!(BundleFormat::from_string("cjs"), Some(BundleFormat::Cjs)); + assert_eq!(BundleFormat::from_string("unknown"), None); + assert_eq!(BundleFormat::from_string(""), None); + assert_eq!(BundleFormat::from_string("ESM"), None); // case sensitive + } +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 83fd38e905..5cf0a4c26e 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,16 +10,16 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-sandbox/private"] +private = [] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-sandbox/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth bigquery = ["dep:gcp_auth"] benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"] -parquet = ["windmill-common/parquet", "windmill-sandbox/parquet", "dep:object_store"] +parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] flow_testing = [] cloud = [] sqlx = [] @@ -45,7 +45,8 @@ windmill-queue.workspace = true windmill-dep-map.workspace = true windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker. windmill-common = { workspace = true, default-features = false } -windmill-sandbox.workspace = true +windmill-types.workspace = true +windmill-object-store.workspace = true windmill-jseval.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } @@ -120,9 +121,6 @@ hex.workspace = true tiberius = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } tar.workspace = true -flate2.workspace = true -tempfile.workspace = true -object_store = { workspace = true, optional = true} convert_case.workspace = true yaml-rust.workspace = true backon.workspace = true diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs index 946c922fed..7eb6bb6597 100644 --- a/backend/windmill-worker/src/ai/image_handler.rs +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -1,7 +1,8 @@ use base64::Engine; use futures; use ulid; -use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; +use windmill_common::{client::AuthedClient, error::Error}; +use windmill_types::s3::S3Object; use windmill_queue::MiniPulledJob; use crate::ai::types::*; diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 97176869c5..04f1b4b548 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -1,8 +1,9 @@ use async_trait::async_trait; use windmill_common::{ - client::AuthedClient, error::Error, s3_helpers::S3Object, worker::Connection, + client::AuthedClient, error::Error, worker::Connection, }; use windmill_queue::MiniPulledJob; +use windmill_types::s3::S3Object; use crate::{ ai::{ diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index bd59ee79cc..eebe09bc77 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -19,8 +19,8 @@ use windmill_common::{ error::Error, flow_status::AgentAction, flows::FlowModule, - s3_helpers::S3Object, }; +use windmill_types::s3::S3Object; use windmill_parser::Typ; // Re-export shared types from windmill_common::ai_types diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index d9d34682ba..2f748fe22c 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -6,7 +6,7 @@ use reqwest::Client; use serde_json::{json, value::RawValue, Value}; use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; -use windmill_common::s3_helpers::convert_json_line_stream; +use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 992c592646..7e0b16bd0b 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -27,11 +27,11 @@ use crate::{ }; use windmill_common::{ client::AuthedClient, - s3_helpers::BundleFormat, scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang}, utils::WarnAfterExt, workspace_dependencies::WorkspaceDependenciesPrefetched, }; +use windmill_types::s3::BundleFormat; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -43,12 +43,13 @@ use tokio::io::AsyncReadExt; use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, - worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING}, + worker::{write_file, Connection, DISABLE_BUNDLING}, DB, }; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use windmill_common::s3_helpers::attempt_fetch_bytes; +use windmill_object_store::attempt_fetch_bytes; +use crate::global_cache::{exists_in_cache, save_cache}; use windmill_parser::Typ; @@ -699,7 +700,7 @@ struct PulledCodebase { is_esm: bool, } async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result { - let path = windmill_common::s3_helpers::bundle(&w_id, &id); + let path = windmill_object_store::bundle(&w_id, &id); let CodebaseInfo { is_tar, is_esm } = id_to_codebase_info(id); let bun_cache_path = format!( @@ -719,7 +720,7 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result = None; @@ -977,7 +978,7 @@ pub async fn handle_bun_job( }; let (cache, logs) = - windmill_common::worker::load_cache(&local_path, &remote_path, false).await; + crate::global_cache::load_cache(&local_path, &remote_path, false).await; (cache, logs, local_path, remote_path) } else { (false, "".to_string(), "".to_string(), "".to_string()) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 9984248f79..e03f90525e 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -16,9 +16,9 @@ use tokio::{fs::File, io::AsyncReadExt}; use windmill_common::flows::Step; #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::{ - get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object, -}; +use windmill_types::s3::{LargeFileStorage, ObjectStoreResource, S3Object}; +#[cfg(feature = "parquet")] +use windmill_object_store::get_etag_or_empty; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt}; use windmill_common::worker::{ to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType, @@ -893,9 +893,8 @@ async fn get_workspace_s3_resource_path( storage: Option<&String>, job_id: &Uuid, ) -> windmill_common::error::Result> { - use windmill_common::{ - job_s3_helpers_oss::get_s3_resource_internal, s3_helpers::StorageResourceType, - }; + use windmill_object_store::job_s3_helpers_oss::get_s3_resource_internal; + use windmill_types::s3::StorageResourceType; let raw_lfs_opt = if let Some(storage) = storage { sqlx::query_scalar!( @@ -948,7 +947,10 @@ async fn get_workspace_s3_resource_path( resource_path.to_string(), ) } - Some(LargeFileStorage::FilesystemStorage(_)) | None => { + Some(LargeFileStorage::FilesystemStorage(fs)) => { + (StorageResourceType::Filesystem, fs.root_path.clone()) + } + None => { return Ok(None); } }; @@ -962,21 +964,22 @@ async fn get_workspace_s3_resource_path( let object_store_resource = get_s3_resource_internal( rt, s3_resource_value_raw, - windmill_common::job_s3_helpers_oss::TokenGenerator::AsClient(client), + windmill_object_store::job_s3_helpers_oss::TokenGenerator::AsClient(client), db, ) .await?; // Check bucket workspace restrictions - use windmill_common::s3_helpers::ObjectStoreResource; + use windmill_object_store::ObjectStoreResource; let bucket_name = match &object_store_resource { ObjectStoreResource::S3(s3_resource) => Some(&s3_resource.bucket), ObjectStoreResource::Azure(azure_resource) => Some(&azure_resource.container_name), ObjectStoreResource::Gcs(gcs_resource) => Some(&gcs_resource.bucket), + ObjectStoreResource::Filesystem(_) => None, }; if let Some(bucket) = bucket_name { - windmill_common::s3_helpers::check_bucket_workspace_restriction(bucket, workspace_id)?; + windmill_object_store::check_bucket_workspace_restriction(bucket, workspace_id)?; } Ok(Some(object_store_resource)) @@ -1364,8 +1367,8 @@ impl S3ModeWorkerData { .await } - pub fn to_return_s3_obj(&self) -> windmill_common::s3_helpers::S3Object { - windmill_common::s3_helpers::S3Object { + pub fn to_return_s3_obj(&self) -> windmill_types::s3::S3Object { + windmill_types::s3::S3Object { s3: self.object_key.clone(), storage: self.storage.clone(), ..Default::default() @@ -1554,3 +1557,4 @@ mod tests { assert!(result.is_err()); } } + diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index fbe8392110..880e89ae5e 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -15,11 +15,13 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] use windmill_common::{ utils::calculate_hash, - worker::{save_cache, write_file}, + worker::write_file, }; use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] +use crate::global_cache::save_cache; +#[cfg(feature = "csharp")] use windmill_queue::append_logs; use windmill_queue::CanceledBy; @@ -518,7 +520,7 @@ pub async fn handle_csharp_job( let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); let (cache, cache_logs) = - windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; + crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { #[cfg(unix)] diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 79cffe839d..8345cbd7c2 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -10,7 +10,8 @@ use serde_json::value::RawValue; use serde_json::{json, Value}; use uuid::Uuid; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::s3_helpers::{S3Object, S3_PROXY_LAST_ERRORS_CACHE}; +use windmill_types::s3::S3Object; +use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE; use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ @@ -29,7 +30,7 @@ use crate::mysql_executor::MysqlDatabase; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; use crate::sql_utils::remove_comments; use windmill_common::client::AuthedClient; -use windmill_common::s3_helpers::DEFAULT_STORAGE; +use windmill_object_store::DEFAULT_STORAGE; pub async fn do_duckdb( job: &MiniPulledJob, diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 6ab2acb441..812be2fc7a 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -1,11 +1,8 @@ -// #[cfg(feature = "enterprise")] -// use rand::Rng; - use tokio::time::Instant; use windmill_common::error; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use object_store::ObjectStore; +use windmill_object_store::object_store_reexports::ObjectStore; #[cfg(all(feature = "enterprise", feature = "parquet"))] use std::sync::Arc; @@ -21,7 +18,7 @@ pub async fn build_tar_and_push( custom_folder_name: Option, platform_agnostic: bool, ) -> error::Result<()> { - use object_store::path::Path; + use windmill_object_store::object_store_reexports::Path; use tokio::fs::create_dir_all; use crate::TAR_PYBASE_CACHE_DIR; @@ -94,7 +91,7 @@ pub async fn pull_from_tar( custom_folder_name: Option, platform_agnostic: bool, ) -> error::Result<()> { - use windmill_common::s3_helpers::attempt_fetch_bytes; + use windmill_object_store::attempt_fetch_bytes; let folder_name = if let Some(name) = custom_folder_name { name @@ -147,3 +144,142 @@ pub fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { ); Ok(()) } + +/// Two-tier cache load: check local disk first, then fall back to instance object store. +/// Returns `(hit, log_message)`. +pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) { + if tokio::fs::metadata(&bin_path).await.is_ok() { + (true, format!("loaded from local cache: {}\n", bin_path)) + } else { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = windmill_object_store::get_object_store().await { + let started = std::time::Instant::now(); + + if let Ok(mut x) = windmill_object_store::attempt_fetch_bytes(os, _remote_path).await { + if is_dir { + if let Err(e) = windmill_common::worker::extract_tar(x, bin_path).await { + tracing::error!("could not write tar archive locally: {e:?}"); + return ( + false, + "error writing tar archive from object store".to_string(), + ); + } + } else { + if let Err(e) = windmill_common::worker::write_binary_file(bin_path, &mut x) { + tracing::error!("could not write bundle/bin file locally: {e:?}"); + return ( + false, + "error writing bundle/bin file from object store".to_string(), + ); + } + } + tracing::info!("loaded from object store {}", bin_path); + return ( + true, + format!( + "loaded bin/bundle from object store {} in {}ms", + bin_path, + started.elapsed().as_millis() + ), + ); + } + } + let _ = is_dir; + (false, "".to_string()) + } +} + +/// Check whether a binary/bundle exists in local cache or instance object store. +pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { + if tokio::fs::metadata(&bin_path).await.is_ok() { + return true; + } else { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = windmill_object_store::get_object_store().await { + return os + .get(&windmill_object_store::object_store_reexports::Path::from(_remote_path)) + .await + .is_ok(); + } + return false; + } +} + +/// Two-tier cache write: upload to instance object store, then copy to local disk. +pub async fn save_cache( + local_cache_path: &str, + _remote_cache_path: &str, + origin: &str, + is_dir: bool, +) -> windmill_common::error::Result { + use std::path::PathBuf; + + let mut _cached_to_s3 = false; + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = windmill_object_store::get_object_store().await { + use windmill_object_store::object_store_reexports::Path; + let file_to_cache = if is_dir { + let tar_path = format!( + "{}/tar/{}_tar.tar", + windmill_common::worker::ROOT_CACHE_DIR, + local_cache_path + .split("/") + .last() + .unwrap_or(&uuid::Uuid::new_v4().to_string()) + ); + let tar_file = std::fs::File::create(&tar_path)?; + let mut tar = tar::Builder::new(tar_file); + tar.append_dir_all(".", &origin)?; + let tar_metadata = tokio::fs::metadata(&tar_path).await; + if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 { + tracing::info!("Failed to tar cache: {origin}"); + return Err(error::Error::ExecutionErr(format!( + "Failed to tar cache: {origin}" + ))); + } + tar_path + } else { + origin.to_owned() + }; + + if let Err(e) = os + .put( + &Path::from(_remote_cache_path), + std::fs::read(&file_to_cache)?.into(), + ) + .await + { + tracing::error!( + "Failed to put bin to object store: {_remote_cache_path}. Error: {:?}", + e + ); + } else { + _cached_to_s3 = true; + if is_dir { + tokio::fs::remove_dir_all(&file_to_cache).await?; + } + } + } + + if true { + if is_dir { + windmill_common::worker::copy_dir_recursively( + &PathBuf::from(origin), + &PathBuf::from(local_cache_path), + )?; + } else { + std::fs::copy(origin, local_cache_path)?; + } + Ok(format!( + "\nwrote cached binary: {} (backed by EE distributed object store: {_cached_to_s3})\n", + local_cache_path + )) + } else if _cached_to_s3 { + Ok(format!( + "wrote cached binary to object store {}\n", + local_cache_path + )) + } else { + Ok("".to_string()) + } +} diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index da780ce7ea..d8936d1dc4 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -13,8 +13,9 @@ use uuid::Uuid; use windmill_common::{ error::{self, Error}, utils::calculate_hash, - worker::{save_cache, write_file, Connection, GoAnnotations}, + worker::{write_file, Connection, GoAnnotations}, }; +use crate::global_cache::save_cache; use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -110,7 +111,7 @@ pub async fn handle_go_job( let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); let (cache, cache_logs) = - windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; + crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let (skip_go_mod, skip_tidy) = if cache { (true, true) diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index e4dad0777f..b9bde76577 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -13,8 +13,9 @@ use uuid::Uuid; use windmill_common::{ error::{self, Error}, utils::calculate_hash, - worker::{copy_dir_recursively, save_cache, write_file, Connection}, + worker::{copy_dir_recursively, write_file, Connection}, }; +use crate::global_cache::save_cache; use windmill_parser::Arg; use windmill_parser_java::parse_java_sig_meta; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -466,7 +467,7 @@ async fn compile<'a>( let hash = compute_hash(inner_content, *requirements_o); let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); - let (cache, ..) = windmill_common::worker::load_cache(&bin_path, &remote_path, true).await; + let (cache, ..) = crate::global_cache::load_cache(&bin_path, &remote_path, true).await; if cache { let target = format!("{job_dir}/target"); diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index e836018235..3a62481935 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -11,7 +11,7 @@ use tiberius::{ use tokio::net::TcpStream; use tokio_util::compat::TokioAsyncWriteCompatExt; use uuid::Uuid; -use windmill_common::s3_helpers::convert_json_line_stream; +use windmill_object_store::convert_json_line_stream; use windmill_common::utils::merge_raw_values_to_object; use windmill_common::worker::SqlResultCollectionStrategy; use windmill_common::{ diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 99f67ed456..a3a9468459 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -15,9 +15,9 @@ use tokio::sync::Mutex; use windmill_common::{ client::AuthedClient, error::{to_anyhow, Error}, - s3_helpers::convert_json_line_stream, worker::{to_raw_value, Connection, SqlResultCollectionStrategy}, }; +use windmill_object_store::convert_json_line_stream; use windmill_parser_sql::{ parse_db_resource, parse_mysql_sig, parse_s3_mode, parse_sql_blocks, parse_sql_statement_named_params, RE_ARG_MYSQL_NAMED, diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index 95537a658c..bc90ba7811 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, value::RawValue, Value}; use windmill_common::{ error::{to_anyhow, Error}, - s3_helpers::convert_json_line_stream, worker::{to_raw_value, Connection, SqlResultCollectionStrategy}, }; +use windmill_object_store::convert_json_line_stream; use windmill_queue::MiniPulledJob; use windmill_parser_sql::{ diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 3540cd0bdd..7738aea58f 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -24,7 +24,7 @@ use tokio_postgres::{ use uuid::Uuid; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; -use windmill_common::s3_helpers::convert_json_line_stream; +use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{ to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, }; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 0e4e870f86..d469721248 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -93,7 +93,7 @@ struct PiptarUploadTask { #[cfg(all(feature = "enterprise", feature = "parquet", unix))] async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver) { use crate::global_cache::build_tar_and_push; - use windmill_common::s3_helpers::get_object_store; + use windmill_object_store::get_object_store; while let Some(task) = rx.recv().await { if let Some(os) = get_object_store().await { @@ -123,7 +123,7 @@ const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); use crate::global_cache::pull_from_tar; #[cfg(all(feature = "enterprise", feature = "parquet", unix))] -use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; +use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ @@ -1920,7 +1920,7 @@ pub async fn handle_python_reqs( let start = std::time::Instant::now(); #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if is_not_pro { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + if let Some(os) = windmill_object_store::get_object_store().await { tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(Error::from(anyhow::anyhow!("S3 pull was canceled"))), diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index b4994b3c51..95eb71999b 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -14,8 +14,9 @@ use tokio::{ use windmill_common::{ error::{self, Error}, utils::calculate_hash, - worker::{save_cache, write_file, Connection}, + worker::{write_file, Connection}, }; +use crate::global_cache::save_cache; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -611,7 +612,7 @@ pub async fn handle_rust_job( get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let (cache, cache_logs) = - windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; + crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { let target = format!("{job_dir}/main"); diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 4c68d76135..9f89f2fc44 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,7 +9,7 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; -use windmill_common::s3_helpers::convert_json_line_stream; +use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index c16583ff76..f129371ca1 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -175,7 +175,7 @@ pub async fn par_install_language_dependencies_all_at_once< mark_success(path.clone(), job_id, w_id).await; #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + if let Some(os) = windmill_object_store::get_object_store().await { let language_name = _language_name.to_owned(); tokio::spawn(async move { if let Err(e) = crate::global_cache::build_tar_and_push( @@ -524,7 +524,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] let s3_pull_future = if is_not_pro { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + if let Some(os) = windmill_object_store::get_object_store().await { Some(crate::global_cache::pull_from_tar( os, dep.path.clone(), @@ -624,7 +624,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + if let Some(os) = windmill_object_store::get_object_store().await { let language_name = _language_name.to_string(); let platform_agnostic = _platform_agnostic; let path = dep.path.clone(); @@ -685,7 +685,7 @@ async fn print_success( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if windmill_common::s3_helpers::OBJECT_STORE_SETTINGS + if windmill_object_store::OBJECT_STORE_SETTINGS .read() .await .is_none() diff --git a/backend/write_latest_ee_ref.sh b/backend/write_latest_ee_ref.sh index 476d4a7d57..dc35506551 100755 --- a/backend/write_latest_ee_ref.sh +++ b/backend/write_latest_ee_ref.sh @@ -1,7 +1,7 @@ #!/bin/bash # Navigate to the target directory -cd ../../windmill-ee-private || { echo "Directory not found"; exit 1; } +cd ../../windmill-ee-private || cd ~/windmill-ee-private || { echo "Directory not found"; exit 1; } # Get the current commit hash commit_hash=$(git rev-parse HEAD) @@ -13,4 +13,4 @@ cd - || exit echo -n "$commit_hash" > ./ee-repo-ref.txt # Confirmation message -echo "Commit hash $commit_hash written to ee-repo-ref.txt" \ No newline at end of file +echo "Commit hash $commit_hash written to ee-repo-ref.txt"