From 543d2a1577efbbd4b80a5135d95ce2fd06b39b1f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 15 Mar 2024 15:09:05 +0100 Subject: [PATCH] fix: s3 cache is an instance settings (#3421) * s3 * s3 --- README.md | 76 +++++++++---------- backend/src/main.rs | 10 ++- backend/src/monitor.rs | 15 +++- .../windmill-common/src/global_settings.rs | 7 +- backend/windmill-common/src/s3_helpers.rs | 8 +- backend/windmill-worker/src/global_cache.rs | 7 +- .../windmill-worker/src/python_executor.rs | 6 +- backend/windmill-worker/src/worker.rs | 12 +-- .../lib/components/InstanceSettings.svelte | 9 ++- frontend/src/lib/components/Password.svelte | 7 +- .../src/lib/components/instanceSettings.ts | 9 +++ 11 files changed, 99 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 66d95de81c..93a0989e97 100644 --- a/README.md +++ b/README.md @@ -198,9 +198,8 @@ page. ### Sandboxing -Windmill uses [nsjail](https://github.com/google/nsjail) and deno's sandboxing -capabilities. It is production multi-tenant grade secure. Do not take our word -for it, take +Windmill can use [nsjail](https://github.com/google/nsjail). It is production +multi-tenant grade secure. Do not take our word for it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/). ### Secrets, credentials and sensitive values @@ -317,42 +316,41 @@ you to have it being synced automatically everyday. ## Environment Variables -| Environment Variable name | Default | Description | Api Server/Worker/All | -| ------------------------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | -| DATABASE_URL | | The Postgres database url. | All | -| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker | -| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server | All | -| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All | -| JSON_FMT | false | Output the logs in json format instead of logfmt | All | -| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server | -| TIMEOUT | 60 _ 60 _ 24 \* 7 (1 week) | The maximum time of execution of a script. When reached, the job is failed as having timedout. | -| SCRIPT_TOKEN_EXPIRY | 900 | The default duration period of the ephemeral-token generated at the beginning of a script | Worker | -| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | -| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | -| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | -| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker | -| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker | -| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | -| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | -| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker | -| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | -| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | -| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker | -| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker | -| GO_PATH | /usr/bin/go | The path to the go binary. | Worker | -| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | -| GOPROXY | | The GOPROXY env variable to use | Worker | -| NETRC | | The netrc content to use a private go registry | Worker | -| PIP_INDEX_URL | None | The index url to pass for pip. | Worker | -| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker | -| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker | -| PATH | None | The path environment variable, usually inherited | Worker | -| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | -| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | -| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | -| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | -| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | -| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | +| Environment Variable name | Default | Description | Api Server/Worker/All | +| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| DATABASE_URL | | The Postgres database url. | All | +| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker | +| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server | All | +| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All | +| JSON_FMT | false | Output the logs in json format instead of logfmt | All | +| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server | +| SCRIPT_TOKEN_EXPIRY | 900 | The default duration period of the ephemeral-token generated at the beginning of a script | Worker | +| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | +| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | +| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | +| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker | +| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker | +| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | +| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | +| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker | +| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | +| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | +| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker | +| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker | +| GO_PATH | /usr/bin/go | The path to the go binary. | Worker | +| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | +| GOPROXY | | The GOPROXY env variable to use | Worker | +| NETRC | | The netrc content to use a private go registry | Worker | +| PIP_INDEX_URL | None | The index url to pass for pip. | Worker | +| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker | +| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker | +| PATH | None | The path environment variable, usually inherited | Worker | +| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | +| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | +| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | +| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | +| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | +| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | ## Contributors diff --git a/backend/src/main.rs b/backend/src/main.rs index ef8b2cafed..d51e396ff6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -24,7 +24,8 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + RETENTION_PERIOD_SECS_SETTING, S3_CACHE_BUCKET_SETTING, SAML_METADATA_SETTING, + SCIM_TOKEN_SETTING, }, stats_ee::schedule_stats, utils::{rd_string, Mode}, @@ -47,8 +48,8 @@ use crate::monitor::{ monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_scim_token_setting, reload_server_config, - reload_worker_config, + reload_retention_period_setting, reload_s3_cache_bucket_setting, reload_scim_token_setting, + reload_server_config, reload_worker_config, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -440,6 +441,9 @@ Windmill Community Edition {GIT_VERSION} JOB_DEFAULT_TIMEOUT_SECS_SETTING => { reload_job_default_timeout_setting(&db).await }, + S3_CACHE_BUCKET_SETTING => { + reload_s3_cache_bucket_setting(&db).await + }, SCIM_TOKEN_SETTING => { reload_scim_token_setting(&db).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 08ebc9edd3..79cb319778 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -27,10 +27,12 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + RETENTION_PERIOD_SECS_SETTING, S3_CACHE_BUCKET_SETTING, SAML_METADATA_SETTING, + SCIM_TOKEN_SETTING, }, jobs::QueuedJob, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, + s3_helpers::S3_CACHE_BUCKET, server::load_server_config, users::truncate_token, worker::{ @@ -126,6 +128,7 @@ pub async fn initial_load( tracing::error!("Error reloading base url: {:?}", e) } + reload_s3_cache_bucket_setting(&db).await; if server_mode { reload_server_config(&db).await; reload_retention_period_setting(&db).await; @@ -399,6 +402,16 @@ pub async fn reload_retention_period_setting(db: &DB) { } } +pub async fn reload_s3_cache_bucket_setting(db: &DB) { + reload_option_setting_with_tracing( + db, + S3_CACHE_BUCKET_SETTING, + "S3_CACHE_BUCKET", + S3_CACHE_BUCKET.clone(), + ) + .await; +} + pub async fn reload_job_default_timeout_setting(db: &DB) { reload_option_setting_with_tracing( db, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index ec4da9da70..497ee5f9d3 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -21,10 +21,11 @@ pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics"; pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics"; pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; +pub const S3_CACHE_BUCKET_SETTING: &str = "s3_cache_bucket"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; -pub const ENV_SETTINGS: [&str; 54] = [ +pub const ENV_SETTINGS: [&str; 50] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -69,10 +70,6 @@ pub const ENV_SETTINGS: [&str; 54] = [ "WAIT_RESULT_FAST_POLL_INTERVAL_MS", "EXIT_AFTER_NO_JOB_FOR_SECS", "REQUEST_SIZE_LIMIT", - "SMTP_HOST", - "SMTP_USERNAME", - "SMTP_PORT", - "SMTP_TLS_IMPLICIT", "CREATE_WORKSPACE_REQUIRE_SUPERADMIN", "GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE", "MAX_WAIT_FOR_SIGINT", diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 2ea94fc604..f0fd08874d 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -7,8 +7,14 @@ use object_store::ObjectStore; #[cfg(feature = "parquet")] use object_store::{aws::AmazonS3Builder, ClientOptions}; use serde::{Deserialize, Serialize}; -#[cfg(feature = "parquet")] use std::sync::Arc; +use tokio::sync::RwLock; + +lazy_static::lazy_static! { + + pub static ref S3_CACHE_BUCKET: Arc>> = Arc::new(RwLock::new(None)); + +} #[derive(Serialize, Deserialize, Debug)] #[serde(tag = "type")] diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 4845e3187f..87d2087f2b 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -22,7 +22,7 @@ use windmill_common::error; // const TAR_CACHE_FILENAME: &str = "denogocache.tar"; #[cfg(feature = "enterprise")] -pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<()> { +pub async fn build_tar_and_push(bucket: String, folder: String) -> error::Result<()> { tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); let folder_name = folder.split("/").last().unwrap(); @@ -47,6 +47,8 @@ pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<( ))); } + let bucket = bucket.trim_start_matches("s3://"); + if let Err(e) = execute_command( ROOT_TMP_CACHE_DIR, "rclone", @@ -84,6 +86,8 @@ pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> { let start = Instant::now(); let tar_path = format!("tar/pip/{folder_name}.tar"); let target = format!("{ROOT_TMP_CACHE_DIR}/{tar_path}.single"); + let bucket = bucket.trim_start_matches("s3://"); + if let Err(e) = execute_command( ROOT_TMP_CACHE_DIR, "rclone", @@ -389,6 +393,7 @@ pub async fn copy_all_piptars_from_bucket(bucket: &str) { let start = Instant::now(); + let bucket = bucket.trim_start_matches("s3://"); if let Err(e) = execute_command( ROOT_CACHE_DIR, "rclone", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index eec71aefe7..4d41aae993 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -51,7 +51,7 @@ const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); use crate::global_cache::{build_tar_and_push, pull_from_tar}; #[cfg(feature = "enterprise")] -use crate::S3_CACHE_BUCKET; +use windmill_common::s3_helpers::S3_CACHE_BUCKET; use crate::{ common::{ @@ -779,7 +779,7 @@ pub async fn handle_python_reqs( } #[cfg(feature = "enterprise")] - if let Some(ref bucket) = *S3_CACHE_BUCKET { + if let Some(ref bucket) = S3_CACHE_BUCKET.read().await.clone() { if matches!(get_license_plan().await, LicensePlan::Pro) { tracing::warn!("S3 cache not available in the pro plan"); } else { @@ -922,7 +922,7 @@ pub async fn handle_python_reqs( child?; #[cfg(feature = "enterprise")] - if let Some(ref bucket) = *S3_CACHE_BUCKET { + if let Some(bucket) = S3_CACHE_BUCKET.read().await.clone() { if matches!(get_license_plan().await, LicensePlan::Pro) { tracing::warn!("S3 cache not available in the pro plan"); } else { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index cd31dc8e5a..472977518c 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -59,6 +59,9 @@ use windmill_queue::{ #[cfg(feature = "prometheus")] use windmill_queue::register_metric; +#[cfg(feature = "enterprise")] +use windmill_common::s3_helpers::S3_CACHE_BUCKET; + use serde_json::{json, value::RawValue, Value}; #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -328,11 +331,6 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(60 * 10); - pub static ref S3_CACHE_BUCKET: Option = std::env::var("S3_CACHE_BUCKET") - .ok() - .map(|e| Some(e)) - .unwrap_or(None); - pub static ref EXIT_AFTER_NO_JOB_FOR_SECS: Option = std::env::var("EXIT_AFTER_NO_JOB_FOR_SECS") .ok() @@ -944,12 +942,10 @@ pub async fn run_worker {/if} {:else if setting.fieldType == 'license_key'} -
-