diff --git a/backend/src/main.rs b/backend/src/main.rs index 2339b7c7c2..13e0d8b46a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -22,11 +22,12 @@ use tokio::{ use windmill_api::HTTP_CLIENT; use windmill_common::{ global_settings::{ - BASE_URL_SETTING, CUSTOM_TAGS_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CUSTOM_TAGS_SETTING, + DISABLE_STATS_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, }, stats::schedule_stats, utils::{rd_string, Mode}, @@ -42,9 +43,10 @@ use windmill_worker::{ use crate::monitor::{ initial_load, load_keep_job_dir, load_require_preexisting_user, monitor_db, monitor_pool, - reload_base_url_setting, reload_extra_pip_index_url_setting, - reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting, - reload_retention_period_setting, reload_server_config, reload_worker_config, + reload_base_url_setting, reload_bunfig_install_scopes_setting, + reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key, + reload_npm_config_registry_setting, reload_retention_period_setting, reload_server_config, + reload_worker_config, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -426,6 +428,9 @@ Windmill Community Edition {GIT_VERSION} NPM_CONFIG_REGISTRY_SETTING => { reload_npm_config_registry_setting(&db).await }, + BUNFIG_INSTALL_SCOPES_SETTING => { + reload_bunfig_install_scopes_setting(&db).await + }, KEEP_JOB_DIR_SETTING => { load_keep_job_dir(&db).await; }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d372f7c5cb..758b0e02ac 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -21,9 +21,9 @@ use windmill_api::{ use windmill_common::{ error, global_settings::{ - BASE_URL_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, + EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, }, @@ -35,8 +35,9 @@ use windmill_common::{ BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED, }; use windmill_worker::{ - create_token_for_owner, handle_job_error, AuthedClient, SendResult, JOB_DEFAULT_TIMEOUT, - KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY, + create_token_for_owner, handle_job_error, AuthedClient, SendResult, BUNFIG_INSTALL_SCOPES, + JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, + SCRIPT_TOKEN_EXPIRY, }; #[cfg(feature = "enterprise")] @@ -138,6 +139,9 @@ pub async fn initial_load( if worker_mode { reload_npm_config_registry_setting(&db).await; } + if worker_mode { + reload_bunfig_install_scopes_setting(&db).await; + } } pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> { @@ -302,29 +306,33 @@ pub async fn delete_expired_items(db: &DB) -> () { } pub async fn reload_extra_pip_index_url_setting(db: &DB) { - if let Err(e) = reload_option_setting( + reload_option_setting_with_tracing( db, EXTRA_PIP_INDEX_URL_SETTING, "PIP_EXTRA_INDEX_URL", PIP_EXTRA_INDEX_URL.clone(), ) - .await - { - tracing::error!("Error reloading extra_pip_index_url period: {:?}", e) - } + .await; } pub async fn reload_npm_config_registry_setting(db: &DB) { - if let Err(e) = reload_option_setting( + reload_option_setting_with_tracing( db, NPM_CONFIG_REGISTRY_SETTING, "NPM_CONFIG_REGISTRY", NPM_CONFIG_REGISTRY.clone(), ) - .await - { - tracing::error!("Error reloading npm_config_registry period: {:?}", e) - } + .await; +} + +pub async fn reload_bunfig_install_scopes_setting(db: &DB) { + reload_option_setting_with_tracing( + db, + BUNFIG_INSTALL_SCOPES_SETTING, + "BUNFIG_INSTALL_SCOPES", + BUNFIG_INSTALL_SCOPES.clone(), + ) + .await; } pub async fn reload_retention_period_setting(db: &DB) { @@ -343,16 +351,13 @@ pub async fn reload_retention_period_setting(db: &DB) { } pub async fn reload_job_default_timeout_setting(db: &DB) { - if let Err(e) = reload_option_setting( + reload_option_setting_with_tracing( db, JOB_DEFAULT_TIMEOUT_SECS_SETTING, "JOB_DEFAULT_TIMEOUT_SECS", JOB_DEFAULT_TIMEOUT.clone(), ) - .await - { - tracing::error!("Error reloading job default timeout: {:?}", e) - } + .await; } pub async fn reload_request_size(db: &DB) { @@ -400,6 +405,16 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> { Ok(()) } +pub async fn reload_option_setting_with_tracing( + db: &DB, + setting_name: &str, + std_env_var: &str, + lock: Arc>>, +) { + if let Err(e) = reload_option_setting(db, setting_name, std_env_var, lock.clone()).await { + tracing::error!("Error reloading setting {}: {:?}", setting_name, e) + } +} pub async fn reload_option_setting( db: &DB, setting_name: &str, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 12e12e5bb8..1264eece99 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -7,6 +7,8 @@ pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout"; pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; +pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; + pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const UNIQUE_ID_SETTING: &str = "uid"; pub const DISABLE_STATS_SETTING: &str = "disable_stats"; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index d2672caecf..4f2679c934 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -15,8 +15,8 @@ use crate::{ create_args_and_out_file, get_reserved_variables, handle_child, parse_npm_config, read_result, set_logs, start_child_process, write_file, write_file_binary, }, - AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, - NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV, + AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, + DISABLE_NUSER, HOME_ENV, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; use tokio::{ @@ -50,7 +50,6 @@ pub const EMPTY_FILE: &str = ""; lazy_static::lazy_static! { pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap(); - pub static ref BUN_BUNFIG_EXTRA: Option = std::env::var("BUN_BUNFIG_EXTRA").ok(); } pub async fn gen_lockfile( @@ -88,7 +87,8 @@ pub async fn gen_lockfile( // if custom NPM registry is being used, write bunfig.toml at the root of the job dir let registry = NPM_CONFIG_REGISTRY.read().await.clone(); - if registry.is_some() || BUN_BUNFIG_EXTRA.is_some() { + let bunfig_install_scopes = BUNFIG_INSTALL_SCOPES.read().await.clone(); + if registry.is_some() || bunfig_install_scopes.is_some() { let (url, token_opt) = if let Some(ref s) = registry { let url = s.trim(); if url.is_empty() { @@ -112,7 +112,9 @@ registry = {} {} "#, registry_toml_string, - BUN_BUNFIG_EXTRA.as_ref().unwrap_or(&"".to_string()) + bunfig_install_scopes + .map(|x| format!("[install.scopes]\n{x}")) + .unwrap_or("".to_string()) ); tracing::debug!("Writing following bunfig.toml: {bunfig_toml}"); let _ = write_file(&job_dir, "bunfig.toml", &bunfig_toml).await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index b77f68b89b..4758e3a001 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -663,7 +663,7 @@ pub async fn handle_child( if line.is_empty() { continue; } - append_with_limit(&mut joined, &line, &mut log_remaining, child_name == "powershell run" || child_name == "bash run"); + append_with_limit(&mut joined, &line, &mut log_remaining); if log_remaining == 0 { tracing::info!(%job_id, "Too many logs lines for job {job_id}"); let _ = set_too_many_logs.send(true); @@ -848,15 +848,16 @@ pub fn lines_to_stream( }) } +lazy_static::lazy_static! { + static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap(); +} // as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields, // so this pushes \n to thd destination string in each call -fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize, remove_x00: bool) { +fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { let src_str; - let src = if remove_x00 { - src_str = src.replace('\u{00}', ""); - src_str.as_str() - } else { - src + let src = { + src_str = RE_00.replace_all(src, ""); + src_str.as_ref() }; if !*CLOUD_HOSTED { dst.push('\n'); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b1e0381001..ef2d38153b 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -248,6 +248,8 @@ lazy_static::lazy_static! { pub static ref NETRC: Option = std::env::var("NETRC").ok(); pub static ref NPM_CONFIG_REGISTRY: Arc>> = Arc::new(RwLock::new(None)); + pub static ref BUNFIG_INSTALL_SCOPES: Arc>> = Arc::new(RwLock::new(None)); + pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index b0c4eb44b0..b20840d0a3 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -78,26 +78,7 @@ export const settings: Record = { placeholder: 'only needed to prepare upgrade to EE', storage: 'setting' }, - { - label: 'Pip Extra Index Url', - description: 'Add private PIP registry', - key: 'pip_extra_index_url', - fieldType: 'text', - placeholder: 'https://username:password@pypi.company.com/simple', - storage: 'setting', - ee_only: - 'You can still set this setting by using PIP_EXTRA_INDEX_URL as env variable to the worker containers' - }, - { - label: 'Npm Config Registry', - description: 'Add private NPM registry', - key: 'npm_config_registry', - fieldType: 'text', - placeholder: 'https://yourregistry', - storage: 'setting', - ee_only: - 'You can still set this setting by using NPM_CONFIG_REGISTRY as env variable to the worker containers' - }, + { label: 'Expose metrics', description: 'Expose prometheus metrics for workers and servers on port 8001 at /metrics', @@ -117,6 +98,44 @@ export const settings: Record = { 'You can still set this setting by using OPENAI_AZURE_BASE_PATH as env variable to the server containers' } ], + 'SSO/OAuth': [ + { + label: 'Require users to have been added manually to windmill to sign in through OAuth', + key: 'require_preexisting_user_for_oauth', + fieldType: 'boolean', + storage: 'setting' + } + ], + Registries: [ + { + label: 'Pip Extra Index Url', + description: 'Add private PIP registry', + key: 'pip_extra_index_url', + fieldType: 'text', + placeholder: 'https://username:password@pypi.company.com/simple', + storage: 'setting', + ee_only: '' + }, + { + label: 'Npm Config Registry', + description: 'Add private NPM registry', + key: 'npm_config_registry', + fieldType: 'text', + placeholder: 'https://yourregistry', + storage: 'setting', + ee_only: '' + }, + { + label: 'Bunfig Install Scopes', + description: + 'Add private scoped registries for Bun, See: https://bun.sh/docs/install/registries', + key: 'bunfig_install_scopes', + fieldType: 'text', + placeholder: '"@myorg3" = { token = "mytoken", url = "https://registry.myorg.com/" }', + storage: 'setting', + ee_only: '' + } + ], SMTP: [ { label: 'Host', @@ -159,14 +178,6 @@ export const settings: Record = { storage: 'config' } ], - 'SSO/OAuth': [ - { - label: 'Require users to have been added manually to windmill to sign in through OAuth', - key: 'require_preexisting_user_for_oauth', - fieldType: 'boolean', - storage: 'setting' - } - ], Debug: [ { label: 'Keep Job Directories',