starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-23 08:00:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add support for bun install scopes

Browse Source
This commit is contained in:
Ruben Fiszel
2024-01-12 23:48:58 +01:00
parent 596222f650
commit 22f502d2a8
7 changed files with 106 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main.rs
+13 -8
View File
@@ -22,11 +22,12 @@ use tokio::{
use windmill_api::HTTP_CLIENT;
use windmill_common::{
global_settings::{
BASE_URL_SETTING, CUSTOM_TAGS_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CUSTOM_TAGS_SETTING,
DISABLE_STATS_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING,
},
stats::schedule_stats,
utils::{rd_string, Mode},
@@ -42,9 +43,10 @@ use windmill_worker::{
use crate::monitor::{
initial_load, load_keep_job_dir, load_require_preexisting_user, monitor_db, monitor_pool,
reload_base_url_setting, reload_extra_pip_index_url_setting,
reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting,
reload_retention_period_setting, reload_server_config, reload_worker_config,
reload_base_url_setting, reload_bunfig_install_scopes_setting,
reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key,
reload_npm_config_registry_setting, reload_retention_period_setting, reload_server_config,
reload_worker_config,
};
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
@@ -426,6 +428,9 @@ Windmill Community Edition {GIT_VERSION}
NPM_CONFIG_REGISTRY_SETTING => {
reload_npm_config_registry_setting(&db).await
},
BUNFIG_INSTALL_SCOPES_SETTING => {
reload_bunfig_install_scopes_setting(&db).await
},
KEEP_JOB_DIR_SETTING => {
load_keep_job_dir(&db).await;
},
backend/src/monitor.rs
+35 -20
View File
@@ -21,9 +21,9 @@ use windmill_api::{
use windmill_common::{
error,
global_settings::{
BASE_URL_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, EXPOSE_DEBUG_METRICS_SETTING,
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING,
},
@@ -35,8 +35,9 @@ use windmill_common::{
BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
};
use windmill_worker::{
create_token_for_owner, handle_job_error, AuthedClient, SendResult, JOB_DEFAULT_TIMEOUT,
KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
create_token_for_owner, handle_job_error, AuthedClient, SendResult, BUNFIG_INSTALL_SCOPES,
JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL,
SCRIPT_TOKEN_EXPIRY,
};
#[cfg(feature = "enterprise")]
@@ -138,6 +139,9 @@ pub async fn initial_load(
if worker_mode {
reload_npm_config_registry_setting(&db).await;
}
if worker_mode {
reload_bunfig_install_scopes_setting(&db).await;
}
}
pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
@@ -302,29 +306,33 @@ pub async fn delete_expired_items(db: &DB) -> () {
}
pub async fn reload_extra_pip_index_url_setting(db: &DB) {
if let Err(e) = reload_option_setting(
reload_option_setting_with_tracing(
db,
EXTRA_PIP_INDEX_URL_SETTING,
"PIP_EXTRA_INDEX_URL",
PIP_EXTRA_INDEX_URL.clone(),
)
.await
{
tracing::error!("Error reloading extra_pip_index_url period: {:?}", e)
}
.await;
}
pub async fn reload_npm_config_registry_setting(db: &DB) {
if let Err(e) = reload_option_setting(
reload_option_setting_with_tracing(
db,
NPM_CONFIG_REGISTRY_SETTING,
"NPM_CONFIG_REGISTRY",
NPM_CONFIG_REGISTRY.clone(),
)
.await
{
tracing::error!("Error reloading npm_config_registry period: {:?}", e)
}
.await;
}
pub async fn reload_bunfig_install_scopes_setting(db: &DB) {
reload_option_setting_with_tracing(
db,
BUNFIG_INSTALL_SCOPES_SETTING,
"BUNFIG_INSTALL_SCOPES",
BUNFIG_INSTALL_SCOPES.clone(),
)
.await;
}
pub async fn reload_retention_period_setting(db: &DB) {
@@ -343,16 +351,13 @@ pub async fn reload_retention_period_setting(db: &DB) {
}
pub async fn reload_job_default_timeout_setting(db: &DB) {
if let Err(e) = reload_option_setting(
reload_option_setting_with_tracing(
db,
JOB_DEFAULT_TIMEOUT_SECS_SETTING,
"JOB_DEFAULT_TIMEOUT_SECS",
JOB_DEFAULT_TIMEOUT.clone(),
)
.await
{
tracing::error!("Error reloading job default timeout: {:?}", e)
}
.await;
}
pub async fn reload_request_size(db: &DB) {
@@ -400,6 +405,16 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> {
Ok(())
}
pub async fn reload_option_setting_with_tracing<T: FromStr + DeserializeOwned>(
db: &DB,
setting_name: &str,
std_env_var: &str,
lock: Arc<RwLock<Option<T>>>,
) {
if let Err(e) = reload_option_setting(db, setting_name, std_env_var, lock.clone()).await {
tracing::error!("Error reloading setting {}: {:?}", setting_name, e)
}
}
pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
db: &DB,
setting_name: &str,
backend/windmill-common/src/global_settings.rs
+2
View File
@@ -7,6 +7,8 @@ pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout";
pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
pub const LICENSE_KEY_SETTING: &str = "license_key";
pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes";
pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
pub const UNIQUE_ID_SETTING: &str = "uid";
pub const DISABLE_STATS_SETTING: &str = "disable_stats";
backend/windmill-worker/src/bun_executor.rs
+7 -5
View File
@@ -15,8 +15,8 @@ use crate::{
create_args_and_out_file, get_reserved_variables, handle_child, parse_npm_config,
read_result, set_logs, start_child_process, write_file, write_file_binary,
},
AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV,
NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL,
DISABLE_NUSER, HOME_ENV, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
};
use tokio::{
@@ -50,7 +50,6 @@ pub const EMPTY_FILE: &str = "<empty>";
lazy_static::lazy_static! {
pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap();
pub static ref BUN_BUNFIG_EXTRA: Option<String> = std::env::var("BUN_BUNFIG_EXTRA").ok();
}
pub async fn gen_lockfile(
@@ -88,7 +87,8 @@ pub async fn gen_lockfile(
// if custom NPM registry is being used, write bunfig.toml at the root of the job dir
let registry = NPM_CONFIG_REGISTRY.read().await.clone();
if registry.is_some() || BUN_BUNFIG_EXTRA.is_some() {
let bunfig_install_scopes = BUNFIG_INSTALL_SCOPES.read().await.clone();
if registry.is_some() || bunfig_install_scopes.is_some() {
let (url, token_opt) = if let Some(ref s) = registry {
let url = s.trim();
if url.is_empty() {
@@ -112,7 +112,9 @@ registry = {}
{}
"#,
registry_toml_string,
BUN_BUNFIG_EXTRA.as_ref().unwrap_or(&"".to_string())
bunfig_install_scopes
.map(|x| format!("[install.scopes]\n{x}"))
.unwrap_or("".to_string())
);
tracing::debug!("Writing following bunfig.toml: {bunfig_toml}");
let _ = write_file(&job_dir, "bunfig.toml", &bunfig_toml).await?;
backend/windmill-worker/src/common.rs
+8 -7
View File
@@ -663,7 +663,7 @@ pub async fn handle_child(
if line.is_empty() {
continue;
}
append_with_limit(&mut joined, &line, &mut log_remaining, child_name == "powershell run" || child_name == "bash run");
append_with_limit(&mut joined, &line, &mut log_remaining);
if log_remaining == 0 {
tracing::info!(%job_id, "Too many logs lines for job {job_id}");
let _ = set_too_many_logs.send(true);
@@ -848,15 +848,16 @@ pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
})
}
lazy_static::lazy_static! {
static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
}
// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
// so this pushes \n to thd destination string in each call
fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize, remove_x00: bool) {
fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
let src_str;
let src = if remove_x00 {
src_str = src.replace('\u{00}', "");
src_str.as_str()
} else {
src
let src = {
src_str = RE_00.replace_all(src, "");
src_str.as_ref()
};
if !*CLOUD_HOSTED {
dst.push('\n');
backend/windmill-worker/src/worker.rs
+2
View File
@@ -248,6 +248,8 @@ lazy_static::lazy_static! {
pub static ref NETRC: Option<String> = std::env::var("NETRC").ok();
pub static ref NPM_CONFIG_REGISTRY: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref BUNFIG_INSTALL_SCOPES: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref PIP_EXTRA_INDEX_URL: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref JOB_DEFAULT_TIMEOUT: Arc<RwLock<Option<i32>>> = Arc::new(RwLock::new(None));
frontend/src/lib/components/instanceSettings.ts
+39 -28
View File
@@ -78,26 +78,7 @@ export const settings: Record<string, Setting[]> = {
placeholder: 'only needed to prepare upgrade to EE',
storage: 'setting'
},
{
label: 'Pip Extra Index Url',
description: 'Add private PIP registry',
key: 'pip_extra_index_url',
fieldType: 'text',
placeholder: 'https://username:password@pypi.company.com/simple',
storage: 'setting',
ee_only:
'You can still set this setting by using PIP_EXTRA_INDEX_URL as env variable to the worker containers'
},
{
label: 'Npm Config Registry',
description: 'Add private NPM registry',
key: 'npm_config_registry',
fieldType: 'text',
placeholder: 'https://yourregistry',
storage: 'setting',
ee_only:
'You can still set this setting by using NPM_CONFIG_REGISTRY as env variable to the worker containers'
},
{
label: 'Expose metrics',
description: 'Expose prometheus metrics for workers and servers on port 8001 at /metrics',
@@ -117,6 +98,44 @@ export const settings: Record<string, Setting[]> = {
'You can still set this setting by using OPENAI_AZURE_BASE_PATH as env variable to the server containers'
}
],
'SSO/OAuth': [
{
label: 'Require users to have been added manually to windmill to sign in through OAuth',
key: 'require_preexisting_user_for_oauth',
fieldType: 'boolean',
storage: 'setting'
}
],
Registries: [
{
label: 'Pip Extra Index Url',
description: 'Add private PIP registry',
key: 'pip_extra_index_url',
fieldType: 'text',
placeholder: 'https://username:password@pypi.company.com/simple',
storage: 'setting',
ee_only: ''
},
{
label: 'Npm Config Registry',
description: 'Add private NPM registry',
key: 'npm_config_registry',
fieldType: 'text',
placeholder: 'https://yourregistry',
storage: 'setting',
ee_only: ''
},
{
label: 'Bunfig Install Scopes',
description:
'Add private scoped registries for Bun, See: https://bun.sh/docs/install/registries',
key: 'bunfig_install_scopes',
fieldType: 'text',
placeholder: '"@myorg3" = { token = "mytoken", url = "https://registry.myorg.com/" }',
storage: 'setting',
ee_only: ''
}
],
SMTP: [
{
label: 'Host',
@@ -159,14 +178,6 @@ export const settings: Record<string, Setting[]> = {
storage: 'config'
}
],
'SSO/OAuth': [
{
label: 'Require users to have been added manually to windmill to sign in through OAuth',
key: 'require_preexisting_user_for_oauth',
fieldType: 'boolean',
storage: 'setting'
}
],
Debug: [
{
label: 'Keep Job Directories',