From b2c7d7791b1e1bd5a3ccd0e85f3ef940072487a4 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 29 Sep 2025 15:24:41 +0200 Subject: [PATCH] feat: powershell private repo support (#6684) * feat: powershell private repo support * nits --- backend/src/main.rs | 16 ++- backend/src/monitor.rs | 28 ++++- .../windmill-common/src/global_settings.rs | 2 + backend/windmill-worker/src/bash_executor.rs | 105 +++++++++++++++--- backend/windmill-worker/src/worker.rs | 8 ++ .../src/lib/components/instanceSettings.ts | 18 +++ 6 files changed, 155 insertions(+), 22 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 4738a11639..8e508db009 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -10,9 +10,10 @@ use monitor::{ load_base_url, load_otel, reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, - reload_no_default_maven_setting, reload_nuget_config_setting, reload_ruby_repos_setting, - reload_timeout_wait_result_setting, send_current_log_file_to_object_store, - send_logs_to_object_store, WORKERS_NAMES, + reload_no_default_maven_setting, reload_nuget_config_setting, + reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, + reload_ruby_repos_setting, reload_timeout_wait_result_setting, + send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::postgres::PgListener; @@ -45,7 +46,8 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, @@ -1054,6 +1056,12 @@ Windmill Community Edition {GIT_VERSION} NUGET_CONFIG_SETTING => { reload_nuget_config_setting(&conn).await }, + POWERSHELL_REPO_URL_SETTING => { + reload_powershell_repo_url_setting(&conn).await + }, + POWERSHELL_REPO_PAT_SETTING => { + reload_powershell_repo_pat_setting(&conn).await + }, MAVEN_REPOS_SETTING => { reload_maven_repos_setting(&conn).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index bc5667c130..e012887d70 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -53,7 +53,8 @@ use windmill_common::{ HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -83,7 +84,8 @@ use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; use windmill_worker::{ handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, - NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, + NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, + POWERSHELL_REPO_URL, }; #[cfg(feature = "parquet")] @@ -298,6 +300,8 @@ pub async fn initial_load( reload_bunfig_install_scopes_setting(&conn).await; reload_instance_python_version_setting(&conn).await; reload_nuget_config_setting(&conn).await; + reload_powershell_repo_url_setting(&conn).await; + reload_powershell_repo_pat_setting(&conn).await; reload_maven_repos_setting(&conn).await; reload_no_default_maven_setting(&conn).await; reload_ruby_repos_setting(&conn).await; @@ -1125,6 +1129,26 @@ pub async fn reload_nuget_config_setting(conn: &Connection) { .await; } +pub async fn reload_powershell_repo_url_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + POWERSHELL_REPO_URL_SETTING, + "POWERSHELL_REPO_URL", + POWERSHELL_REPO_URL.clone(), + ) + .await; +} + +pub async fn reload_powershell_repo_pat_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + POWERSHELL_REPO_PAT_SETTING, + "POWERSHELL_REPO_PAT", + POWERSHELL_REPO_PAT.clone(), + ) + .await; +} + pub async fn reload_maven_repos_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index c26bdc2f7e..443ee5dbb9 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -11,6 +11,8 @@ pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; pub const NUGET_CONFIG_SETTING: &str = "nuget_config"; +pub const POWERSHELL_REPO_URL_SETTING: &str = "powershell_repo_url"; +pub const POWERSHELL_REPO_PAT_SETTING: &str = "powershell_repo_pat"; pub const MAVEN_REPOS_SETTING: &str = "maven_repos"; pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven"; pub const RUBY_REPOS_SETTING: &str = "ruby_repos"; diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index e55ee7eeaa..0e86a65cc2 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -33,7 +33,7 @@ const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str = include_str!("../nsjail/run.powershell.config.proto"); lazy_static::lazy_static! { - static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap(); + static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^\s"]+)"?"#).unwrap(); } #[cfg(feature = "dind")] @@ -46,7 +46,7 @@ use crate::{ }, handle_child::handle_child, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, - POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, + POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -520,21 +520,77 @@ fn raw_to_string(x: &str) -> String { } } -const POWERSHELL_INSTALL_CODE: &str = r#" +fn generate_powershell_install_code_with_private_repo() -> String { + r#" +$ErrorActionPreference = 'Stop' +$availableModules = Get-Module -ListAvailable +$path = '{path}' +$jobId = '{job_id}' +$privateRepoUrl = '{private_repo_url}' +$privateRepoPat = '{private_repo_pat}' + +# Register temporary private repository with job ID +$repoName = "windmill-private-$jobId" +$repoUri = "$privateRepoUrl" + +# Create PSCredential for authentication +$username = "token" +$patToken = ConvertTo-SecureString $privateRepoPat -AsPlainText -Force +$credentials = New-Object System.Management.Automation.PSCredential($username, $patToken) + +Write-Host "Registering temporary repository: $repoName" + +# Remove repository if it already exists +Unregister-PSResourceRepository -Name $repoName -ErrorAction SilentlyContinue +Register-PSResourceRepository -Name $repoName -Uri $repoUri -Trusted + +try { + $moduleNames = @({modules}) + foreach ($module in $moduleNames) { + if (-not ($availableModules | Where-Object { $_.Name -eq $module })) { + $moduleFound = $false + # First try private repository if configured + $privateModule = Find-PSResource -Name $module -Repository $repoName -ErrorAction SilentlyContinue -Credential $credentials + if ($privateModule) { + $moduleFound = $true + Write-Host "Found module $module in private repository, installing from there..." + Save-PSResource -Name $module -Path $path -Repository $repoName -Credential $credentials + } + + # If not found in private repo, try all repositories + if (-not $moduleFound) { + Write-Host "Installing module $module from all repositories..." + Save-PSResource -Name $module -Path $path -TrustRepository + } + } else { + Write-Host "Module $module already installed" + } + } +} finally { + Write-Host "Unregistering temporary repository: $repoName" + Unregister-PSResourceRepository -Name $repoName +} +"#.to_string() +} + +fn generate_powershell_install_code_public_only() -> String { + r#" +$ErrorActionPreference = 'Stop' $availableModules = Get-Module -ListAvailable $path = '{path}' $moduleNames = @({modules}) - foreach ($module in $moduleNames) { if (-not ($availableModules | Where-Object { $_.Name -eq $module })) { Write-Host "Installing module $module..." - Save-Module -Name $module -Path $path -Force + Save-PSResource -Name $module -Path $path -TrustRepository } else { Write-Host "Module $module already installed" } } -"#; +"# + .to_string() +} #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_powershell_job( @@ -617,16 +673,33 @@ pub async fn handle_powershell_job( } if !modules_to_install.is_empty() { - let install_string = POWERSHELL_INSTALL_CODE - .replace("{path}", POWERSHELL_CACHE_DIR) - .replace( - "{modules}", - &modules_to_install - .iter() - .map(|x| format!("'{x}'")) - .collect::>() - .join(", "), - ); + let powershell_repo_url = POWERSHELL_REPO_URL.read().await.clone(); + let powershell_repo_pat = POWERSHELL_REPO_PAT.read().await.clone(); + let has_private_repo = powershell_repo_url.is_some() && powershell_repo_pat.is_some(); + let install_template = if has_private_repo { + generate_powershell_install_code_with_private_repo() + } else { + generate_powershell_install_code_public_only() + }; + + let modules_list = modules_to_install + .iter() + .map(|x| format!("'{x}'")) + .collect::>() + .join(", "); + + let install_string = if has_private_repo { + install_template + .replace("{path}", POWERSHELL_CACHE_DIR) + .replace("{job_id}", &job.id.to_string()) + .replace("{private_repo_url}", &powershell_repo_url.unwrap()) + .replace("{private_repo_pat}", &powershell_repo_pat.unwrap()) + .replace("{modules}", &modules_list) + } else { + install_template + .replace("{path}", POWERSHELL_CACHE_DIR) + .replace("{modules}", &modules_list) + }; let mut cmd = Command::new(POWERSHELL_PATH.as_str()); cmd.args(&["-Command", &install_string]) .stdout(Stdio::piped()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 6c9f278d59..cbac637464 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -262,6 +262,12 @@ const DOTNET_DEFAULT_PATH: &str = "/usr/bin/dotnet"; pub const SAME_WORKER_REQUIREMENTS: &'static str = "SameWorkerSender is required because this job may be part of a flow"; +#[derive(Deserialize, Clone)] +pub struct PowershellRepo { + pub url: String, + pub pat: String, +} + lazy_static::lazy_static! { pub static ref SLEEP_QUEUE: u64 = std::env::var("SLEEP_QUEUE") @@ -343,6 +349,8 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); pub static ref NUGET_CONFIG: Arc>> = Arc::new(RwLock::new(None)); + pub static ref POWERSHELL_REPO_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref POWERSHELL_REPO_PAT: Arc>> = Arc::new(RwLock::new(None)); pub static ref MAVEN_REPOS: Arc>> = Arc::new(RwLock::new(None)); pub static ref NO_DEFAULT_MAVEN: AtomicBool = AtomicBool::new(std::env::var("NO_DEFAULT_MAVEN") .ok() diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 9bf4b6bad6..2176b50c34 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -340,6 +340,24 @@ export const settings: Record = { placeholder: 'https://user:password@gems.foo.com/', storage: 'setting', ee_only: '' + }, + { + label: 'PowerShell Repository URL', + description: 'Add private PowerShell repository URL', + key: 'powershell_repo_url', + placeholder: + 'https://pkgs.dev.azure.com///_packaging//nuget/v3/index.json', + fieldType: 'text', + storage: 'setting', + ee_only: '' + }, + { + label: 'PowerShell Repository PAT', + description: 'Add private PowerShell repository Personal Access Token', + key: 'powershell_repo_pat', + fieldType: 'text', + storage: 'setting', + ee_only: '' } ], Alerts: [