diff --git a/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json b/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json new file mode 100644 index 0000000000..10141b3a78 --- /dev/null +++ b/backend/.sqlx/query-0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT j.id, j.runnable_path, j.args, j.kind::text AS \"kind!\"\n FROM v2_job j\n JOIN v2_job_queue q ON j.id = q.id\n WHERE j.runnable_path = $1\n AND j.kind = 'deploymentcallback'\n ORDER BY j.created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + null + ] + }, + "hash": "0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f" +} diff --git a/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json b/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json index 90f38c05a7..da0ce60709 100644 --- a/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json +++ b/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + false ] }, "hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91" diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json index 0a06188897..c96961eac4 100644 --- a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + false ] }, "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 75a2b893b2..0e277038b7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9df755c57fbfc88f4a724e1ea51b1d5f5af4fe52 +09dfb247f6f59c61b7f2431932c4557fb26c22d8 diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs new file mode 100644 index 0000000000..c0f0fe0463 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -0,0 +1,467 @@ +/*! + * Integration tests for workspace dependencies git sync. + * + * These tests verify that creating, archiving, and deleting workspace dependencies + * triggers deployment callback jobs with the correct arguments for git sync. + * + * Run with enterprise features: + * ```bash + * cargo test --test workspace_dependencies_git_sync --features enterprise,private + * ``` + */ + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::time::Duration; + +use windmill_test_utils::*; + +/// Row shape for querying deployment callback jobs from v2_job_queue +#[derive(Debug)] +#[allow(dead_code)] +struct DeploymentCallbackJob { + id: uuid::Uuid, + runnable_path: Option, + args: Option, + kind: String, +} + +/// Poll for deployment callback jobs in the queue for a given script path +async fn get_deployment_callback_jobs( + db: &Pool, + script_path: &str, + timeout: Duration, +) -> anyhow::Result> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let rows = sqlx::query_as!( + DeploymentCallbackJob, + r#" + SELECT j.id, j.runnable_path, j.args, j.kind::text AS "kind!" + FROM v2_job j + JOIN v2_job_queue q ON j.id = q.id + WHERE j.runnable_path = $1 + AND j.kind = 'deploymentcallback' + ORDER BY j.created_at DESC + "#, + script_path, + ) + .fetch_all(db) + .await?; + + if !rows.is_empty() { + return Ok(rows); + } + + if tokio::time::Instant::now() >= deadline { + // Return empty if timeout - caller will handle assertion + return Ok(vec![]); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// Configure git sync for the test workspace with workspace dependencies enabled +async fn setup_git_sync_config(db: &Pool, sync_script_path: &str) -> anyhow::Result<()> { + let git_sync_config = json!({ + "include_type": ["workspacedependencies"], + "include_path": ["**"], + "repositories": [{ + "script_path": sync_script_path, + "git_repo_resource_path": "$res:u/test-user/test_git_repo", + "use_individual_branch": false, + "group_by_folder": false + }] + }); + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + git_sync_config, + "test-workspace" + ) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a git repository resource for testing +async fn create_git_repo_resource(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/test_git_repo', $1::jsonb, 'git_repository', '{}'::jsonb, 'test-user') + ON CONFLICT (workspace_id, path) DO NOTHING + "#, + ) + .bind(json!({ + "url": "https://github.com/test/test.git", + "branch": "main", + "token": "test-token" + })) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a dummy sync script for testing (with version >= 28103 for debouncing support) +async fn create_sync_script(db: &Pool, path: &str) -> anyhow::Result { + let hash: i64 = rand::random::().unsigned_abs() as i64; + sqlx::query( + r#" + INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, kind, lock) + VALUES ('test-workspace', $1, $2, 'sync script', '', + 'export function main(items: any[]) { return { synced: items.length }; }', + 'test-user', 'bun', 'script', '') + "#, + ) + .bind(hash) + .bind(path) + .execute(db) + .await?; + Ok(hash) +} + +/// Create a folder for the versioned script path +async fn create_folder(db: &Pool, name: &str) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) + VALUES ('test-workspace', $1, $1, ARRAY['u/test-user'], '{}'::jsonb, 'test-user') + ON CONFLICT (workspace_id, name) DO NOTHING + "#, + ) + .bind(name) + .execute(db) + .await?; + Ok(()) +} + +// ============================================================================ +// Tests +// ============================================================================ + +/// Test that creating a workspace dependency triggers a git sync deployment callback +/// with the correct path_type and path arguments. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_workspace_dependencies_triggers_git_sync( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup: Create folder, git repo resource, sync script, and configure git sync + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + // Start API server + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency via API + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "test-deps", + "content": "requests==2.28.0\nnumpy==1.24.0" + })) + .send() + .await?; + + assert!( + response.status().is_success(), + "Failed to create workspace dependency: {:?}", + response.text().await + ); + + // Wait for deployment callback job to be created + tokio::time::sleep(Duration::from_millis(500)).await; + + // Query for deployment callback jobs + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + + assert!( + !jobs.is_empty(), + "Expected at least one deployment callback job to be created" + ); + + // Verify the job arguments + let job = &jobs[0]; + let args = job.args.as_ref().expect("Job should have args"); + + // Check that path_type is "workspace_dependencies" (or check items array) + // The exact structure depends on whether debouncing is enabled + if let Some(items) = args.get("items") { + // Debounced format: items is an array + let items_arr = items.as_array().expect("items should be an array"); + assert!(!items_arr.is_empty(), "items array should not be empty"); + + let item = &items_arr[0]; + assert_eq!( + item.get("path_type").and_then(|v| v.as_str()), + Some("workspace_dependencies"), + "path_type should be 'workspace_dependencies'" + ); + + // Path should be "dependencies/test-deps.requirements.in" or similar + let path = item.get("path").and_then(|v| v.as_str()).unwrap_or(""); + assert!( + path.contains("dependencies") || path.contains("requirements"), + "path should contain dependencies or requirements: got {}", + path + ); + } else if let Some(path_type) = args.get("path_type") { + // Non-debounced format: path_type is a direct field + assert_eq!( + path_type.as_str(), + Some("workspace_dependencies"), + "path_type should be 'workspace_dependencies'" + ); + } else { + panic!( + "Job args should contain either 'items' array or 'path_type' field: {:?}", + args + ); + } + + Ok(()) +} + +/// Test that archiving a workspace dependency triggers a git sync deployment callback +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_archive_workspace_dependencies_triggers_git_sync( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_archive"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // First create a workspace dependency + let create_response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "archive-test-deps", + "content": "flask==2.0.0" + })) + .send() + .await?; + + assert!(create_response.status().is_success()); + + // Wait a bit for the create job to be processed + tokio::time::sleep(Duration::from_millis(300)).await; + + // Now archive it + let archive_response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/archive/python3?name=archive-test-deps", + client.baseurl() + )) + .send() + .await?; + + assert!( + archive_response.status().is_success(), + "Failed to archive workspace dependency: {:?}", + archive_response.text().await + ); + + // Wait for deployment callback job + tokio::time::sleep(Duration::from_millis(500)).await; + + // Should have at least 2 items total (create + archive) + // Note: With debouncing enabled (5s window), both operations may be combined into + // a single job with 2 items in the "items" array + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + + let total_items: usize = jobs + .iter() + .map(|job| { + job.args + .as_ref() + .and_then(|args| args.get("items")) + .and_then(|items| items.as_array()) + .map(|arr| arr.len()) + .unwrap_or(1) // Non-debounced jobs count as 1 item + }) + .sum(); + + assert!( + total_items >= 2, + "Expected at least 2 total items (create + archive), got {} items across {} jobs", + total_items, + jobs.len() + ); + + Ok(()) +} + +/// Test that workspace dependencies are NOT synced when workspacedependencies type is excluded +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_dependencies_respects_include_type_filter( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup with git sync that EXCLUDES workspacedependencies + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_filter"; + create_sync_script(&db, sync_script_path).await?; + + // Configure git sync to only include scripts (not workspace dependencies) + let git_sync_config = json!({ + "include_type": ["script"], // Note: workspacedependencies is NOT included + "include_path": ["**"], + "repositories": [{ + "script_path": sync_script_path, + "git_repo_resource_path": "$res:u/test-user/test_git_repo", + "use_individual_branch": false, + "group_by_folder": false + }] + }); + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + git_sync_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "python3", + "name": "filtered-deps", + "content": "django==4.0.0" + })) + .send() + .await?; + + assert!(response.status().is_success()); + + // Wait a bit + tokio::time::sleep(Duration::from_secs(1)).await; + + // Should NOT have any deployment callback jobs because workspacedependencies is filtered out + let jobs = + get_deployment_callback_jobs(&db, sync_script_path, Duration::from_millis(500)).await?; + + assert!( + jobs.is_empty(), + "Expected NO deployment callback jobs when workspacedependencies is not in include_type, got {}", + jobs.len() + ); + + Ok(()) +} + +/// Test that the commit message is correctly generated for workspace dependencies +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_workspace_dependencies_commit_message(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Setup + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_msg"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let (client, _port, _server) = init_client(db.clone()).await; + + // Create workspace dependency + let response = client + .client() + .post(format!( + "{}/w/test-workspace/workspace_dependencies/create", + client.baseurl() + )) + .json(&json!({ + "workspace_id": "test-workspace", + "language": "bun", + "name": null, // unnamed/default dependency + "content": "lodash: ^4.17.21" + })) + .send() + .await?; + + assert!(response.status().is_success()); + + tokio::time::sleep(Duration::from_millis(500)).await; + + let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?; + assert!(!jobs.is_empty()); + + let job = &jobs[0]; + let args = job.args.as_ref().expect("Job should have args"); + + // Check commit message format + if let Some(items) = args.get("items") { + let items_arr = items.as_array().expect("items should be an array"); + if !items_arr.is_empty() { + let commit_msg = items_arr[0] + .get("commit_msg") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + assert!( + commit_msg.contains("[WM]"), + "Commit message should contain '[WM]' prefix: {}", + commit_msg + ); + assert!( + commit_msg.to_lowercase().contains("workspace") + || commit_msg.to_lowercase().contains("dependency") + || commit_msg.to_lowercase().contains("deployed"), + "Commit message should mention workspace dependency or deployed: {}", + commit_msg + ); + } + } else if let Some(commit_msg) = args.get("commit_msg").and_then(|v| v.as_str()) { + assert!( + commit_msg.contains("[WM]"), + "Commit message should contain '[WM]' prefix: {}", + commit_msg + ); + } + + Ok(()) +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 14069590eb..443ec00256 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fe28092e87..854514dfd8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -18459,6 +18459,7 @@ components: - trigger - settings - key + - workspacedependencies AIProviderModel: type: object diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index d887923f5b..e5c9377b01 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -16,6 +16,7 @@ use windmill_common::{ use windmill_dep_map::workspace_dependencies::{ trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies, }; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use crate::db::ApiAuthed; @@ -37,21 +38,36 @@ async fn create( ) -> error::Result<(StatusCode, String)> { tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; - Ok(( - StatusCode::CREATED, - format!( - "{}", - nwd.create( - ( - authed.email, - username_to_permissioned_as(&authed.username), - authed.username, - ), - db - ) - .await? - ), - )) + + let dep_path = WorkspaceDependencies::to_path(&nwd.name, nwd.language)?; + let w_id = nwd.workspace_id.clone(); + let email = authed.email.clone(); + let username = authed.username.clone(); + + let id = nwd + .create( + ( + authed.email, + username_to_permissioned_as(&authed.username), + authed.username, + ), + db.clone(), + ) + .await?; + + handle_deployment_metadata( + &email, + &username, + &db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path }, + None, + true, + None, + ) + .await?; + + Ok((StatusCode::CREATED, format!("{}", id))) } #[axum::debug_handler] @@ -92,8 +108,21 @@ async fn archive( tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; let db = &db; + let dep_path = WorkspaceDependencies::to_path(¶ms.name, language)?; WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?; + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path.clone() }, + None, + true, + None, + ) + .await?; + trigger_dependents_to_recompute_dependencies_in_the_background( params.name.is_none(), w_id, @@ -103,7 +132,7 @@ async fn archive( username_to_permissioned_as(&authed.username), authed.username, ), - WorkspaceDependencies::to_path(¶ms.name, language)?, + dep_path, db.clone(), ) .await; @@ -121,8 +150,21 @@ async fn delete( tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies"); require_admin(authed.is_admin, &authed.username)?; let db = &db; + let dep_path = WorkspaceDependencies::to_path(¶ms.name, language)?; WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?; + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + &w_id, + DeployedObject::WorkspaceDependencies { path: dep_path.clone() }, + None, + true, + None, + ) + .await?; + trigger_dependents_to_recompute_dependencies_in_the_background( params.name.is_none(), w_id, @@ -132,7 +174,7 @@ async fn delete( username_to_permissioned_as(&authed.username), authed.username, ), - WorkspaceDependencies::to_path(¶ms.name, language)?, + dep_path, db.clone(), ) .await; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index f56648f61e..5aba7162e1 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -146,6 +146,7 @@ pub enum ObjectType { Trigger, Settings, Key, + WorkspaceDependencies, } #[derive(Serialize, Deserialize, Debug)] diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 645e477a62..f9cecd46ce 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -12,6 +12,10 @@ use windmill_common::{scripts::ScriptHash, DB}; pub mod git_sync_ee; pub mod git_sync_oss; +#[cfg(feature = "private")] +pub use git_sync_ee::{handle_deployment_metadata, handle_fork_branch_creation}; + +#[cfg(not(feature = "private"))] pub use git_sync_oss::{handle_deployment_metadata, handle_fork_branch_creation}; #[derive(Clone, Debug)] @@ -38,6 +42,7 @@ pub enum DeployedObject { EmailTrigger { path: String, parent_path: Option }, Settings { setting_type: String }, Key { key_type: String }, + WorkspaceDependencies { path: String }, } impl DeployedObject { @@ -65,6 +70,7 @@ impl DeployedObject { DeployedObject::EmailTrigger { path, .. } => path.to_owned(), DeployedObject::Settings { .. } => "settings.yaml".to_string(), DeployedObject::Key { .. } => "encryption_key.yaml".to_string(), + DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(), } } @@ -74,7 +80,8 @@ impl DeployedObject { | Self::Group { .. } | Self::ResourceType { .. } | Self::Settings { .. } - | Self::Key { .. } => true, + | Self::Key { .. } + | Self::WorkspaceDependencies { .. } => true, _ => false, } } @@ -103,6 +110,7 @@ impl DeployedObject { DeployedObject::EmailTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::Settings { .. } => None, DeployedObject::Key { .. } => None, + DeployedObject::WorkspaceDependencies { .. } => None, } } @@ -130,6 +138,244 @@ impl DeployedObject { DeployedObject::EmailTrigger { .. } => "email_trigger", DeployedObject::Settings { .. } => "settings", DeployedObject::Key { .. } => "key", - }.to_string() + DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies", + } + .to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::scripts::ScriptHash; + + // --- DeployedObject::get_path tests --- + + #[test] + fn test_get_path_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_path(), "f/folder/script"); + } + + #[test] + fn test_get_path_flow() { + let obj = DeployedObject::Flow { + path: "f/folder/flow".to_string(), + parent_path: Some("f/folder/old_flow".to_string()), + version: 1, + }; + assert_eq!(obj.get_path(), "f/folder/flow"); + } + + #[test] + fn test_get_path_user() { + let obj = DeployedObject::User { email: "user@example.com".to_string() }; + assert_eq!(obj.get_path(), "users/user@example.com"); + } + + #[test] + fn test_get_path_group() { + let obj = DeployedObject::Group { name: "admins".to_string() }; + assert_eq!(obj.get_path(), "groups/admins"); + } + + #[test] + fn test_get_path_settings() { + let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() }; + assert_eq!(obj.get_path(), "settings.yaml"); + } + + #[test] + fn test_get_path_key() { + let obj = DeployedObject::Key { key_type: "encryption".to_string() }; + assert_eq!(obj.get_path(), "encryption_key.yaml"); + } + + #[test] + fn test_get_path_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_path(), "workspace-dependencies/python"); + } + + // --- DeployedObject::get_ignore_regex_filter tests --- + + #[test] + fn test_ignore_regex_filter_user() { + let obj = DeployedObject::User { email: "user@example.com".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_group() { + let obj = DeployedObject::Group { name: "admins".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_resource_type() { + let obj = DeployedObject::ResourceType { path: "postgresql".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_settings() { + let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_key() { + let obj = DeployedObject::Key { key_type: "encryption".to_string() }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert!(obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert!(!obj.get_ignore_regex_filter()); + } + + #[test] + fn test_ignore_regex_filter_flow() { + let obj = DeployedObject::Flow { + path: "f/folder/flow".to_string(), + parent_path: None, + version: 1, + }; + assert!(!obj.get_ignore_regex_filter()); + } + + // --- DeployedObject::get_parent_path tests --- + + #[test] + fn test_get_parent_path_script_with_parent() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: Some("f/folder/old_script".to_string()), + }; + assert_eq!(obj.get_parent_path(), Some("f/folder/old_script".to_string())); + } + + #[test] + fn test_get_parent_path_script_without_parent() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "f/folder/script".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_parent_path(), None); + } + + #[test] + fn test_get_parent_path_folder() { + let obj = DeployedObject::Folder { path: "f/folder".to_string() }; + assert_eq!(obj.get_parent_path(), None); + } + + #[test] + fn test_get_parent_path_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_parent_path(), None); + } + + // --- DeployedObject::get_kind tests --- + + #[test] + fn test_get_kind_script() { + let obj = DeployedObject::Script { + hash: ScriptHash(123), + path: "test".to_string(), + parent_path: None, + }; + assert_eq!(obj.get_kind(), "script"); + } + + #[test] + fn test_get_kind_flow() { + let obj = DeployedObject::Flow { + path: "test".to_string(), + parent_path: None, + version: 1, + }; + assert_eq!(obj.get_kind(), "flow"); + } + + #[test] + fn test_get_kind_app() { + let obj = DeployedObject::App { + path: "test".to_string(), + version: 1, + parent_path: None, + }; + assert_eq!(obj.get_kind(), "app"); + } + + #[test] + fn test_get_kind_workspace_dependencies() { + let obj = DeployedObject::WorkspaceDependencies { + path: "workspace-dependencies/python".to_string(), + }; + assert_eq!(obj.get_kind(), "workspace_dependencies"); + } + + #[test] + fn test_get_kind_all_triggers() { + assert_eq!( + DeployedObject::HttpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "http_trigger" + ); + assert_eq!( + DeployedObject::WebsocketTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "websocket_trigger" + ); + assert_eq!( + DeployedObject::KafkaTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "kafka_trigger" + ); + assert_eq!( + DeployedObject::NatsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "nats_trigger" + ); + assert_eq!( + DeployedObject::PostgresTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "postgres_trigger" + ); + assert_eq!( + DeployedObject::MqttTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "mqtt_trigger" + ); + assert_eq!( + DeployedObject::SqsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "sqs_trigger" + ); + assert_eq!( + DeployedObject::GcpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "gcp_trigger" + ); + assert_eq!( + DeployedObject::EmailTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "email_trigger" + ); } } diff --git a/backend/windmill-test-utils/Cargo.toml b/backend/windmill-test-utils/Cargo.toml index e805ee5a3d..d1729a7511 100644 --- a/backend/windmill-test-utils/Cargo.toml +++ b/backend/windmill-test-utils/Cargo.toml @@ -10,8 +10,8 @@ path = "src/lib.rs" [features] default = [] -private = [] -enterprise = [] +private = ["windmill-api/private"] +enterprise = ["windmill-api/enterprise"] python = ["windmill-common/python"] deno_core = ["dep:windmill-runtime-nativets"] agent_worker_server = ["dep:windmill-api-agent-workers"] diff --git a/cli/dev.nu b/cli/dev.nu new file mode 100644 index 0000000000..91a0faa8e3 --- /dev/null +++ b/cli/dev.nu @@ -0,0 +1,59 @@ +#!/usr/bin/env nu + +let cli_cache = "/tmp/windmill/cache_nomount/bun/" +let bundle_cache = "/tmp/windmill/cache/bun/" + +# Clean CLI package cache +def "main clean" [] { + rm -rf ($cli_cache ++ "windmill-cli@*") + rm -rf ($cli_cache ++ "windmill-cli/") + print "Cleaned CLI cache" +} + +# Clear bundle cache (forces hub scripts to re-bundle) +def "main clear-bundles" [] { + rm -rf ($bundle_cache ++ "*") + print "Cleared bundle cache" +} + +# Patch CLI cache with local build +def "main patch" [] { + print "Patching CLI cache..." + + let versions = (ls $cli_cache | where name =~ "windmill-cli@" | get name) + + if ($versions | is-empty) { + print "No CLI versions found in cache" + return + } + + for path in $versions { + rm -rf ($path ++ "/esm") + ^cp -r npm/esm ($path ++ "/esm") + ^cp npm/package.json ($path ++ "/package.json") + print $"Patched ($path | path basename)" + } + + print "Done!" +} + +# Build CLI, patch cache, and clear bundles +def main [ + --patch(-p) # Only patch existing cache (skip build) + --clean(-c) # Clean CLI cache first +] { + if $clean { + main clean + } + + if $patch { + main patch + } else { + print "Building CLI..." + bun run build + main patch + } + + # Always clear bundle cache so hub scripts use patched CLI + main clear-bundles +} diff --git a/cli/src/commands/gitsync-settings/converter.ts b/cli/src/commands/gitsync-settings/converter.ts index 89600fd5d1..6861603bb4 100644 --- a/cli/src/commands/gitsync-settings/converter.ts +++ b/cli/src/commands/gitsync-settings/converter.ts @@ -33,6 +33,7 @@ export class GitSyncSettingsConverter { includeGroups: includeTypes.includes("group"), includeSettings: includeTypes.includes("settings"), includeKey: includeTypes.includes("key"), + skipWorkspaceDependencies: !includeTypes.includes("workspacedependencies"), }; // Only include extraIncludes if it has content @@ -61,6 +62,7 @@ export class GitSyncSettingsConverter { if (opts.includeGroups) includeTypes.push("group"); if (opts.includeSettings) includeTypes.push("settings"); if (opts.includeKey) includeTypes.push("key"); + if (!opts.skipWorkspaceDependencies) includeTypes.push("workspacedependencies"); const result: BackendGitSyncSettings = { include_path: opts.includes || [], @@ -99,6 +101,7 @@ export class GitSyncSettingsConverter { includeGroups: opts.includeGroups ?? false, includeSettings: opts.includeSettings ?? false, includeKey: opts.includeKey ?? false, + skipWorkspaceDependencies: opts.skipWorkspaceDependencies ?? false, }; } @@ -122,6 +125,7 @@ export class GitSyncSettingsConverter { includeGroups: opts.includeGroups, includeSettings: opts.includeSettings, includeKey: opts.includeKey, + skipWorkspaceDependencies: opts.skipWorkspaceDependencies, }; } diff --git a/cli/src/commands/gitsync-settings/types.ts b/cli/src/commands/gitsync-settings/types.ts index 36b51aedbc..7bebb1f1a1 100644 --- a/cli/src/commands/gitsync-settings/types.ts +++ b/cli/src/commands/gitsync-settings/types.ts @@ -37,6 +37,7 @@ export const GIT_SYNC_FIELDS = [ "includeGroups", "includeSettings", "includeKey", + "skipWorkspaceDependencies", ] as const; export type GitSyncField = typeof GIT_SYNC_FIELDS[number]; @@ -57,6 +58,7 @@ export const INCLUDE_TYPE_MAPPINGS = { group: "includeGroups", settings: "includeSettings", key: "includeKey", + workspacedependencies: "skipWorkspaceDependencies", } as const; // Write mode for branch-based configuration diff --git a/cli/test/gitsync_converter_unit.test.ts b/cli/test/gitsync_converter_unit.test.ts new file mode 100644 index 0000000000..e84c2ffa11 --- /dev/null +++ b/cli/test/gitsync_converter_unit.test.ts @@ -0,0 +1,227 @@ +/** + * Unit tests for GitSyncSettingsConverter. + * Tests conversion between backend format (include_type array) and + * SyncOptions format (boolean flags like skipWorkspaceDependencies). + */ + +import { expect, test, describe } from "bun:test"; +import { GitSyncSettingsConverter } from "../src/commands/gitsync-settings/converter.ts"; + +// ============================================================================= +// fromBackendFormat - converts backend include_type array to SyncOptions +// ============================================================================= + +describe("GitSyncSettingsConverter.fromBackendFormat", () => { + test("converts workspacedependencies in include_type to skipWorkspaceDependencies: false", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "workspacedependencies"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(false); + }); + + test("sets skipWorkspaceDependencies: true when workspacedependencies is absent", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(true); + }); + + test("handles empty include_type array", () => { + const backend = { + include_path: [], + include_type: [], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipWorkspaceDependencies).toBe(true); + expect(result.skipScripts).toBe(true); + expect(result.skipFlows).toBe(true); + }); + + test("converts all standard types correctly", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "app", "folder", "variable", "resource", "resourcetype", "secret", "schedule", "trigger", "user", "group", "settings", "key", "workspacedependencies"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + + expect(result.skipScripts).toBe(false); + expect(result.skipFlows).toBe(false); + expect(result.skipApps).toBe(false); + expect(result.skipFolders).toBe(false); + expect(result.skipVariables).toBe(false); + expect(result.skipResources).toBe(false); + expect(result.skipResourceTypes).toBe(false); + expect(result.skipSecrets).toBe(false); + expect(result.includeSchedules).toBe(true); + expect(result.includeTriggers).toBe(true); + expect(result.includeUsers).toBe(true); + expect(result.includeGroups).toBe(true); + expect(result.includeSettings).toBe(true); + expect(result.includeKey).toBe(true); + expect(result.skipWorkspaceDependencies).toBe(false); + }); +}); + +// ============================================================================= +// toBackendFormat - converts SyncOptions to backend include_type array +// ============================================================================= + +describe("GitSyncSettingsConverter.toBackendFormat", () => { + test("adds workspacedependencies when skipWorkspaceDependencies is false", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: false, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + expect(result.include_type).toContain("workspacedependencies"); + }); + + test("does not add workspacedependencies when skipWorkspaceDependencies is true", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: true, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + expect(result.include_type).not.toContain("workspacedependencies"); + }); + + test("adds workspacedependencies when skipWorkspaceDependencies is undefined (defaults to false)", () => { + const opts = { + includes: ["f/**"], + // skipWorkspaceDependencies not set + }; + const normalized = GitSyncSettingsConverter.normalize(opts); + const result = GitSyncSettingsConverter.toBackendFormat(normalized); + expect(result.include_type).toContain("workspacedependencies"); + }); + + test("converts all boolean flags to include_type correctly", () => { + const opts = { + includes: ["f/**"], + skipScripts: false, + skipFlows: false, + skipApps: false, + skipFolders: false, + skipVariables: false, + skipResources: false, + skipResourceTypes: false, + skipSecrets: false, + includeSchedules: true, + includeTriggers: true, + includeUsers: true, + includeGroups: true, + includeSettings: true, + includeKey: true, + skipWorkspaceDependencies: false, + }; + const result = GitSyncSettingsConverter.toBackendFormat(opts); + + expect(result.include_type).toContain("script"); + expect(result.include_type).toContain("flow"); + expect(result.include_type).toContain("app"); + expect(result.include_type).toContain("folder"); + expect(result.include_type).toContain("variable"); + expect(result.include_type).toContain("resource"); + expect(result.include_type).toContain("resourcetype"); + expect(result.include_type).toContain("secret"); + expect(result.include_type).toContain("schedule"); + expect(result.include_type).toContain("trigger"); + expect(result.include_type).toContain("user"); + expect(result.include_type).toContain("group"); + expect(result.include_type).toContain("settings"); + expect(result.include_type).toContain("key"); + expect(result.include_type).toContain("workspacedependencies"); + }); +}); + +// ============================================================================= +// normalize - applies defaults for undefined fields +// ============================================================================= + +describe("GitSyncSettingsConverter.normalize", () => { + test("defaults skipWorkspaceDependencies to false", () => { + const opts = { includes: ["f/**"] }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(false); + }); + + test("preserves explicit skipWorkspaceDependencies: true", () => { + const opts = { includes: ["f/**"], skipWorkspaceDependencies: true }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(true); + }); + + test("preserves explicit skipWorkspaceDependencies: false", () => { + const opts = { includes: ["f/**"], skipWorkspaceDependencies: false }; + const result = GitSyncSettingsConverter.normalize(opts); + expect(result.skipWorkspaceDependencies).toBe(false); + }); +}); + +// ============================================================================= +// Round-trip conversion tests +// ============================================================================= + +describe("GitSyncSettingsConverter round-trip", () => { + test("backend -> SyncOptions -> backend preserves workspacedependencies", () => { + const original = { + include_path: ["f/**"], + include_type: ["script", "flow", "workspacedependencies"], + }; + + const syncOpts = GitSyncSettingsConverter.fromBackendFormat(original); + const backAgain = GitSyncSettingsConverter.toBackendFormat(syncOpts); + + expect(backAgain.include_type).toContain("workspacedependencies"); + expect(backAgain.include_type).toContain("script"); + expect(backAgain.include_type).toContain("flow"); + }); + + test("backend without workspacedependencies -> SyncOptions -> backend still excludes it", () => { + const original = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + + const syncOpts = GitSyncSettingsConverter.fromBackendFormat(original); + const backAgain = GitSyncSettingsConverter.toBackendFormat(syncOpts); + + expect(backAgain.include_type).not.toContain("workspacedependencies"); + expect(backAgain.include_type).toContain("script"); + expect(backAgain.include_type).toContain("flow"); + }); + + test("SyncOptions with defaults -> backend includes workspacedependencies", () => { + const opts = { + includes: ["f/**"], + skipScripts: false, + skipFlows: false, + // skipWorkspaceDependencies not set - should default to false + }; + + const normalized = GitSyncSettingsConverter.normalize(opts); + const backend = GitSyncSettingsConverter.toBackendFormat(normalized); + + expect(backend.include_type).toContain("workspacedependencies"); + }); +}); + +// ============================================================================= +// extractGitSyncFields +// ============================================================================= + +describe("GitSyncSettingsConverter.extractGitSyncFields", () => { + test("includes skipWorkspaceDependencies in extracted fields", () => { + const opts = { + includes: ["f/**"], + skipWorkspaceDependencies: true, + someOtherField: "ignored", + }; + const result = GitSyncSettingsConverter.extractGitSyncFields(opts); + expect(result.skipWorkspaceDependencies).toBe(true); + }); +}); diff --git a/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte b/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte index c9f208a62b..e5da7bb417 100644 --- a/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte @@ -24,6 +24,7 @@ triggers: boolean settings: boolean key: boolean + workspaceDependencies: boolean } let { @@ -68,7 +69,8 @@ groups: effectiveIncludeTypes.includes('group'), triggers: effectiveIncludeTypes.includes('trigger'), settings: effectiveIncludeTypes.includes('settings'), - key: effectiveIncludeTypes.includes('key') + key: effectiveIncludeTypes.includes('key'), + workspaceDependencies: effectiveIncludeTypes.includes('workspacedependencies') }) // Tab selection for filter kinds @@ -90,7 +92,8 @@ groups: 'group', triggers: 'trigger', settings: 'settings', - key: 'key' + key: 'key', + workspaceDependencies: 'workspacedependencies' } if (value) { @@ -304,6 +307,14 @@ options={{ right: 'Encryption key' }} /> +
+ updateIncludeType('workspaceDependencies', e.detail)} + options={{ right: 'Workspace dependencies' }} + /> +