diff --git a/.claude/skills/native-trigger/SKILL.md b/.claude/skills/native-trigger/SKILL.md index 5952179d6a..026c1900bf 100644 --- a/.claude/skills/native-trigger/SKILL.md +++ b/.claude/skills/native-trigger/SKILL.md @@ -581,7 +581,28 @@ In `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte`: 2. Add to `baseConfig` with `countKey` (the dynamic `availableNativeServices` loop does NOT set `countKey`) 3. Add to the `allTypes` array -### Step 14: Update OpenAPI Spec and Regenerate Types +### Step 14: Update TriggersWrapper.svelte + +In `frontend/src/lib/components/triggers/TriggersWrapper.svelte`: + +Add a `{:else if selectedTrigger.type === 'yourservice'}` case that renders `` with the same props pattern as the existing native trigger cases (e.g., `nextcloud`). + +### Step 15: Update AddTriggersButton.svelte + +In `frontend/src/lib/components/triggers/AddTriggersButton.svelte`: + +1. Add `yourserviceAvailable` state variable +2. Add `setYourserviceState()` async function using `isServiceAvailable('yourservice', $workspaceStore!)` +3. Call it at module level +4. Add a dropdown entry to `addTriggerItems` with `hidden: !yourserviceAvailable` + +### Step 16: Update TriggersEditor.svelte Delete Handling + +In `frontend/src/lib/components/triggers/TriggersEditor.svelte`: + +Add your service to the `nativeTriggerServices` map in `deleteDeployedTrigger()`. Native triggers use `NativeTriggerService.deleteNativeTrigger({ workspace, serviceName, externalId })` instead of the standard `path`-based delete. + +### Step 17: Update OpenAPI Spec and Regenerate Types Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then: diff --git a/CHANGELOG.md b/CHANGELOG.md index 55dc62778b..c8ff641c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,43 @@ # Changelog +## [1.638.2](https://github.com/windmill-labs/windmill/compare/v1.638.1...v1.638.2) (2026-02-17) + + +### Bug Fixes + +* **backend:** gcp private key parsing ([#7979](https://github.com/windmill-labs/windmill/issues/7979)) ([5b7bb2f](https://github.com/windmill-labs/windmill/commit/5b7bb2fb84a12433c48f1cdfc022edff0cbc88ea)) +* yaml settings UI mask rsa_keys and jwt_secret ([71608bf](https://github.com/windmill-labs/windmill/commit/71608bf669658241b4ce4e1da3a83f1045dea1f6)) + +## [1.638.1](https://github.com/windmill-labs/windmill/compare/v1.638.0...v1.638.1) (2026-02-17) + + +### Bug Fixes + +* **operator:** improve configmap handling of older license keys ([b7bec1a](https://github.com/windmill-labs/windmill/commit/b7bec1a83d97a823ff6fc7d7fa549b975f848066)) + +## [1.638.0](https://github.com/windmill-labs/windmill/compare/v1.637.0...v1.638.0) (2026-02-17) + + +### Features + +* add native_mode as typed field on WorkerGroupConfig ([3e313cc](https://github.com/windmill-labs/windmill/commit/3e313cc4e864108d7dee866e784dff428883cadf)) +* show all settings in YAML UI and protect from empty overwrites ([#7976](https://github.com/windmill-labs/windmill/issues/7976)) ([b3eeee4](https://github.com/windmill-labs/windmill/commit/b3eeee413114cb54b5932542b14d8904a3c6c93c)) + + +### Bug Fixes + +* add missing google native triggers to triggers panel ([#7966](https://github.com/windmill-labs/windmill/issues/7966)) ([bb03c62](https://github.com/windmill-labs/windmill/commit/bb03c62c2819d40acd676d10cc586958f4117b5d)) +* download audit logs ([#7965](https://github.com/windmill-labs/windmill/issues/7965)) ([bba319b](https://github.com/windmill-labs/windmill/commit/bba319b2826f4d264ecebef3258d3c3f16237cc5)) +* improve operator ConfigMap settings handling ([#7975](https://github.com/windmill-labs/windmill/issues/7975)) ([2019aec](https://github.com/windmill-labs/windmill/commit/2019aecf4253edcf7b33e30862f642b303948440)) + +## [1.637.0](https://github.com/windmill-labs/windmill/compare/v1.636.0...v1.637.0) (2026-02-17) + + +### Features + +* **frontend:** inline edit summary & path from header ([#7968](https://github.com/windmill-labs/windmill/issues/7968)) ([eb5a8da](https://github.com/windmill-labs/windmill/commit/eb5a8dab74822eb3e43557cf1c85bf14d6e1910f)) +* native mode ([#7939](https://github.com/windmill-labs/windmill/issues/7939)) ([535e108](https://github.com/windmill-labs/windmill/commit/535e108cbf5070a6a23183389007db63fb07a58f)) + ## [1.636.0](https://github.com/windmill-labs/windmill/compare/v1.635.1...v1.636.0) (2026-02-16) diff --git a/README.md b/README.md index 1c8010db84..f51d184943 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,7 @@ On self-hosted instances, you might want to import all the approved resource typ | BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server | | ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | | RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | +| NATIVE_MODE | false | Enable native mode: sets NUM_WORKERS=8, rejects non-native jobs (nativets, postgresql, mysql, etc.) | Worker | | SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | | KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | | LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | diff --git a/backend/.sqlx/query-97c61b6a9a5112ea484565236959a544511d5d501fb737da8110a8725b883465.json b/backend/.sqlx/query-298fa4f8eb05b4c3f33b608b0cdb6ed918af2df012de33acb3befd3fcccbc257.json similarity index 50% rename from backend/.sqlx/query-97c61b6a9a5112ea484565236959a544511d5d501fb737da8110a8725b883465.json rename to backend/.sqlx/query-298fa4f8eb05b4c3f33b608b0cdb6ed918af2df012de33acb3befd3fcccbc257.json index ef9b7ffc2f..e0e7a16624 100644 --- a/backend/.sqlx/query-97c61b6a9a5112ea484565236959a544511d5d501fb737da8110a8725b883465.json +++ b/backend/.sqlx/query-298fa4f8eb05b4c3f33b608b0cdb6ed918af2df012de33acb3befd3fcccbc257.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode", "describe": { "columns": [], "parameters": { @@ -15,10 +15,11 @@ "Varchar", "Int8", "Int8", - "Text" + "Text", + "Bool" ] }, "nullable": [] }, - "hash": "97c61b6a9a5112ea484565236959a544511d5d501fb737da8110a8725b883465" + "hash": "298fa4f8eb05b4c3f33b608b0cdb6ed918af2df012de33acb3befd3fcccbc257" } diff --git a/backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json b/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json similarity index 86% rename from backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json rename to backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json index 644cb25123..583d2bcc6b 100644 --- a/backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json +++ b/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -102,6 +102,11 @@ "ordinal": 19, "name": "job_isolation", "type_info": "Text" + }, + { + "ordinal": 20, + "name": "native_mode", + "type_info": "Bool" } ], "parameters": { @@ -132,8 +137,9 @@ true, true, true, - true + true, + false ] }, - "hash": "771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30" + "hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca" } diff --git a/backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json b/backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json similarity index 73% rename from backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json rename to backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json index 520fb2c559..0141c379b4 100644 --- a/backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json +++ b/backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6", "describe": { "columns": [], "parameters": { @@ -15,10 +15,11 @@ "Int8", "Float4", "Float4", - "Float4" + "Float4", + "Bool" ] }, "nullable": [] }, - "hash": "aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd" + "hash": "a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159" } diff --git a/backend/.sqlx/query-2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac.json b/backend/.sqlx/query-dcc6928bc273fcbe52bcef43f9d06d8bb8c68a1b04b3c2cce7491dde5d727446.json similarity index 82% rename from backend/.sqlx/query-2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac.json rename to backend/.sqlx/query-dcc6928bc273fcbe52bcef43f9d06d8bb8c68a1b04b3c2cce7491dde5d727446.json index 3007e7435b..7ddaf715cb 100644 --- a/backend/.sqlx/query-2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac.json +++ b/backend/.sqlx/query-dcc6928bc273fcbe52bcef43f9d06d8bb8c68a1b04b3c2cce7491dde5d727446.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, worker_group, vcpus, memory, ping_at, started_at, custom_tags, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", + "query": "SELECT worker, worker_instance, worker_group, vcpus, memory, ping_at, started_at, custom_tags, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, native_mode FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", "describe": { "columns": [ { @@ -57,6 +57,11 @@ "ordinal": 10, "name": "occupancy_rate_30m", "type_info": "Float4" + }, + { + "ordinal": 11, + "name": "native_mode", + "type_info": "Bool" } ], "parameters": { @@ -73,8 +78,9 @@ true, true, true, - true + true, + false ] }, - "hash": "2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac" + "hash": "dcc6928bc273fcbe52bcef43f9d06d8bb8c68a1b04b3c2cce7491dde5d727446" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aa14a38c33..0f37cd171d 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2323,9 +2323,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.58" +version = "4.5.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" +checksum = "c5caf74d17c3aec5495110c34cc3f78644bfa89af6c8993ed4de2790e49b6499" dependencies = [ "clap_builder", "clap_derive", @@ -2333,9 +2333,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.58" +version = "4.5.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" +checksum = "370daa45065b80218950227371916a1633217ae42b2715b2287b606dcd618e24" dependencies = [ "anstream", "anstyle", @@ -14451,9 +14451,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.8+spec-1.1.0" +version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0742ff5ff03ea7e67c8ae6c93cac239e0d9784833362da3f9a9c1da8dfefcbdc" +checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ "winnow 0.7.14", ] @@ -15725,7 +15725,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-nats", @@ -15788,7 +15788,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -15801,7 +15801,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "argon2", @@ -15938,7 +15938,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -15961,7 +15961,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -15974,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -16000,7 +16000,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.636.0" +version = "1.638.2" dependencies = [ "reqwest 0.12.28", "serde", @@ -16010,7 +16010,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16027,7 +16027,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16050,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -16073,7 +16073,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16089,7 +16089,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16109,7 +16109,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16129,7 +16129,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16143,7 +16143,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-nats", @@ -16169,7 +16169,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -16194,7 +16194,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "flate2", @@ -16210,7 +16210,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -16231,7 +16231,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16251,7 +16251,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16280,7 +16280,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -16294,6 +16294,7 @@ dependencies = [ "rsa", "serde", "serde_json", + "serde_yml", "sha2 0.10.9", "sqlx", "tokio", @@ -16306,7 +16307,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.636.0" +version = "1.638.2" dependencies = [ "lazy_static", "serde", @@ -16318,7 +16319,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.636.0" +version = "1.638.2" dependencies = [ "argon2", "axum 0.7.9", @@ -16341,7 +16342,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16355,7 +16356,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.636.0" +version = "1.638.2" dependencies = [ "axum 0.7.9", "chrono", @@ -16384,7 +16385,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.636.0" +version = "1.638.2" dependencies = [ "chrono", "lazy_static", @@ -16398,7 +16399,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -16417,7 +16418,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.636.0" +version = "1.638.2" dependencies = [ "aes-gcm", "anyhow", @@ -16517,7 +16518,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.636.0" +version = "1.638.2" dependencies = [ "chrono", "itertools 0.14.0", @@ -16536,7 +16537,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.636.0" +version = "1.638.2" dependencies = [ "regex", "serde", @@ -16551,7 +16552,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16575,7 +16576,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "futures", @@ -16592,7 +16593,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.636.0" +version = "1.638.2" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16608,7 +16609,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -16629,7 +16630,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -16660,7 +16661,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-oauth2", @@ -16684,14 +16685,12 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", - "chrono", "futures", "k8s-openapi", "kube", - "schemars 0.8.22", "serde", "serde_json", "serde_yml", @@ -16704,7 +16703,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.636.0" +version = "1.638.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -16713,7 +16712,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "lazy_static", @@ -16725,7 +16724,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "serde_json", @@ -16737,7 +16736,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "gosyn", @@ -16749,7 +16748,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "lazy_static", @@ -16761,7 +16760,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "serde_json", @@ -16773,7 +16772,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "nu-parser", @@ -16784,7 +16783,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16795,7 +16794,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16808,7 +16807,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-recursion", @@ -16832,7 +16831,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "lazy_static", @@ -16846,7 +16845,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16863,7 +16862,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "lazy_static", @@ -16877,7 +16876,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "lazy_static", @@ -16896,7 +16895,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "serde", @@ -16907,7 +16906,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-recursion", @@ -16944,7 +16943,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "const_format", @@ -16982,7 +16981,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.636.0" +version = "1.638.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -16992,7 +16991,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-recursion", @@ -17021,7 +17020,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "axum 0.7.9", @@ -17044,7 +17043,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17077,7 +17076,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17097,7 +17096,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17131,7 +17130,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17165,7 +17164,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17188,7 +17187,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17212,7 +17211,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-nats", @@ -17236,7 +17235,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17271,7 +17270,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17299,7 +17298,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-trait", @@ -17322,7 +17321,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "chrono", @@ -17339,7 +17338,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.636.0" +version = "1.638.2" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9f03532048..2aac20ff57 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.636.0" +version = "1.638.2" authors.workspace = true edition.workspace = true @@ -75,7 +75,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.636.0" +version = "1.638.2" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b4ff5f38f7..a288a4215f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9f6e1e533df7711600ec2b8d5f0c958448db1a20 \ No newline at end of file +931813b75b8260faa13ddc07f36a11607b7e3bf6 \ No newline at end of file diff --git a/backend/migrations/20260213000000_worker_ping_native_mode.down.sql b/backend/migrations/20260213000000_worker_ping_native_mode.down.sql new file mode 100644 index 0000000000..77ebbd9dae --- /dev/null +++ b/backend/migrations/20260213000000_worker_ping_native_mode.down.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping DROP COLUMN IF EXISTS native_mode; diff --git a/backend/migrations/20260213000000_worker_ping_native_mode.up.sql b/backend/migrations/20260213000000_worker_ping_native_mode.up.sql new file mode 100644 index 0000000000..9966cc6543 --- /dev/null +++ b/backend/migrations/20260213000000_worker_ping_native_mode.up.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS native_mode BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/src/main.rs b/backend/src/main.rs index a55d95affd..3304f0c9df 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -47,8 +47,8 @@ use windmill_common::{ JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, @@ -61,8 +61,8 @@ use windmill_common::{ MODE_AND_ADDONS, }, worker::{ - reload_custom_tags_setting, Connection, HUB_CACHE_DIR, HUB_RT_CACHE_DIR, TMP_DIR, - TMP_LOGS_DIR, WORKER_GROUP, + is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR, + HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, }, KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED, }; @@ -485,8 +485,7 @@ fn print_help() { println!(" cache [hubPaths.json] Pre-cache hub scripts (default: ./hubPaths.json)"); println!(" cache-rt Pre-cache hub resource types"); println!(" sync-config Sync instance config from a YAML file to the database"); - println!(" operator Run the Kubernetes operator (watches WindmillInstance CRDs)"); - println!(" operator crd Print the WindmillInstance CRD YAML to stdout"); + println!(" operator Run the Kubernetes operator (watches a ConfigMap)"); println!(); println!("Environment variables (name = default):"); println!(" DATABASE_URL = The Postgres database url."); @@ -633,17 +632,11 @@ async fn windmill_main() -> anyhow::Result<()> { } #[cfg(feature = "operator")] "operator" => { - let sub_arg = std::env::args().nth(2).unwrap_or_default(); - if sub_arg == "crd" { - windmill_operator::print_crd_yaml(); - return Ok(()); - } - tracing_subscriber::fmt::init(); tracing::info!("Starting Windmill Kubernetes operator..."); tracing::info!("Connecting to database..."); let db = crate::db_connect::initial_connection().await?; - tracing::info!("Database connected. Starting controller..."); + tracing::info!("Database connected. Starting ConfigMap watcher..."); windmill_operator::run(db).await?; return Ok(()); } @@ -653,6 +646,9 @@ async fn windmill_main() -> anyhow::Result<()> { #[allow(unused_mut)] let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer || mode == Mode::MCP { 0 + } else if is_native_mode_from_env() { + println!("Native mode enabled: forcing NUM_WORKERS=8"); + 8 } else { std::env::var("NUM_WORKERS") .ok() @@ -660,11 +656,21 @@ async fn windmill_main() -> anyhow::Result<()> { .unwrap_or(DEFAULT_NUM_WORKERS as i32) }; - // TODO: maybe gate behind debug_assertions? - if num_workers > 1 && !std::env::var("WORKER_GROUP").is_ok_and(|x| x == "native") { - println!( - "We STRONGLY recommend using at most 1 worker per container, use at your own risks" - ); + if num_workers > 1 && !is_native_mode_from_env() { + if std::env::var("I_ACK_NUM_WORKERS_IS_UNSAFE").is_ok_and(|x| x == "1" || x == "true") { + println!( + "WARNING: Running with NUM_WORKERS={} without native mode. \ + This is not recommended. Use at your own risk.", + num_workers + ); + } else { + eprintln!( + "WARNING: NUM_WORKERS={} > 1 is only safe for native workers. \ + Falling back to NUM_WORKERS=1. Set NATIVE_MODE=true for native-only workers.", + num_workers + ); + num_workers = 1; + } } let server_mode = !std::env::var("DISABLE_SERVER") @@ -924,6 +930,16 @@ Windmill Community Edition {GIT_VERSION} ) .await; + // native_mode may also be set via DB worker group config (not just env). + // NATIVE_MODE_RESOLVED is updated by load_worker_config during initial_load. + if worker_mode + && !is_native_mode_from_env() + && NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) + { + num_workers = 8; + tracing::info!("Native mode detected from worker config: forcing NUM_WORKERS=8"); + } + monitor_db( &conn, &base_internal_url, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d36f2739cb..2e70267605 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -239,11 +239,16 @@ pub async fn initial_load( Connection::Http(_) => { // TODO: reload worker config from http let mut config = WORKER_CONFIG.write().await; + let worker_tags = DECODED_AGENT_TOKEN + .as_ref() + .map(|x| x.tags.clone()) + .unwrap_or_default(); + // we only check from env as native_mode is not stored in the token + let native_mode = windmill_common::worker::is_native_mode_from_env(); + windmill_common::worker::NATIVE_MODE_RESOLVED + .store(native_mode, std::sync::atomic::Ordering::Relaxed); *config = WorkerConfig { - worker_tags: DECODED_AGENT_TOKEN - .as_ref() - .map(|x| x.tags.clone()) - .unwrap_or_default(), + worker_tags, env_vars: load_env_vars( load_whitelist_env_vars_from_env(), &std::collections::HashMap::new(), @@ -257,6 +262,7 @@ pub async fn initial_load( cache_clear: None, additional_python_paths: None, pip_local_dependencies: None, + native_mode, }; } } @@ -2242,6 +2248,11 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b tracing::info!("Periodic script interval config changed, sending killpill. Expecting to be restarted by supervisor."); let _ = tx.send(); } + + if (*wc).native_mode != config.native_mode { + tracing::info!("Native mode config changed, sending killpill. Expecting to be restarted by supervisor."); + let _ = tx.send(); + } } drop(wc); diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index 4c85906563..cc0b8bae2c 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -240,7 +240,7 @@ async fn test_from_db_worker_config_prefix_stripping(db: Pool) { config.worker_configs.contains_key("my_group_name"), "worker__ prefix should be stripped" ); - assert_eq!(config.worker_configs["my_group_name"].cache_clear, Some(5)); + assert_eq!(config.worker_configs["my_group_name"].extra["cache_clear"], serde_json::json!(5)); } #[sqlx::test(fixtures("base"))] @@ -280,6 +280,7 @@ async fn test_apply_settings_diff_upserts_only(db: Pool) { m }, deletes: vec![], + ..Default::default() }; apply_settings_diff(&db, &diff).await.unwrap(); @@ -303,6 +304,7 @@ async fn test_apply_settings_diff_deletes_only(db: Pool) { let diff = SettingsDiff { upserts: BTreeMap::new(), deletes: vec!["to_delete_1".to_string(), "to_delete_2".to_string()], + ..Default::default() }; apply_settings_diff(&db, &diff).await.unwrap(); @@ -323,6 +325,7 @@ async fn test_apply_settings_diff_upserts_and_deletes(db: Pool) { m }, deletes: vec!["old_key".to_string()], + ..Default::default() }; apply_settings_diff(&db, &diff).await.unwrap(); @@ -358,6 +361,7 @@ async fn test_apply_settings_diff_upsert_overwrites(db: Pool) { m }, deletes: vec![], + ..Default::default() }; apply_settings_diff(&db, &diff).await.unwrap(); @@ -385,6 +389,7 @@ async fn test_apply_settings_diff_complex_json(db: Pool) { m }, deletes: vec![], + ..Default::default() }; apply_settings_diff(&db, &diff).await.unwrap(); @@ -396,7 +401,11 @@ async fn test_apply_settings_diff_complex_json(db: Pool) { #[sqlx::test(fixtures("base"))] async fn test_apply_settings_diff_delete_nonexistent_is_noop(db: Pool) { let diff = - SettingsDiff { upserts: BTreeMap::new(), deletes: vec!["does_not_exist".to_string()] }; + SettingsDiff { + upserts: BTreeMap::new(), + deletes: vec!["does_not_exist".to_string()], + ..Default::default() + }; // Should not error apply_settings_diff(&db, &diff).await.unwrap(); @@ -648,7 +657,11 @@ async fn test_roundtrip_to_settings_map_from_db_consistency(db: Pool) }; let map = original.to_settings_map(); - let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![] }; + let diff = SettingsDiff { + upserts: map.into_iter().collect(), + deletes: vec![], + ..Default::default() + }; apply_settings_diff(&db, &diff).await.unwrap(); @@ -694,6 +707,7 @@ async fn test_idempotent_apply(db: Pool) { m }, deletes: vec![], + ..Default::default() }; // Apply twice @@ -852,7 +866,7 @@ async fn test_full_config_roundtrip(db: Pool) { assert_eq!(otel.tracing_enabled, Some(true)); assert_eq!(config.worker_configs.len(), 2); - assert_eq!(config.worker_configs["default"].cache_clear, Some(7)); + assert_eq!(config.worker_configs["default"].extra["cache_clear"], serde_json::json!(7)); let gpu_auto = config.worker_configs["gpu"].autoscaling.as_ref().unwrap(); assert!(gpu_auto.enabled); assert_eq!(gpu_auto.min_workers, Some(0)); diff --git a/backend/windmill-api-settings/Cargo.toml b/backend/windmill-api-settings/Cargo.toml index 0a79246175..c2efd408b2 100644 --- a/backend/windmill-api-settings/Cargo.toml +++ b/backend/windmill-api-settings/Cargo.toml @@ -28,6 +28,7 @@ lazy_static.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true +serde_yml.workspace = true sqlx.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index e2768a8a0f..a36ca3a467 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -24,6 +24,8 @@ use windmill_common::usernames::generate_instance_username_for_all_users; use axum::{ extract::{Extension, Path}, routing::{get, post}, + body::Body, + response::Response, Json, Router, }; #[cfg(feature = "enterprise")] @@ -62,6 +64,7 @@ pub fn global_service() -> Router { "/instance_config", get(get_instance_config).put(set_instance_config), ) + .route("/instance_config/yaml", get(get_instance_config_yaml)) .route("/test_smtp", post(test_email)) .route("/test_license_key", post(test_license_key)) .route("/send_stats", post(send_stats)) @@ -275,13 +278,29 @@ pub async fn set_global_setting_internal( key: String, value: serde_json::Value, ) -> error::Result<()> { + let value = if key == "retention_period_secs" { + instance_config::clamp_retention_period(value) + } else { + value + }; + run_setting_pre_write_hook(db, &key, &value).await?; match value { serde_json::Value::Null => { + if instance_config::PROTECTED_SETTINGS.contains(&key.as_str()) { + return Err(error::Error::BadRequest( + format!("{key} is a protected setting and cannot be deleted"), + )); + } delete_global_setting(db, &key).await?; } serde_json::Value::String(x) if x.is_empty() => { + if instance_config::PROTECTED_SETTINGS.contains(&key.as_str()) { + return Err(error::Error::BadRequest( + format!("{key} is a protected setting and cannot be set to empty"), + )); + } delete_global_setting(db, &key).await?; } v => { @@ -409,6 +428,22 @@ async fn get_instance_config( Ok(Json(config)) } +async fn get_instance_config_yaml( + Extension(db): Extension, + authed: ApiAuthed, +) -> error::Result { + require_super_admin(&db, &authed.email).await?; + let config = InstanceConfig::from_db(&db) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; + let yaml = config.to_sorted_yaml() + .map_err(|e| error::Error::internal_err(e))?; + Response::builder() + .header("content-type", "application/yaml") + .body(Body::from(yaml)) + .map_err(|e| error::Error::internal_err(e.to_string())) +} + async fn set_instance_config( Extension(db): Extension, authed: ApiAuthed, @@ -1097,3 +1132,161 @@ async fn sync_cached_resource_types( cached_types.len() - synced_count )) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use windmill_common::instance_config::{GlobalSettings, InstanceConfig, WorkerGroupConfig}; + + #[test] + fn instance_config_yaml_round_trip() { + let config = InstanceConfig { + global_settings: GlobalSettings { + base_url: Some("https://windmill.example.com".to_string()), + retention_period_secs: Some(86400), + expose_metrics: Some(true), + ..Default::default() + }, + worker_configs: BTreeMap::from([( + "default".to_string(), + WorkerGroupConfig { + worker_tags: Some(vec!["deno".to_string(), "python3".to_string()]), + init_bash: Some("apt-get update".to_string()), + ..Default::default() + }, + )]), + }; + + let yaml = config.to_sorted_yaml().unwrap(); + + // Verify key fields appear in the YAML output + assert!(yaml.contains("base_url: https://windmill.example.com")); + assert!(yaml.contains("retention_period_secs: 86400")); + assert!(yaml.contains("expose_metrics: true")); + assert!(yaml.contains("default:")); + assert!(yaml.contains("- deno")); + assert!(yaml.contains("- python3")); + assert!(yaml.contains("init_bash: apt-get update")); + + // Round-trip back to struct + let deserialized: InstanceConfig = serde_yml::from_str(&yaml).unwrap(); + assert_eq!( + deserialized.global_settings.base_url.as_deref(), + Some("https://windmill.example.com") + ); + assert_eq!(deserialized.global_settings.retention_period_secs, Some(86400)); + assert_eq!(deserialized.global_settings.expose_metrics, Some(true)); + let wc = &deserialized.worker_configs["default"]; + assert_eq!( + wc.worker_tags.as_deref(), + Some(["deno".to_string(), "python3".to_string()].as_slice()) + ); + assert_eq!(wc.init_bash.as_deref(), Some("apt-get update")); + } + + #[test] + fn sorted_yaml_global_settings_alphabetical() { + let config = InstanceConfig { + global_settings: GlobalSettings { + retention_period_secs: Some(3600), + base_url: Some("https://test.com".to_string()), + expose_metrics: Some(true), + email_domain: Some("example.com".to_string()), + ..Default::default() + }, + worker_configs: BTreeMap::new(), + }; + + let yaml = config.to_sorted_yaml().unwrap(); + + // Keys must appear in alphabetical order + let base_url_pos = yaml.find("base_url:").unwrap(); + let email_pos = yaml.find("email_domain:").unwrap(); + let expose_pos = yaml.find("expose_metrics:").unwrap(); + let retention_pos = yaml.find("retention_period_secs:").unwrap(); + + assert!( + base_url_pos < email_pos + && email_pos < expose_pos + && expose_pos < retention_pos, + "global_settings keys should be alphabetically sorted, got yaml:\n{yaml}" + ); + } + + #[test] + fn sorted_yaml_worker_configs_default_and_native_first() { + let config = InstanceConfig { + global_settings: GlobalSettings::default(), + worker_configs: BTreeMap::from([ + ("gpu".to_string(), WorkerGroupConfig { + init_bash: Some("echo gpu".to_string()), + ..Default::default() + }), + ("native".to_string(), WorkerGroupConfig { + init_bash: Some("echo native".to_string()), + ..Default::default() + }), + ("default".to_string(), WorkerGroupConfig { + init_bash: Some("echo default".to_string()), + ..Default::default() + }), + ("alpha".to_string(), WorkerGroupConfig { + init_bash: Some("echo alpha".to_string()), + ..Default::default() + }), + ]), + }; + + let yaml = config.to_sorted_yaml().unwrap(); + + let default_pos = yaml.find("default:").unwrap(); + let native_pos = yaml.find("native:").unwrap(); + let alpha_pos = yaml.find("alpha:").unwrap(); + let gpu_pos = yaml.find("gpu:").unwrap(); + + assert!( + default_pos < native_pos + && native_pos < alpha_pos + && alpha_pos < gpu_pos, + "worker_configs should have default, native first, then rest alphabetically, got yaml:\n{yaml}" + ); + } + + #[test] + fn sorted_yaml_roundtrips() { + let config = InstanceConfig { + global_settings: GlobalSettings { + base_url: Some("https://rt.test".to_string()), + retention_period_secs: Some(7200), + expose_metrics: Some(false), + ..Default::default() + }, + worker_configs: BTreeMap::from([ + ("default".to_string(), WorkerGroupConfig { + worker_tags: Some(vec!["deno".to_string()]), + ..Default::default() + }), + ("native".to_string(), WorkerGroupConfig { + init_bash: Some("echo hi".to_string()), + ..Default::default() + }), + ]), + }; + + let yaml = config.to_sorted_yaml().unwrap(); + let deserialized: InstanceConfig = serde_yml::from_str(&yaml).unwrap(); + + assert_eq!(deserialized.global_settings.base_url.as_deref(), Some("https://rt.test")); + assert_eq!(deserialized.global_settings.retention_period_secs, Some(7200)); + assert_eq!(deserialized.global_settings.expose_metrics, Some(false)); + assert_eq!(deserialized.worker_configs.len(), 2); + assert_eq!( + deserialized.worker_configs["default"].worker_tags.as_deref(), + Some(["deno".to_string()].as_slice()) + ); + assert_eq!( + deserialized.worker_configs["native"].init_bash.as_deref(), + Some("echo hi") + ); + } +} diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 93c9deae84..6cfa398ac4 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -24,7 +24,7 @@ use windmill_common::{ DB, }; -use windmill_api_auth::{ApiAuthed, require_super_admin}; +use windmill_api_auth::{require_super_admin, ApiAuthed}; pub fn global_service() -> Router { Router::new() @@ -76,6 +76,8 @@ struct WorkerPing { wm_memory_usage: Option, #[serde(skip_serializing_if = "Option::is_none")] job_isolation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + native_mode: Option, } // #[derive(Serialize, Deserialize)] @@ -108,7 +110,7 @@ async fn list_worker_pings( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6eb5f9c139..0c4dfa3d3a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.636.0 + version: 1.638.2 title: Windmill API contact: @@ -22142,6 +22142,8 @@ components: type: number job_isolation: type: string + native_mode: + type: boolean required: - worker - worker_instance diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 27c30732b6..7270577e88 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -335,10 +335,18 @@ impl GlobalSettings { // including `extra` into a single map. skip_serializing_if ensures // None fields are omitted. let value = serde_json::to_value(self).expect("GlobalSettings serialization cannot fail"); - match value { + let mut map: BTreeMap = match value { serde_json::Value::Object(map) => map.into_iter().collect(), _ => unreachable!(), + }; + // Strip runtime-only `databases` sub-field from custom_instance_pg_databases. + // It contains setup status/logs managed by the setup endpoint, not configuration. + if let Some(pg) = map.get_mut("custom_instance_pg_databases") { + if let Some(obj) = pg.as_object_mut() { + obj.remove("databases"); + } } + map } } @@ -701,8 +709,6 @@ pub struct WorkerGroupConfig { #[serde(skip_serializing_if = "Option::is_none")] pub periodic_script_interval_seconds: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub cache_clear: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub additional_python_paths: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub pip_local_dependencies: Option>, @@ -714,6 +720,8 @@ pub struct WorkerGroupConfig { pub min_alive_workers_alert_threshold: Option, #[serde(skip_serializing_if = "Option::is_none")] pub autoscaling: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub native_mode: Option, /// Catch-all for fields not yet covered by typed fields. #[serde(flatten)] @@ -738,6 +746,10 @@ pub enum ApplyMode { pub struct SettingsDiff { pub upserts: BTreeMap, pub deletes: Vec, + /// Previous values for keys being updated or deleted (for logging). + pub previous_values: BTreeMap, + /// Count of desired keys that matched the current value exactly. + pub unchanged_count: usize, } /// The diff result for worker configs. @@ -754,10 +766,112 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "custom_instance_pg_databases", "uid", "rsa_keys", + "jwt_secret", + "min_keep_alive_version", ]; /// Internal settings that are never exposed via the API or included in config exports. -pub const HIDDEN_SETTINGS: &[&str] = &["uid", "rsa_keys", "jwt_secret", "min_keep_alive_version"]; +/// Note: jwt_secret is intentionally NOT hidden — it is included in YAML exports so that +/// operators can set it via ConfigMap. It is protected from deletion (PROTECTED_SETTINGS) +/// and from being set to empty/null, and its value is partially redacted in log output. +pub const HIDDEN_SETTINGS: &[&str] = &["uid", "min_keep_alive_version", "automate_username_creation"]; + +/// Top-level settings whose entire value is sensitive and must be fully redacted in logs. +const SENSITIVE_SETTINGS: &[&str] = &[ + "jwt_secret", + "rsa_keys", + "scim_token", + "hub_api_secret", + "license_key", + "ducklake_user_pg_pwd", + "pip_index_url", + "pip_extra_index_url", + "npm_config_registry", + "bunfig_install_scopes", + "maven_repos", + "ruby_repos", + "powershell_repo_pat", +]; + +/// Object-valued settings that contain sensitive sub-fields. +/// Maps a top-level key to the sub-field names that must be redacted. +const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[ + ("smtp_settings", &["smtp_password"]), + ("secret_backend", &["token"]), + ("object_store_cache_config", &["secret_key", "serviceAccountKey"]), +]; + +fn redact_json_value(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::String(s) => serde_json::json!(redact_string(s)), + _ => serde_json::json!("[redacted]"), + } +} + +fn mask_nested_sensitive(key: &str, value: &serde_json::Value) -> serde_json::Value { + for &(parent_key, sub_fields) in NESTED_SENSITIVE_FIELDS { + if key == parent_key { + if let serde_json::Value::Object(map) = value { + let mut masked = map.clone(); + for &field in sub_fields { + if let Some(v) = masked.get(field) { + masked.insert(field.to_string(), redact_json_value(v)); + } + } + return serde_json::Value::Object(masked); + } + } + } + // Settings that are maps-of-objects where each child has a sensitive sub-field. + const NESTED_MAP_SENSITIVE: &[(&str, &str)] = &[ + ("oauths", "secret"), + ("custom_instance_pg_databases", "user_pwd"), + ]; + for &(parent_key, child_field) in NESTED_MAP_SENSITIVE { + if key == parent_key { + if let serde_json::Value::Object(entries) = value { + let mut masked = entries.clone(); + for (_entry_key, entry_val) in masked.iter_mut() { + if let serde_json::Value::Object(ref mut obj) = entry_val { + if let Some(v) = obj.get(child_field) { + obj.insert(child_field.to_string(), redact_json_value(v)); + } + } + } + return serde_json::Value::Object(masked); + } + } + } + value.clone() +} + +fn redact_string(s: &str) -> String { + let char_count = s.chars().count(); + if char_count <= 6 { + "****".to_string() + } else { + let show = (char_count / 4).min(4); + let prefix: String = s.chars().take(show).collect(); + let suffix: String = s.chars().skip(char_count - show).collect(); + format!("{prefix}****{suffix}") + } +} + +fn format_setting_value(key: &str, value: &serde_json::Value) -> String { + if SENSITIVE_SETTINGS.contains(&key) { + return match value { + serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)), + _ => "[redacted]".to_string(), + }; + } + let value = mask_nested_sensitive(key, value); + let s = value.to_string(); + if s.len() > 200 { + format!("{}...", &s[..197]) + } else { + s + } +} /// Extract the expiry timestamp from a license key JSON value. /// @@ -772,9 +886,9 @@ fn license_key_expiry(value: &serde_json::Value) -> Option { parts[1].parse::().ok() } -/// Returns true if two license key values share the same client ID and signature -/// (i.e. they differ only in the expiry field). -fn license_keys_same_except_expiry(a: &serde_json::Value, b: &serde_json::Value) -> bool { +/// Returns true if two license key values share the same client ID +/// (i.e. they belong to the same customer, possibly with different expiry/signature). +fn license_keys_same_client(a: &serde_json::Value, b: &serde_json::Value) -> bool { let (Some(a_str), Some(b_str)) = (a.as_str(), b.as_str()) else { return false; }; @@ -783,7 +897,37 @@ fn license_keys_same_except_expiry(a: &serde_json::Value, b: &serde_json::Value) if a_parts.len() != 3 || b_parts.len() != 3 { return false; } - a_parts[0] == b_parts[0] && a_parts[2] == b_parts[2] + a_parts[0] == b_parts[0] +} + +fn is_empty_or_null(value: &serde_json::Value) -> bool { + value.is_null() || value.as_str().map_or(false, |s| s.is_empty()) +} + +/// Maximum retention period in seconds for CE builds (30 days). +pub const CE_MAX_RETENTION_PERIOD_SECS: i64 = 30 * 24 * 3600; + +/// Clamp `retention_period_secs` to `CE_MAX_RETENTION_PERIOD_SECS` on CE builds. +/// Returns the (possibly clamped) value. On EE builds this is a no-op. +pub fn clamp_retention_period(value: serde_json::Value) -> serde_json::Value { + #[cfg(feature = "enterprise")] + { + value + } + #[cfg(not(feature = "enterprise"))] + { + if let Some(secs) = value.as_i64() { + if secs > CE_MAX_RETENTION_PERIOD_SECS { + tracing::warn!( + "Clamping retention_period_secs from {} to {} (CE max: 30 days)", + secs, + CE_MAX_RETENTION_PERIOD_SECS + ); + return serde_json::json!(CE_MAX_RETENTION_PERIOD_SECS); + } + } + value + } } /// Compute the diff between current and desired global settings. @@ -793,40 +937,81 @@ pub fn diff_global_settings( mode: ApplyMode, ) -> SettingsDiff { let mut upserts = BTreeMap::new(); - for (key, value) in desired { + let mut previous_values = BTreeMap::new(); + let mut unchanged_count: usize = 0; + for (key, desired_value) in desired { + if PROTECTED_SETTINGS.contains(&key.as_str()) + && is_empty_or_null(desired_value) + && current.contains_key(key) + { + tracing::warn!( + "Skipping {key} update: protected setting cannot be overwritten with empty/null value" + ); + continue; + } + let mut value = if key == "retention_period_secs" { + clamp_retention_period(desired_value.clone()) + } else { + desired_value.clone() + }; + // Preserve the runtime-only `databases` sub-field inside + // `custom_instance_pg_databases` so that config sync never wipes + // setup status/logs that are managed by the setup endpoint. + if key == "custom_instance_pg_databases" { + if let Some(existing) = current.get(key) { + if let Some(databases) = existing.get("databases") { + if let Some(obj) = value.as_object_mut() { + obj.entry("databases").or_insert_with(|| databases.clone()); + } + } + } + } match current.get(key) { - Some(existing) if existing == value => {} // no change + Some(existing) if *existing == value => { + unchanged_count += 1; + } Some(existing) if key == LICENSE_KEY_SETTING => { - if license_keys_same_except_expiry(existing, value) { + if license_keys_same_client(existing, &value) { let current_expiry = license_key_expiry(existing).unwrap_or(0); - let desired_expiry = license_key_expiry(value).unwrap_or(0); + let desired_expiry = license_key_expiry(&value).unwrap_or(0); if desired_expiry > current_expiry { - upserts.insert(key.clone(), value.clone()); + previous_values.insert(key.clone(), existing.clone()); + upserts.insert(key.clone(), value); } else { tracing::info!( "Skipping license_key update: desired expiry ({}) is not posterior to current expiry ({})", desired_expiry, current_expiry ); + unchanged_count += 1; } } else { - upserts.insert(key.clone(), value.clone()); + previous_values.insert(key.clone(), existing.clone()); + upserts.insert(key.clone(), value); } } - _ => { - upserts.insert(key.clone(), value.clone()); + Some(existing) => { + previous_values.insert(key.clone(), existing.clone()); + upserts.insert(key.clone(), value); + } + None => { + upserts.insert(key.clone(), value); } } } let mut deletes = Vec::new(); if matches!(mode, ApplyMode::Replace) { for key in current.keys() { - if !desired.contains_key(key) && !PROTECTED_SETTINGS.contains(&key.as_str()) { + if !desired.contains_key(key) + && !PROTECTED_SETTINGS.contains(&key.as_str()) + && !HIDDEN_SETTINGS.contains(&key.as_str()) + { + previous_values.insert(key.clone(), current[key].clone()); deletes.push(key.clone()); } } } - SettingsDiff { upserts, deletes } + SettingsDiff { upserts, deletes, previous_values, unchanged_count } } /// Compute the diff between current and desired worker configs. @@ -869,14 +1054,35 @@ pub async fn apply_settings_diff( .bind(value) .execute(db) .await?; - tracing::info!("Synced global setting: {key}"); + if let Some(old_value) = diff.previous_values.get(key) { + tracing::info!( + "Updated global setting: {key} ({} -> {})", + format_setting_value(key, old_value), + format_setting_value(key, value) + ); + } else { + tracing::info!( + "Created global setting: {key} (value: {})", + format_setting_value(key, value) + ); + } } for key in &diff.deletes { sqlx::query("DELETE FROM global_settings WHERE name = $1") .bind(key) .execute(db) .await?; - tracing::info!("Deleted global setting: {key}"); + if let Some(old_value) = diff.previous_values.get(key) { + tracing::info!( + "Deleted global setting: {key} (was: {})", + format_setting_value(key, old_value) + ); + } else { + tracing::info!("Deleted global setting: {key}"); + } + } + if diff.unchanged_count > 0 { + tracing::info!("{} global setting(s) unchanged", diff.unchanged_count); } Ok(()) } @@ -1038,6 +1244,72 @@ impl InstanceConfig { Ok(()) } + + /// Serialize to YAML with sorted global_settings keys and worker_configs + /// ordered with "default" and "native" first. + pub fn to_sorted_yaml(&self) -> Result { + let settings_map = self.global_settings.to_settings_map(); + + let mut yaml = String::from("global_settings:\n"); + for (key, value) in &settings_map { + let value_yaml = serde_yml::to_string(value) + .map_err(|e| format!("YAML serialization failed for {key}: {e}"))?; + write_yaml_field(&mut yaml, key, value, &value_yaml, 1); + } + + if !self.worker_configs.is_empty() { + yaml.push_str("worker_configs:\n"); + let priority_keys = ["default", "native"]; + for &pk in &priority_keys { + if let Some(wc) = self.worker_configs.get(pk) { + let wc_value = serde_json::to_value(wc) + .map_err(|e| format!("JSON serialization failed for {pk}: {e}"))?; + let wc_yaml = serde_yml::to_string(&wc_value) + .map_err(|e| format!("YAML serialization failed for {pk}: {e}"))?; + write_yaml_field(&mut yaml, pk, &wc_value, &wc_yaml, 1); + } + } + for (key, wc) in &self.worker_configs { + if priority_keys.contains(&key.as_str()) { + continue; + } + let wc_value = serde_json::to_value(wc) + .map_err(|e| format!("JSON serialization failed for {key}: {e}"))?; + let wc_yaml = serde_yml::to_string(&wc_value) + .map_err(|e| format!("YAML serialization failed for {key}: {e}"))?; + write_yaml_field(&mut yaml, key, &wc_value, &wc_yaml, 1); + } + } + + Ok(yaml) + } +} + +fn write_yaml_field( + yaml: &mut String, + key: &str, + value: &serde_json::Value, + value_yaml: &str, + indent: usize, +) { + use std::fmt::Write; + + let prefix = " ".repeat(indent); + let trimmed = value_yaml.trim(); + let is_nested = match value { + serde_json::Value::Object(m) => !m.is_empty(), + serde_json::Value::Array(a) => !a.is_empty(), + _ => false, + }; + if is_nested { + let _ = writeln!(yaml, "{prefix}{key}:"); + let inner_prefix = " ".repeat(indent + 1); + for line in trimmed.lines() { + let _ = writeln!(yaml, "{inner_prefix}{line}"); + } + } else { + let _ = writeln!(yaml, "{prefix}{key}: {trimmed}"); + } } // --------------------------------------------------------------------------- @@ -1393,7 +1665,7 @@ mod tests { } #[test] - fn diff_global_settings_replace_protects_all_three() { + fn diff_global_settings_replace_protects_all() { let mut current = BTreeMap::new(); for key in PROTECTED_SETTINGS { current.insert(key.to_string(), serde_json::json!("val")); @@ -1576,7 +1848,7 @@ mod tests { assert_eq!(config.init_bash.as_deref(), Some("apt-get install -y curl")); assert_eq!(config.periodic_script_bash.as_deref(), Some("echo ping")); assert_eq!(config.periodic_script_interval_seconds, Some(300)); - assert_eq!(config.cache_clear, Some(7)); + assert_eq!(config.extra["cache_clear"], serde_json::json!(7)); assert_eq!(config.additional_python_paths.as_ref().unwrap().len(), 1); assert_eq!(config.pip_local_dependencies.as_ref().unwrap().len(), 1); assert_eq!(config.env_vars_static.as_ref().unwrap()["FOO"], "bar"); @@ -2024,7 +2296,7 @@ mod tests { } #[test] - fn diff_license_key_skips_equal_expiry() { + fn diff_license_key_skips_equal_expiry_different_sig() { let mut current = BTreeMap::new(); current.insert( "license_key".to_string(), @@ -2038,10 +2310,9 @@ mod tests { ); let diff = diff_global_settings(¤t, &desired, ApplyMode::Merge); - assert_eq!( - diff.upserts.len(), - 1, - "Different signature means different key, should update" + assert!( + diff.upserts.is_empty(), + "Same client and equal expiry should not update even with different signature" ); } @@ -2067,6 +2338,45 @@ mod tests { ); } + #[test] + fn diff_license_key_updates_newer_expiry_different_sig() { + let mut current = BTreeMap::new(); + current.insert( + "license_key".to_string(), + serde_json::json!("client1.1000000000.sig_old"), + ); + + let mut desired = BTreeMap::new(); + desired.insert( + "license_key".to_string(), + serde_json::json!("client1.2000000000.sig_new"), + ); + + let diff = diff_global_settings(¤t, &desired, ApplyMode::Merge); + assert_eq!(diff.upserts.len(), 1, "Same client with newer expiry should update even with different signature"); + } + + #[test] + fn diff_license_key_skips_older_expiry_different_sig() { + let mut current = BTreeMap::new(); + current.insert( + "license_key".to_string(), + serde_json::json!("client1.2000000000.sig_new"), + ); + + let mut desired = BTreeMap::new(); + desired.insert( + "license_key".to_string(), + serde_json::json!("client1.1000000000.sig_old"), + ); + + let diff = diff_global_settings(¤t, &desired, ApplyMode::Merge); + assert!( + diff.upserts.is_empty(), + "Same client with older expiry should not update even with different signature" + ); + } + #[test] fn diff_license_key_updates_different_client() { let mut current = BTreeMap::new(); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 46f13d4444..0ffb9a9272 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -156,6 +156,8 @@ lazy_static::lazy_static! { pub static ref NO_LOGS: bool = std::env::var("NO_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); + pub static ref NATIVE_MODE: bool = std::env::var("NATIVE_MODE").ok().is_some_and(|x| x == "1" || x == "true"); + pub static ref CGROUP_V2_PATH_RE: Regex = Regex::new(r#"(?m)^0::(/.*)$"#).unwrap(); pub static ref CGROUP_V2_CPU_RE: Regex = Regex::new(r#"(?m)^(\d+) \S+$"#).unwrap(); pub static ref CGROUP_V1_INACTIVE_FILE_RE: Regex = Regex::new(r#"(?m)^total_inactive_file (\d+)$"#).unwrap(); @@ -227,6 +229,7 @@ lazy_static::lazy_static! { additional_python_paths: Default::default(), pip_local_dependencies: Default::default(), env_vars: Default::default(), + native_mode: false, })); pub static ref WORKER_PULL_QUERIES: Arc>> = Arc::new(RwLock::new(vec![])); @@ -273,6 +276,17 @@ lazy_static::lazy_static! { pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); +/// Whether native mode is forced by the environment (NATIVE_MODE=true env var or WORKER_GROUP=native). +/// This does NOT account for native_mode set in the DB worker group config — for that, read +/// `WORKER_CONFIG.native_mode` which combines all sources. +pub fn is_native_mode_from_env() -> bool { + *NATIVE_MODE || *WORKER_GROUP == "native" +} + +/// Cached resolved native mode flag, updated when worker config is reloaded. +/// Use this for hot-path checks (e.g. per-job dispatch) to avoid read-locking WORKER_CONFIG. +pub static NATIVE_MODE_RESOLVED: AtomicBool = AtomicBool::new(false); + pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false); #[derive(Clone)] pub struct HttpClient { @@ -1248,6 +1262,7 @@ pub struct Ping { pub occupancy_rate_5m: Option, pub occupancy_rate_30m: Option, pub job_isolation: Option, + pub native_mode: Option, pub ping_type: PingType, } pub async fn update_ping_http( @@ -1271,6 +1286,7 @@ pub async fn update_ping_http( insert_ping.occupancy_rate_15s, insert_ping.occupancy_rate_5m, insert_ping.occupancy_rate_30m, + insert_ping.native_mode.unwrap_or(false), db, ) .await? @@ -1297,6 +1313,7 @@ pub async fn update_ping_http( insert_ping.vcpus, insert_ping.memory, insert_ping.job_isolation, + insert_ping.native_mode.unwrap_or(false), db, ) .await?; @@ -1428,11 +1445,12 @@ pub async fn insert_ping_query( vcpus: Option, memory: Option, job_isolation: Option, + native_mode: bool, db: &DB, ) -> anyhow::Result<()> { sqlx::query!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (worker) - DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers", + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) + DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode", worker_instance, worker_name, ip, @@ -1443,7 +1461,8 @@ pub async fn insert_ping_query( version, vcpus, memory, - job_isolation.as_deref() + job_isolation.as_deref(), + native_mode, ) .execute(db) .await?; @@ -1534,12 +1553,13 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_15s: Option, occupancy_rate_5m: Option, occupancy_rate_30m: Option, + native_mode: bool, db: &DB, ) -> anyhow::Result<()> { timeout(Duration::from_secs(10), sqlx::query!( "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), - memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6", jobs_executed, tags, occupancy_rate, @@ -1551,6 +1571,7 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + native_mode, ) .execute(db)) .await??; @@ -1789,6 +1810,9 @@ pub async fn load_worker_config( } } + let native_mode = is_native_mode_from_env() || config.native_mode.unwrap_or(false); + NATIVE_MODE_RESOLVED.store(native_mode, std::sync::atomic::Ordering::Relaxed); + Ok(WorkerConfig { worker_tags, priority_tags_sorted, @@ -1808,6 +1832,7 @@ pub async fn load_worker_config( .additional_python_paths .or_else(|| load_additional_python_paths_from_env()), env_vars: resolved_env_vars, + native_mode, }) } @@ -1896,6 +1921,7 @@ pub struct WorkerConfigOpt { pub pip_local_dependencies: Option>, pub env_vars_static: Option>, pub env_vars_allowlist: Option>, + pub native_mode: Option, } impl Default for WorkerConfigOpt { @@ -1913,6 +1939,7 @@ impl Default for WorkerConfigOpt { pip_local_dependencies: Default::default(), env_vars_static: Default::default(), env_vars_allowlist: Default::default(), + native_mode: Default::default(), } } } @@ -1930,12 +1957,13 @@ pub struct WorkerConfig { pub additional_python_paths: Option>, pub pip_local_dependencies: Option>, pub env_vars: HashMap, + pub native_mode: bool, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", ")) + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode) } } diff --git a/backend/windmill-operator/Cargo.toml b/backend/windmill-operator/Cargo.toml index 9ebef0b93b..30a7c0bb7a 100644 --- a/backend/windmill-operator/Cargo.toml +++ b/backend/windmill-operator/Cargo.toml @@ -18,13 +18,11 @@ serde.workspace = true serde_json.workspace = true sqlx.workspace = true tracing.workspace = true -windmill-common = { workspace = true, default-features = false, features = ["instance_config_schema"] } +windmill-common = { workspace = true, default-features = false } anyhow.workspace = true thiserror.workspace = true kube.workspace = true k8s-openapi.workspace = true tokio.workspace = true futures.workspace = true -chrono.workspace = true -schemars = "0.8" serde_yml.workspace = true diff --git a/backend/windmill-operator/manifests/crd.yaml b/backend/windmill-operator/manifests/crd.yaml deleted file mode 100644 index c6093a873b..0000000000 --- a/backend/windmill-operator/manifests/crd.yaml +++ /dev/null @@ -1,703 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: windmillinstances.windmill.dev -spec: - group: windmill.dev - names: - categories: [] - kind: WindmillInstance - plural: windmillinstances - shortNames: - - wmi - singular: windmillinstance - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: '.status.synced' - name: Synced - type: string - - jsonPath: '.status.lastSyncedAt' - name: Last Synced - type: date - - jsonPath: '.metadata.creationTimestamp' - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: Auto-generated derived type for WindmillInstanceSpec via `CustomResource` - properties: - spec: - description: |- - WindmillInstance CRD spec. - - Declares the desired state for instance-level configuration: - `global_settings` maps directly to the `global_settings` table - `worker_configs` maps to the `config` table with a `worker__` prefix - properties: - global_settings: - default: {} - description: Global settings to sync to the `global_settings` table. - properties: - app_workspaced_route: - nullable: true - type: boolean - base_url: - nullable: true - type: string - bunfig_install_scopes: - nullable: true - type: string - critical_alert_mute_ui: - nullable: true - type: boolean - critical_alerts_on_db_oversize: - description: Configuration for critical alerts when the database exceeds a size threshold. - nullable: true - properties: - enabled: - default: false - type: boolean - value: - default: 0.0 - format: float - type: number - type: object - critical_error_channels: - items: - anyOf: - - required: - - email - - required: - - slack_channel - - required: - - teams_channel - description: A channel for delivering critical error alerts. - properties: - email: - type: string - slack_channel: - type: string - teams_channel: - description: Microsoft Teams channel reference. - properties: - channel_id: - type: string - channel_name: - type: string - team_id: - type: string - team_name: - type: string - required: - - channel_id - - channel_name - - team_id - - team_name - type: object - type: object - nullable: true - type: array - custom_instance_pg_databases: - description: Custom PostgreSQL databases managed by the instance. - nullable: true - properties: - databases: - additionalProperties: - description: Status of a single custom instance database. - properties: - error: - nullable: true - type: string - logs: - default: - super_admin: '' - description: Setup log entries for a custom instance database. - properties: - created_database: - type: string - database_credentials: - type: string - db_connect: - type: string - grant_permissions: - type: string - super_admin: - default: '' - type: string - valid_dbname: - type: string - type: object - success: - default: false - type: boolean - tag: - nullable: true - type: string - type: object - type: object - user_pwd: - nullable: true - type: string - type: object - custom_tags: - items: - type: string - nullable: true - type: array - default_tags_per_workspace: - nullable: true - type: boolean - default_tags_workspaces: - items: - type: string - nullable: true - type: array - dev_instance: - nullable: true - type: boolean - disable_stats: - nullable: true - type: boolean - ducklake_settings: - description: DuckLake catalog database settings. - nullable: true - properties: - ducklakes: - additionalProperties: - description: A single DuckLake instance configuration. - properties: - catalog: - description: DuckLake catalog backend reference. - properties: - resource_path: - type: string - resource_type: - description: The type of database backing a DuckLake catalog. - enum: - - postgresql - - mysql - - instance - type: string - required: - - resource_path - - resource_type - type: object - extra_args: - nullable: true - type: string - storage: - description: DuckLake storage location. - properties: - path: - type: string - storage: - nullable: true - type: string - required: - - path - type: object - required: - - catalog - - storage - type: object - type: object - required: - - ducklakes - type: object - email_domain: - nullable: true - type: string - expose_debug_metrics: - nullable: true - type: boolean - expose_metrics: - nullable: true - type: boolean - hub_accessible_url: - nullable: true - type: string - hub_api_secret: - nullable: true - type: string - hub_base_url: - nullable: true - type: string - indexer_settings: - description: Full-text search indexer configuration. - nullable: true - properties: - commit_job_max_batch_size: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - commit_log_max_batch_size: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - max_indexed_job_log_size: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - refresh_index_period: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - refresh_log_index_period: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - should_clear_job_index: - nullable: true - type: boolean - should_clear_log_index: - nullable: true - type: boolean - writer_memory_budget: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - type: object - instance_python_version: - nullable: true - type: string - job_default_timeout: - format: int64 - nullable: true - type: integer - jwt_secret: - nullable: true - type: string - keep_job_dir: - nullable: true - type: boolean - license_key: - nullable: true - type: string - maven_repos: - nullable: true - type: string - min_keep_alive_version: - nullable: true - type: string - monitor_logs_on_s3: - nullable: true - type: boolean - no_default_maven: - nullable: true - type: boolean - npm_config_registry: - nullable: true - type: string - nuget_config: - nullable: true - type: string - oauths: - additionalProperties: - description: OAuth client configuration for a single provider. - properties: - allowed_domains: - items: - type: string - nullable: true - type: array - connect_config: - description: OAuth provider endpoint configuration. - nullable: true - properties: - auth_url: - type: string - extra_params: - additionalProperties: - type: string - nullable: true - type: object - extra_params_callback: - additionalProperties: - type: string - nullable: true - type: object - req_body_auth: - nullable: true - type: boolean - scopes: - items: - type: string - nullable: true - type: array - token_url: - type: string - userinfo_url: - nullable: true - type: string - required: - - auth_url - - token_url - type: object - id: - type: string - login_config: - description: OAuth provider endpoint configuration. - nullable: true - properties: - auth_url: - type: string - extra_params: - additionalProperties: - type: string - nullable: true - type: object - extra_params_callback: - additionalProperties: - type: string - nullable: true - type: object - req_body_auth: - nullable: true - type: boolean - scopes: - items: - type: string - nullable: true - type: array - token_url: - type: string - userinfo_url: - nullable: true - type: string - required: - - auth_url - - token_url - type: object - secret: - type: string - required: - - id - - secret - type: object - nullable: true - type: object - object_store_cache_config: - nullable: true - openai_azure_base_path: - nullable: true - type: string - otel: - description: OpenTelemetry exporter configuration. - nullable: true - properties: - logs_enabled: - nullable: true - type: boolean - metrics_enabled: - nullable: true - type: boolean - otel_exporter_otlp_compression: - nullable: true - type: string - otel_exporter_otlp_endpoint: - nullable: true - type: string - otel_exporter_otlp_headers: - nullable: true - type: string - otel_exporter_otlp_protocol: - nullable: true - type: string - tracing_enabled: - nullable: true - type: boolean - type: object - otel_tracing_proxy: - description: Per-language HTTP request tracing proxy configuration. - nullable: true - properties: - enabled: - default: false - type: boolean - enabled_languages: - items: - description: Script language identifier. - enum: - - python3 - - deno - - go - - bash - - powershell - - postgresql - - bun - - bunnative - - mysql - - bigquery - - snowflake - - graphql - - nativets - - mssql - - oracledb - - duckdb - - php - - rust - - ansible - - csharp - - nu - - java - - ruby - type: string - type: array - type: object - pip_extra_index_url: - nullable: true - type: string - pip_index_url: - nullable: true - type: string - powershell_repo_pat: - nullable: true - type: string - powershell_repo_url: - nullable: true - type: string - request_size_limit_mb: - format: int64 - nullable: true - type: integer - require_preexisting_user_for_oauth: - nullable: true - type: boolean - retention_period_secs: - format: int64 - nullable: true - type: integer - ruby_repos: - nullable: true - type: string - saml_metadata: - nullable: true - type: string - scim_token: - nullable: true - type: string - secret_backend: - nullable: true - slack: - nullable: true - smtp_settings: - description: SMTP server configuration. - nullable: true - properties: - smtp_disable_tls: - nullable: true - type: boolean - smtp_from: - nullable: true - type: string - smtp_host: - nullable: true - type: string - smtp_password: - nullable: true - type: string - smtp_port: - format: uint16 - minimum: 0.0 - nullable: true - type: integer - smtp_tls_implicit: - nullable: true - type: boolean - smtp_username: - nullable: true - type: string - type: object - teams: - nullable: true - timeout_wait_result: - format: int64 - nullable: true - type: integer - type: object - x-kubernetes-preserve-unknown-fields: true - worker_configs: - additionalProperties: - description: Worker group configuration. - properties: - additional_python_paths: - items: - type: string - nullable: true - type: array - autoscaling: - description: Worker group autoscaling configuration. - nullable: true - properties: - cooldown_seconds: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - custom_tags: - items: - type: string - nullable: true - type: array - dec_scale_occupancy_rate: - format: uint8 - minimum: 0.0 - nullable: true - type: integer - enabled: - default: false - type: boolean - full_scale_cooldown_seconds: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - full_scale_jobs_waiting: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - inc_num_workers: - format: uint32 - minimum: 0.0 - nullable: true - type: integer - inc_scale_num_jobs_waiting: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - inc_scale_occupancy_rate: - format: uint8 - minimum: 0.0 - nullable: true - type: integer - integration: - description: |- - Autoscaling integration backend. - - The `type` field selects the backend: `"script"`, `"dryrun"`, or `"kubernetes"`. For `"script"`, `path` is required and `tag` is optional. - nullable: true - properties: - path: - nullable: true - type: string - tag: - nullable: true - type: string - type: - type: string - required: - - type - type: object - max_workers: - format: uint32 - minimum: 0.0 - nullable: true - type: integer - min_workers: - format: uint32 - minimum: 0.0 - nullable: true - type: integer - type: object - cache_clear: - format: uint32 - minimum: 0.0 - nullable: true - type: integer - dedicated_worker: - nullable: true - type: string - dedicated_workers: - items: - type: string - nullable: true - type: array - env_vars_allowlist: - items: - type: string - nullable: true - type: array - env_vars_static: - additionalProperties: - type: string - nullable: true - type: object - init_bash: - nullable: true - type: string - min_alive_workers_alert_threshold: - format: uint32 - minimum: 0.0 - nullable: true - type: integer - periodic_script_bash: - nullable: true - type: string - periodic_script_interval_seconds: - format: uint64 - minimum: 0.0 - nullable: true - type: integer - pip_local_dependencies: - items: - type: string - nullable: true - type: array - priority_tags: - additionalProperties: - format: uint8 - minimum: 0.0 - type: integer - nullable: true - type: object - worker_tags: - items: - type: string - nullable: true - type: array - type: object - x-kubernetes-preserve-unknown-fields: true - default: {} - description: Worker group configs to sync to the `config` table. Keys are worker group names (e.g. "default", "gpu"). Each key is stored in the DB as `worker__`. - type: object - type: object - status: - description: Status subresource for WindmillInstance. - nullable: true - properties: - lastSyncedAt: - description: Timestamp of the last successful sync. - nullable: true - type: string - message: - default: '' - description: Human-readable status message. - type: string - observedGeneration: - default: 0 - description: The `.metadata.generation` that was last observed. - format: int64 - type: integer - synced: - description: Whether the last reconciliation was successful. - type: boolean - required: - - synced - type: object - required: - - spec - title: WindmillInstance - type: object - served: true - storage: true - subresources: - status: {} - diff --git a/backend/windmill-operator/src/lib.rs b/backend/windmill-operator/src/lib.rs index f49335f16d..ffdc4f804c 100644 --- a/backend/windmill-operator/src/lib.rs +++ b/backend/windmill-operator/src/lib.rs @@ -1,5 +1,5 @@ #[cfg(feature = "private")] -pub mod crd_ee; +pub mod configmap_ee; #[cfg(feature = "private")] pub mod db_sync_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-operator/src/operator_oss.rs b/backend/windmill-operator/src/operator_oss.rs index d20cce548a..83cb7635e5 100644 --- a/backend/windmill-operator/src/operator_oss.rs +++ b/backend/windmill-operator/src/operator_oss.rs @@ -1,19 +1,7 @@ #[cfg(feature = "private")] pub use crate::reconciler_ee::run; -#[cfg(feature = "private")] -pub fn print_crd_yaml() { - use kube::CustomResourceExt; - let crd = crate::crd_ee::WindmillInstance::crd(); - println!("{}", serde_yml::to_string(&crd).unwrap()); -} - #[cfg(not(feature = "private"))] pub async fn run(_db: sqlx::Pool) -> anyhow::Result<()> { anyhow::bail!("K8s operator is not available in this build") } - -#[cfg(not(feature = "private"))] -pub fn print_crd_yaml() { - eprintln!("K8s operator CRD generation is not available in this build"); -} diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index a1538be04b..2339c65e69 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -102,6 +102,21 @@ impl ScriptLang { ) } + pub fn is_native(&self) -> bool { + matches!( + self, + ScriptLang::Bunnative | + ScriptLang::Nativets | + ScriptLang::Postgresql | + ScriptLang::Mysql | + ScriptLang::Graphql | + ScriptLang::Snowflake | + ScriptLang::Mssql | + ScriptLang::Bigquery | + ScriptLang::OracleDB + ) + } + pub fn as_comment_lit(&self) -> String { use ScriptLang::*; match self { @@ -142,9 +157,7 @@ impl FromStr for ScriptLang { "java" => ScriptLang::Java, "ruby" => ScriptLang::Ruby, // for related places search: ADD_NEW_LANG - language => { - return Err(anyhow::anyhow!("{} is currently not supported", language)) - } + language => return Err(anyhow::anyhow!("{} is currently not supported", language)), }; Ok(language) @@ -581,9 +594,7 @@ where pub fn to_i64(s: &str) -> anyhow::Result { let v = hex::decode(s)?; if v.len() < 8 { - return Err(anyhow::anyhow!( - "hex string did not decode to an u64: {s}", - )); + return Err(anyhow::anyhow!("hex string did not decode to an u64: {s}",)); } let nb: u64 = u64::from_be_bytes( v[0..8] diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1be0f474da..76c47e6d6b 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -562,6 +562,7 @@ pub async fn update_worker_ping_for_failed_init_script( memory_usage: None, wm_memory_usage: None, job_isolation: None, + native_mode: None, ping_type: PingType::InitScript, }, ) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index d350f90c63..4481e46a60 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -80,7 +80,7 @@ use windmill_common::{ scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, tracing_init::{QUIET_MODE, VERBOSE_TARGET}, utils::StripPath, - worker::{CLOUD_HOSTED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP}, + worker::{CLOUD_HOSTED, NATIVE_MODE_RESOLVED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP}, DB, IS_READY, }; @@ -3010,6 +3010,17 @@ pub async fn handle_queued_job( _ => {} } + if NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) { + if let Some(lang) = &job.script_lang { + if !lang.is_native() { + return Err(Error::ExecutionErr(format!( + "Worker is in native mode and cannot execute non-native job with language '{}'", + lang.as_str(), + ))); + } + } + } + #[cfg(any(not(feature = "enterprise"), feature = "sqlx"))] match conn { Connection::Sql(db) => { diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 396b9273ab..7a8c1e7ebb 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -7,8 +7,8 @@ use windmill_common::{ worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, - update_worker_ping_main_loop_query, Connection, Ping, PingType, WORKER_CONFIG, - WORKER_GROUP, + update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED, + WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, DB, }; @@ -27,7 +27,10 @@ pub(crate) async fn update_worker_ping_full( occupancy_metrics: &mut OccupancyMetrics, killpill_tx: &KillpillSender, ) { - let tags = WORKER_CONFIG.read().await.worker_tags.clone(); + let wc = WORKER_CONFIG.read().await; + let tags = wc.worker_tags.clone(); + let native_mode = wc.native_mode; + drop(wc); let memory_usage = get_worker_memory_usage(); let wm_memory_usage = get_windmill_memory_usage(); @@ -60,6 +63,7 @@ pub(crate) async fn update_worker_ping_full( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + native_mode, ) }) .retry( @@ -105,6 +109,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_15s: Option, occupancy_rate_5m: Option, occupancy_rate_30m: Option, + native_mode: bool, ) -> anyhow::Result<()> { match conn { Connection::Sql(db) => { @@ -120,6 +125,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + native_mode, db, ) .await?; @@ -148,6 +154,7 @@ async fn update_worker_ping_full_inner( memory_usage: get_worker_memory_usage(), wm_memory_usage: get_windmill_memory_usage(), job_isolation: None, + native_mode: Some(native_mode), ping_type: PingType::MainLoop, }, ) @@ -163,7 +170,7 @@ pub async fn insert_ping( ip: &str, db: &Connection, ) -> anyhow::Result<()> { - let (tags, dw, dws) = { + let (tags, dw, dws, native_mode) = { let wc = WORKER_CONFIG.read().await.clone(); ( wc.worker_tags, @@ -176,6 +183,7 @@ pub async fn insert_ping( .map(|x| format!("{}:{}", x.workspace_id, x.path)) .collect::>() }), + wc.native_mode, ) }; @@ -204,6 +212,7 @@ pub async fn insert_ping( vcpus, memory, job_isolation, + native_mode, db, ) .await?; @@ -232,6 +241,7 @@ pub async fn insert_ping( memory_usage: get_worker_memory_usage(), wm_memory_usage: get_windmill_memory_usage(), job_isolation, + native_mode: Some(native_mode), ping_type: PingType::Initial, }, ) @@ -305,6 +315,9 @@ pub async fn update_worker_ping_from_job( occupancy_rate_5m: occupancy_rate_5m, occupancy_rate_30m: occupancy_rate_30m, job_isolation, + native_mode: Some( + NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed), + ), }, ) .await?; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index a972758cd0..a70bdc01d5 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.636.0"; +export const VERSION = "v1.638.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index d9616d9d71..05fabbdc5b 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -79,7 +79,7 @@ export { // } // }); -export const VERSION = "1.636.0"; +export const VERSION = "1.638.2"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/docker-compose.yml b/docker-compose.yml index 7af40e7025..7152b8bf55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,6 +103,7 @@ services: - DATABASE_URL=${DATABASE_URL} - MODE=worker - WORKER_GROUP=native + - NATIVE_MODE=true - NUM_WORKERS=8 - SLEEP_QUEUE=200 depends_on: diff --git a/examples/deploy/instance-config-iac/README.md b/examples/deploy/instance-config-iac/README.md index a7e8225748..08db7873f0 100644 --- a/examples/deploy/instance-config-iac/README.md +++ b/examples/deploy/instance-config-iac/README.md @@ -173,23 +173,17 @@ If you only want to manage a subset of settings, include all settings you want t ## Kubernetes (Operator) -The Windmill Kubernetes operator watches `WindmillInstance` Custom Resources and continuously reconciles the database to match the declared state. It also supports `secretKeyRef` to pull values from Kubernetes Secrets natively. +The Windmill Kubernetes operator watches a ConfigMap and continuously reconciles the database to match the declared state. It also supports `secretKeyRef` to pull values from Kubernetes Secrets natively. ### Prerequisites - Windmill built with the `operator` feature flag - RBAC permissions for the operator pod (see below) -- The CRD installed in the cluster +- A ConfigMap named `windmill-instance` (or a custom name via the `OPERATOR_CONFIGMAP` env var) ### Setup -**1. Install the CRD**: - -```bash -windmill operator crd | kubectl apply -f - -``` - -**2. Create a Kubernetes Secret for sensitive values**: +**1. Create a Kubernetes Secret for sensitive values**: ```yaml apiVersion: v1 @@ -204,78 +198,91 @@ stringData: google-oauth-secret: "your-google-oauth-secret" ``` -**3. Create the WindmillInstance resource** (`windmill-instance.yaml`): +**2. Create the ConfigMap** (`windmill-instance.yaml`): ```yaml -apiVersion: windmill.dev/v1alpha1 -kind: WindmillInstance +apiVersion: v1 +kind: ConfigMap metadata: - name: production + name: windmill-instance namespace: windmill -spec: - global_settings: - base_url: "https://windmill.example.com" - license_key: - secretKeyRef: - name: windmill-secrets - key: license-key - retention_period_secs: 2592000 - expose_metrics: true - smtp_settings: - smtp_host: "smtp.example.com" - smtp_port: 587 - smtp_from: "windmill@example.com" - smtp_password: +data: + spec: | + global_settings: + base_url: "https://windmill.example.com" + license_key: secretKeyRef: name: windmill-secrets - key: smtp-password - oauths: - google: - id: "google-client-id" - secret: + key: license-key + retention_period_secs: 2592000 + expose_metrics: true + smtp_settings: + smtp_host: "smtp.example.com" + smtp_port: 587 + smtp_from: "windmill@example.com" + smtp_password: secretKeyRef: name: windmill-secrets - key: google-oauth-secret - login_config: - auth_url: "https://accounts.google.com/o/oauth2/v2/auth" - token_url: "https://oauth2.googleapis.com/token" - userinfo_url: "https://openidconnect.googleapis.com/v1/userinfo" - scopes: ["openid", "profile", "email"] - custom_tags: - - gpu - - high-mem + key: smtp-password + oauths: + google: + id: "google-client-id" + secret: + secretKeyRef: + name: windmill-secrets + key: google-oauth-secret + login_config: + auth_url: "https://accounts.google.com/o/oauth2/v2/auth" + token_url: "https://oauth2.googleapis.com/token" + userinfo_url: "https://openidconnect.googleapis.com/v1/userinfo" + scopes: ["openid", "profile", "email"] + custom_tags: + - gpu + - high-mem - worker_configs: - default: - worker_tags: ["deno", "python3", "bun", "go", "bash", "powershell"] - init_bash: "echo 'Worker starting'" - native: - worker_tags: ["nativets"] + worker_configs: + default: + worker_tags: ["deno", "python3", "bun", "go", "bash", "powershell"] + init_bash: "echo 'Worker starting'" + native: + worker_tags: ["nativets"] ``` -**4. Apply**: +The config lives under `data.spec` as a YAML string. This is the same schema used by `sync-config`. + +**3. Apply**: ```bash kubectl apply -f windmill-instance.yaml ``` -**5. Check status**: +**4. Check sync status** via events: ```bash -kubectl get wmi -# NAME SYNCED LAST SYNCED AGE -# production true 2025-01-15T10:30:00Z 2d +kubectl get events --field-selector involvedObject.name=windmill-instance ``` +### License key handling + +If `license_key` is absent or empty in the ConfigMap but already exists in the database, the operator preserves the database value. This lets you manage the license key separately (e.g., via the UI) without the operator overwriting it. + +### Environment variables + +| Variable | Default | Description | +|---|---|---| +| `OPERATOR_NAMESPACE` | Pod's own namespace | Namespace of the ConfigMap | +| `OPERATOR_CONFIGMAP` | `windmill-instance` | Name of the ConfigMap to watch | + ### Using `envRef` in Kubernetes `envRef` also works in the operator context. Values are resolved from the operator pod's environment. This is useful when secrets are injected via pod env vars (e.g., from a vault sidecar): ```yaml -spec: - global_settings: - license_key: - envRef: "WM_LICENSE_KEY" # Read from operator pod env +data: + spec: | + global_settings: + license_key: + envRef: "WM_LICENSE_KEY" # Read from operator pod env ``` The operator pod's Deployment would include: @@ -289,21 +296,22 @@ env: key: license-key ``` -This is functionally equivalent to using `secretKeyRef` directly in the CRD, but lets you use any secret injection mechanism your cluster supports (external-secrets, vault-agent, etc.). +This is functionally equivalent to using `secretKeyRef` directly in the ConfigMap, but lets you use any secret injection mechanism your cluster supports (external-secrets, vault-agent, etc.). ### RBAC -The operator pod needs permissions to read Secrets and manage the CRD. Minimal ClusterRole: +The operator pod needs permissions to read ConfigMaps, Secrets, and create Events. Minimal Role: ```yaml apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: name: windmill-operator + namespace: windmill rules: - - apiGroups: ["windmill.dev"] - resources: ["windmillinstances", "windmillinstances/status"] - verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] @@ -312,6 +320,8 @@ rules: verbs: ["create", "patch"] ``` +Note: This is now a namespace-scoped **Role** (not ClusterRole), since there is no CRD to manage. + ### Running the operator ```bash @@ -334,15 +344,3 @@ DATABASE_URL=postgres://... windmill operator | Requires RBAC for Secrets | No | Yes | **Recommendation**: Use `envRef` for portability across deployment targets. Use `secretKeyRef` when you want direct Kubernetes-native secret binding without intermediate env vars. - ---- - -## Full Settings Reference - -For a complete list of available settings fields, generate the CRD schema: - -```bash -windmill operator crd -``` - -The CRD's OpenAPI schema documents every field, its type, and whether it's optional. The same schema applies to `sync-config` YAML files. diff --git a/examples/deploy/instance-config-iac/k8s-windmill-instance.yaml b/examples/deploy/instance-config-iac/k8s-windmill-instance.yaml index 977dd86d28..9f8fcf13f0 100644 --- a/examples/deploy/instance-config-iac/k8s-windmill-instance.yaml +++ b/examples/deploy/instance-config-iac/k8s-windmill-instance.yaml @@ -1,67 +1,67 @@ -# Example: WindmillInstance CRD for the Kubernetes operator. +# Example: ConfigMap for the Windmill Kubernetes operator. # # Prerequisites: -# 1. Install the CRD: windmill operator crd | kubectl apply -f - -# 2. Create the Secret: kubectl apply -f k8s-secrets.yaml -# 3. Apply this file: kubectl apply -f k8s-windmill-instance.yaml -# 4. Check status: kubectl get wmi +# 1. Create the Secret: kubectl apply -f k8s-secrets.yaml +# 2. Apply this file: kubectl apply -f k8s-windmill-instance.yaml +# 3. Check events: kubectl get events --field-selector involvedObject.name=windmill-instance -apiVersion: windmill.dev/v1alpha1 -kind: WindmillInstance +apiVersion: v1 +kind: ConfigMap metadata: - name: production + name: windmill-instance namespace: windmill -spec: - global_settings: - base_url: "https://windmill.example.com" +data: + spec: | + global_settings: + base_url: "https://windmill.example.com" - # Secret reference: reads "license-key" from K8s Secret "windmill-secrets" - license_key: - secretKeyRef: - name: windmill-secrets - key: license-key - - retention_period_secs: 2592000 - job_default_timeout: 900 - expose_metrics: true - - smtp_settings: - smtp_host: "smtp.example.com" - smtp_port: 587 - smtp_from: "windmill@example.com" - smtp_tls_implicit: false - smtp_password: + # Secret reference: reads "license-key" from K8s Secret "windmill-secrets" + license_key: secretKeyRef: name: windmill-secrets - key: smtp-password + key: license-key - oauths: - google: - id: "your-google-client-id" - secret: + retention_period_secs: 2592000 + job_default_timeout: 900 + expose_metrics: true + + smtp_settings: + smtp_host: "smtp.example.com" + smtp_port: 587 + smtp_from: "windmill@example.com" + smtp_tls_implicit: false + smtp_password: secretKeyRef: name: windmill-secrets - key: google-oauth-secret - login_config: - auth_url: "https://accounts.google.com/o/oauth2/v2/auth" - token_url: "https://oauth2.googleapis.com/token" - userinfo_url: "https://openidconnect.googleapis.com/v1/userinfo" - scopes: ["openid", "profile", "email"] + key: smtp-password - custom_tags: - - gpu - - high-mem + oauths: + google: + id: "your-google-client-id" + secret: + secretKeyRef: + name: windmill-secrets + key: google-oauth-secret + login_config: + auth_url: "https://accounts.google.com/o/oauth2/v2/auth" + token_url: "https://oauth2.googleapis.com/token" + userinfo_url: "https://openidconnect.googleapis.com/v1/userinfo" + scopes: ["openid", "profile", "email"] - worker_configs: - default: - worker_tags: - - deno - - python3 - - bun - - go - - bash - - powershell + custom_tags: + - gpu + - high-mem - native: - worker_tags: - - nativets + worker_configs: + default: + worker_tags: + - deno + - python3 + - bun + - go + - bash + - powershell + + native: + worker_tags: + - nativets diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0d3f31a510..14fb74d9e4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.636.0", + "version": "1.638.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.636.0", + "version": "1.638.2", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index dabafdf3c3..bcaf8c05f7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.636.0", + "version": "1.638.2", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte new file mode 100644 index 0000000000..b9204b7ae9 --- /dev/null +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -0,0 +1,83 @@ + + + + +{#if $subOpen} +
+ {#each subItems as subItem} + {#if subItem.separatorTop} +
+ {/if} + subItem?.action?.(e)} + href={subItem?.href} + target={subItem?.hrefTarget} + disabled={subItem?.disabled} + class={twMerge( + 'px-4 py-2 text-primary font-normal hover:bg-surface-hover cursor-pointer text-xs transition-colors w-full', + 'data-[highlighted]:bg-surface-hover', + 'flex flex-row gap-2 items-center rounded-sm', + subItem?.disabled && 'text-disabled cursor-not-allowed' + )} + item={meltItem} + > + {#if subItem.icon} + + {/if} +

+ {subItem.displayName} +

+ {@render subItem.extra?.()} + {#if subItem.tooltip} + + {#snippet text()} + {subItem.tooltip} + {/snippet} + + {/if} + + {/each} +
+{/if} diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index baea52c9cf..493dec8f1f 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -71,6 +71,7 @@ const { elements: { menu: menuEl, item, trigger }, + builders, states, ids: { menu: dropdownId } } = createDropdownMenu({ @@ -177,7 +178,7 @@ class="bg-surface-tertiary dark:border w-56 origin-top-right rounded-lg shadow-lg focus:outline-none overflow-y-auto py-1" style={`${customWidth ? `width: ${customWidth}px;` : ''} max-height: ${maxHeight || '50vh'};`} > - + {/if} diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index ebb12077d1..35fedd660e 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -1,5 +1,6 @@ - + { newFolder?.closeDrawer() folderCreated = undefined }} > - {#if !folderCreated} -
- - -
- {:else} + {#if folderCreated} {/if} - +
- + { - currentTarget.select() - }} - /> - + {#if !hideFullPath} +
+
+ + Full path + + { + currentTarget.select() + }} + /> + +
+
{error}
-
{error}
-
+ {/if} {#if pathUsageInFlowsPromise || pathUsageInAppsPromise || pathUsageInScriptsPromise} {#await Promise.all( [pathUsageInAppsPromise, pathUsageInFlowsPromise, pathUsageInScriptsPromise] )} diff --git a/frontend/src/lib/components/SummaryPathDisplay.svelte b/frontend/src/lib/components/SummaryPathDisplay.svelte new file mode 100644 index 0000000000..ed2387167e --- /dev/null +++ b/frontend/src/lib/components/SummaryPathDisplay.svelte @@ -0,0 +1,151 @@ + + +{#if editable || onEdit} + + {#snippet trigger()} +
+ {path} + + {emptyString(summary) ? 'Add a summary...' : summary} + +
+ {/snippet} + {#snippet content({ close })} +
+ {#if onEdit} + +
+
Path
+ +
+ + {:else} + +
+
Path
+ +
+ {/if} +
+ {/snippet} + +{:else} +
+ {#if !emptyString(summary)} + {path} + {/if} + + {emptyString(summary) ? (path ?? '') : summary} + +
+{/if} diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 8e6bf79584..07f2cd8b1f 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -32,7 +32,12 @@ import Section from './Section.svelte' import Label from './Label.svelte' import Toggle from './Toggle.svelte' - import { cleanWorkerGroupConfig, defaultTags, nativeTags, type AutoscalingConfig } from './worker_group' + import { + cleanWorkerGroupConfig, + defaultTags, + nativeTags, + type AutoscalingConfig + } from './worker_group' import AutoscalingConfigEditor from './AutoscalingConfigEditor.svelte' import TagsToListenTo from './TagsToListenTo.svelte' import Select from './select/Select.svelte' @@ -87,6 +92,7 @@ pip_local_dependencies?: string[] min_alive_workers_alert_threshold?: number autoscaling?: AutoscalingConfig + native_mode?: boolean } = $state({}) function loadNConfig() { @@ -191,6 +197,7 @@ autoscaling?: AutoscalingConfig periodic_script_bash?: string periodic_script_interval_seconds?: number + native_mode?: boolean } activeWorkers: number customTags: string[] | undefined @@ -257,7 +264,8 @@ // Compute hashed tags for display (actual tags used by the worker) let hashedDedicatedTags: Map = $state(new Map()) $effect(() => { - const dws = config?.dedicated_workers ?? (config?.dedicated_worker ? [config.dedicated_worker] : []) + const dws = + config?.dedicated_workers ?? (config?.dedicated_worker ? [config.dedicated_worker] : []) if (dws.length > 0) { Promise.all(dws.map(async (dw) => [dw, await computeHashedTag(dw)] as const)).then( (entries) => { @@ -276,6 +284,14 @@ ? 'dedicated' : 'normal' ) + let isNativeMode = $derived( + config?.native_mode === true || + name === 'native' || + (workers.length > 0 && + workers.some(([_, pings]) => pings.some((p) => p.native_mode === true))) + ) + let nonNativeTags = $derived((nconfig?.worker_tags ?? []).filter((t) => !nativeTags.includes(t))) + let isAutoNativeMode = $derived(name === 'native') $effect(() => { ;($superadmin || $devopsRole) && listWorkspaces() }) @@ -497,7 +513,11 @@ {#if nconfig !== undefined}
-