From 5f0ef936d1d5d07d01c8e07e26ec254feebef8fb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Mar 2026 07:19:51 +0100 Subject: [PATCH] feat: add sandbox annotations, volume mounts, for AI sandbox starting with claude (#8058) --- Dockerfile | 6 + ...d55e0379cc84188954037165cbe2d198ef71f.json | 16 + ...5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json} | 7 +- ...c6e9b159a54c179cecb108068597536835f7e.json | 22 + ...6c595401dd2e1f7699a28bf3b79db5e3841f4.json | 23 + ...28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json | 23 + ...11a1dfb1422187200119268b2342b47a960c6.json | 25 + ...ef31e6d647377d37747f7bdc834748a59419e.json | 15 + ...6fc8ce983f857339e6fccf799dc6587964aab.json | 17 + ...485957990eb92fa3ce515895eab0d3f28bfdc.json | 25 + ...580b4c7ba25f279644e3233b63f4f6db0ad98.json | 16 + ...2eb169329116cba57780fa90ecf2bdb910f34.json | 18 + ...367b27171894016c714e41497e69115be1468.json | 76 +++ ...0be7046cf8789d842717b6c793c22a2a05daa.json | 23 + ...67cd897dca4722e4f2076308afdb7ee9fc147.json | 29 + ...8bca8450d09982e582266d215dff521256fa6.json | 22 + ...e919cd25538c0b433bc29bb052c7a7b8568ca.json | 18 + ...e142006825a3911addacdf1a026660b5e2b7f.json | 16 + ...03b566fc75e16e0b7a81204816fd50b3346a5.json | 24 + ...9ba8973a2ef1defa36c6d46d9c1c6406a7c33.json | 16 + ...eea76a8a3079ce80c035087f797cdc410f35b.json | 17 + ...d75ca7014b431239ac1b681f2b26380c719c4.json | 23 + ...050ba7b7e4d9eabba03be251ae9a8017b317d.json | 16 + ...53fd6e9c7606231ad3c522e98cb1fcc14361a.json | 23 + ...425e5f7600e28c728a06235f7ff430a4bd77a.json | 23 + ...9fc2c73e855790e17abf5461b96ea30fbbdb7.json | 23 + ...03b2601630f2d31458bdaf70c2702b2998d89.json | 16 + ...156ada29ae320465d9790dce7e1e8a436d4de.json | 15 + ...2550919fcac56deaf1b3cb36b3e15117936e7.json | 29 + ...263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json | 24 + ...6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json | 17 + ...e78697414b35618c50f8693f4e804bf1d7dbb.json | 29 + ...070c9f117da3dd00ce3247c54a61052a6809c.json | 47 ++ ...8d6affad03a1f388024806a5de3f9cc939c04.json | 23 + ...f33554794444f0dc4e2d2fec158eca5ebe865.json | 28 + ...511b9d3183cb91b740a86944a77a2a964b57d.json | 17 + ...706d78a6f24cb0e614d7d81ba1b643805bf06.json | 22 + ...359b064baf73b890efdc25426261d4eadfee0.json | 23 + ...c5a30037abef63efb5b44dc535c5f45d62a06.json | 41 ++ backend/Cargo.lock | 22 + backend/Cargo.toml | 5 + backend/ee-repo-ref.txt | 2 +- .../20260226000000_add_volumes.down.sql | 1 + .../20260226000000_add_volumes.up.sql | 22 + .../windmill-parser/src/asset_parser.rs | 2 + backend/tests/agent_workers.rs | 327 ++++++++- .../tests/scripts/test_volume_with_claude.ts | 102 +++ backend/tests/volume_tests.rs | 637 ++++++++++++++++++ backend/windmill-api-agent-workers/src/lib.rs | 8 + .../windmill-api-groups/src/granular_acls.rs | 45 +- .../windmill-api-workspaces/src/workspaces.rs | 2 + backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/openapi.yaml | 121 +++- backend/windmill-api/src/health.rs | 38 +- backend/windmill-api/src/job_helpers_oss.rs | 10 +- backend/windmill-api/src/lib.rs | 80 ++- .../windmill-api/src/triggers/http/handler.rs | 6 +- backend/windmill-api/src/volumes_oss.rs | 17 + backend/windmill-common/src/assets.rs | 1 + backend/windmill-common/src/auth.rs | 45 ++ backend/windmill-common/src/worker.rs | 99 +++ backend/windmill-native-triggers/src/lib.rs | 2 +- backend/windmill-test-utils/src/lib.rs | 11 +- backend/windmill-types/src/assets.rs | 1 + backend/windmill-worker-volumes/Cargo.toml | 29 + backend/windmill-worker-volumes/src/lib.rs | 544 +++++++++++++++ .../windmill-worker-volumes/src/volume_oss.rs | 116 ++++ backend/windmill-worker/Cargo.toml | 5 +- .../nsjail/run.bun.config.proto | 12 + backend/windmill-worker/src/bun_executor.rs | 16 +- backend/windmill-worker/src/common.rs | 8 +- backend/windmill-worker/src/deno_executor.rs | 10 +- backend/windmill-worker/src/lib.rs | 3 + .../windmill-worker/src/python_executor.rs | 34 +- backend/windmill-worker/src/volume_oss.rs | 112 +++ backend/windmill-worker/src/worker.rs | 161 ++++- docker/DockerfileSlim | 5 + docker/DockerfileSlimEe | 5 + .../src/lib/components/ApiConnectForm.svelte | 7 + frontend/src/lib/components/EditorBar.svelte | 2 +- .../lib/components/ExploreAssetButton.svelte | 15 +- .../src/lib/components/FilesetEditor.svelte | 27 + .../src/lib/components/ResourceEditor.svelte | 22 +- .../src/lib/components/S3FilePicker.svelte | 3 + .../lib/components/S3FilePickerInner.svelte | 26 +- .../src/lib/components/ScriptBuilder.svelte | 26 +- .../src/lib/components/ScriptEditor.svelte | 2 +- frontend/src/lib/components/ShareModal.svelte | 9 +- .../components/assets/JobAssetsViewer.svelte | 12 +- .../assets/VolumeDetailDrawer.svelte | 107 +++ .../components/assets/VolumesDrawer.svelte | 193 ++++++ frontend/src/lib/components/assets/lib.ts | 4 + .../common/languageIcons/LanguageIcon.svelte | 25 +- .../lib/components/copilot/ResourceGen.svelte | 186 +++++ .../flows/content/FlowInputs.svelte | 18 + .../flows/content/FlowInputsQuick.svelte | 21 + .../components/flows/flowStateUtils.svelte.ts | 2 +- .../flows/map/InsertModuleInner.svelte | 64 +- .../flows/pickers/FlowScriptPicker.svelte | 5 +- .../pickers/FlowScriptPickerQuick.svelte | 5 +- .../flows/pickers/TopLevelNode.svelte | 2 + .../components/graph/graphBuilder.svelte.ts | 2 +- .../components/icons/AssetGenericIcon.svelte | 4 +- .../lib/components/icons/ClaudeIcon.svelte | 25 + .../lib/components/raw_apps/fileTreeUtils.ts | 6 +- frontend/src/lib/components/script_builder.ts | 2 +- .../workspaceSettings/StorageSettings.svelte | 60 +- frontend/src/lib/infer.ts | 67 +- frontend/src/lib/script_helpers.ts | 11 +- .../lib/templates/claude_sandbox.ts.template | 85 +++ frontend/src/lib/workspace_settings.ts | 35 +- .../(root)/(logged)/assets/+page.svelte | 52 +- .../(logged)/workspace_settings/+page.svelte | 6 +- openflow.openapi.yaml | 1 + 114 files changed, 4537 insertions(+), 162 deletions(-) create mode 100644 backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json rename backend/.sqlx/{query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json => query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json} (51%) create mode 100644 backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json create mode 100644 backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json create mode 100644 backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json create mode 100644 backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json create mode 100644 backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json create mode 100644 backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json create mode 100644 backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json create mode 100644 backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json create mode 100644 backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json create mode 100644 backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json create mode 100644 backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json create mode 100644 backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json create mode 100644 backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json create mode 100644 backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json create mode 100644 backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json create mode 100644 backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json create mode 100644 backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json create mode 100644 backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json create mode 100644 backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json create mode 100644 backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json create mode 100644 backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json create mode 100644 backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json create mode 100644 backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json create mode 100644 backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json create mode 100644 backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json create mode 100644 backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json create mode 100644 backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json create mode 100644 backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json create mode 100644 backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json create mode 100644 backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json create mode 100644 backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json create mode 100644 backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json create mode 100644 backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json create mode 100644 backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json create mode 100644 backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json create mode 100644 backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json create mode 100644 backend/migrations/20260226000000_add_volumes.down.sql create mode 100644 backend/migrations/20260226000000_add_volumes.up.sql create mode 100644 backend/tests/scripts/test_volume_with_claude.ts create mode 100644 backend/tests/volume_tests.rs create mode 100644 backend/windmill-api/src/volumes_oss.rs create mode 100644 backend/windmill-worker-volumes/Cargo.toml create mode 100644 backend/windmill-worker-volumes/src/lib.rs create mode 100644 backend/windmill-worker-volumes/src/volume_oss.rs create mode 100644 backend/windmill-worker/src/volume_oss.rs create mode 100644 frontend/src/lib/components/assets/VolumeDetailDrawer.svelte create mode 100644 frontend/src/lib/components/assets/VolumesDrawer.svelte create mode 100644 frontend/src/lib/components/copilot/ResourceGen.svelte create mode 100644 frontend/src/lib/components/icons/ClaudeIcon.svelte create mode 100644 frontend/src/lib/templates/claude_sandbox.ts.template diff --git a/Dockerfile b/Dockerfile index 7cca6ab329..0cc19801d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -262,6 +262,12 @@ COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun RUN bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# The installer puts the binary in ~/.local/bin/claude (symlink to ~/.local/share/claude/versions/*) +# Copy it to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json new file mode 100644 index 0000000000..d9d9793cd1 --- /dev/null +++ b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f" +} diff --git a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json similarity index 51% rename from backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json rename to backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json index 84b10ccba6..409faa032f 100644 --- a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json +++ b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT token\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "query": "SELECT group_ FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "token", + "name": "group_", "type_info": "Varchar" } ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -18,5 +19,5 @@ false ] }, - "hash": "90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492" + "hash": "015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11" } diff --git a/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json new file mode 100644 index 0000000000..52fb375962 --- /dev/null +++ b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage->>'volume_storage' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e" +} diff --git a/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json new file mode 100644 index 0000000000..6a6b77e650 --- /dev/null +++ b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4" +} diff --git a/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json new file mode 100644 index 0000000000..0140324406 --- /dev/null +++ b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750" +} diff --git a/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json new file mode 100644 index 0000000000..9a6ae60a49 --- /dev/null +++ b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6" +} diff --git a/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json new file mode 100644 index 0000000000..9514010409 --- /dev/null +++ b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e" +} diff --git a/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json new file mode 100644 index 0000000000..fa67a0797d --- /dev/null +++ b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)\n VALUES ($1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET size_bytes = $3, last_used_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab" +} diff --git a/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json new file mode 100644 index 0000000000..8897e8a7de --- /dev/null +++ b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc" +} diff --git a/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json new file mode 100644 index 0000000000..2010b40667 --- /dev/null +++ b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98" +} diff --git a/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json new file mode 100644 index 0000000000..2fdb1ae80d --- /dev/null +++ b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34" +} diff --git a/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json new file mode 100644 index 0000000000..c73c00c2aa --- /dev/null +++ b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n size_bytes as \"size_bytes!\",\n file_count as \"file_count!\",\n created_at as \"created_at!\",\n created_by as \"created_by!\",\n updated_at,\n updated_by,\n description as \"description!\",\n last_used_at,\n extra_perms as \"extra_perms!\"\n FROM (\n SELECT\n COALESCE(v.name, a.path) as name,\n COALESCE(v.size_bytes, 0) as size_bytes,\n COALESCE(v.file_count, 0) as file_count,\n COALESCE(v.created_at, a.min_created_at) as created_at,\n COALESCE(v.created_by, 'unknown') as created_by,\n v.updated_at,\n v.updated_by,\n COALESCE(v.description, '') as description,\n v.last_used_at,\n COALESCE(v.extra_perms, '{}'::jsonb) as extra_perms\n FROM (\n SELECT path, MIN(created_at) as min_created_at\n FROM asset\n WHERE workspace_id = $1 AND kind = 'volume'\n GROUP BY path\n ) a\n FULL OUTER JOIN volume v ON v.workspace_id = $1 AND v.name = a.path\n WHERE v.workspace_id = $1 OR a.path IS NOT NULL\n ) combined\n ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "file_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "updated_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "description!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "extra_perms!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + true, + true, + null, + true, + null + ] + }, + "hash": "40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468" +} diff --git a/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json new file mode 100644 index 0000000000..2eda021880 --- /dev/null +++ b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2\n AND (lease_until IS NULL OR lease_until < now())\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa" +} diff --git a/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json new file mode 100644 index 0000000000..dd4a011ee1 --- /dev/null +++ b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147" +} diff --git a/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json new file mode 100644 index 0000000000..746c306c8f --- /dev/null +++ b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6" +} diff --git a/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json new file mode 100644 index 0000000000..3df9c6c195 --- /dev/null +++ b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca" +} diff --git a/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json new file mode 100644 index 0000000000..0bf454028b --- /dev/null +++ b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f" +} diff --git a/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json new file mode 100644 index 0000000000..7b85bd9315 --- /dev/null +++ b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5" +} diff --git a/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json new file mode 100644 index 0000000000..8cbe7146b6 --- /dev/null +++ b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33" +} diff --git a/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json new file mode 100644 index 0000000000..594fcf2960 --- /dev/null +++ b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b" +} diff --git a/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json new file mode 100644 index 0000000000..c60239fbce --- /dev/null +++ b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4" +} diff --git a/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json new file mode 100644 index 0000000000..b364fcd4ac --- /dev/null +++ b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = extra_perms - $1\n WHERE workspace_id = $2 AND name = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d" +} diff --git a/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json new file mode 100644 index 0000000000..728141923a --- /dev/null +++ b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now()", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a" +} diff --git a/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json new file mode 100644 index 0000000000..26c254eb0b --- /dev/null +++ b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a" +} diff --git a/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json new file mode 100644 index 0000000000..ee64e97d12 --- /dev/null +++ b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7" +} diff --git a/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json new file mode 100644 index 0000000000..fc33fd7373 --- /dev/null +++ b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89" +} diff --git a/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json new file mode 100644 index 0000000000..86a99d2eaf --- /dev/null +++ b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de" +} diff --git a/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json new file mode 100644 index 0000000000..9051a88a50 --- /dev/null +++ b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms, created_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7" +} diff --git a/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json new file mode 100644 index 0000000000..1f28a0a5a7 --- /dev/null +++ b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, 0, $3)\n ON CONFLICT (workspace_id, name) DO NOTHING\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6" +} diff --git a/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json new file mode 100644 index 0000000000..3c7c1ad52a --- /dev/null +++ b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935" +} diff --git a/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json new file mode 100644 index 0000000000..442d55ed25 --- /dev/null +++ b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb" +} diff --git a/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json new file mode 100644 index 0000000000..0400e6992d --- /dev/null +++ b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, name, size_bytes, created_by, last_used_at\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c" +} diff --git a/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json new file mode 100644 index 0000000000..16fc965f6a --- /dev/null +++ b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04" +} diff --git a/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json new file mode 100644 index 0000000000..91d89df9cd --- /dev/null +++ b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865" +} diff --git a/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json new file mode 100644 index 0000000000..c4a1d4cd07 --- /dev/null +++ b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)\n WHERE workspace_id = $3 AND name = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d" +} diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json new file mode 100644 index 0000000000..c96961eac4 --- /dev/null +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT token as \"token!\"\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" +} diff --git a/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json new file mode 100644 index 0000000000..649ee4c387 --- /dev/null +++ b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0" +} diff --git a/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json new file mode 100644 index 0000000000..e1ca416938 --- /dev/null +++ b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, file_count, leased_by, lease_until\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_count", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "leased_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "lease_until", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c7bba49b17..7dbc9ee58a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15798,7 +15798,9 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-test-utils", + "windmill-types", "windmill-worker", + "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", ] @@ -15952,6 +15954,7 @@ dependencies = [ "windmill-trigger-websocket", "windmill-types", "windmill-worker", + "windmill-worker-volumes", ] [[package]] @@ -17499,9 +17502,28 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-types", + "windmill-worker-volumes", "yaml-rust", ] +[[package]] +name = "windmill-worker-volumes" +version = "1.650.0" +dependencies = [ + "bytes", + "futures", + "lazy_static", + "md-5 0.10.6", + "object_store", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "windmill-common", +] + [[package]] name = "windows" version = "0.56.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9e8694fbd0..a417b66266 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -70,6 +70,7 @@ members = [ "./parsers/windmill-parser-py-imports", "./parsers/windmill-sql-datatype-parser-wasm", "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", + "./windmill-worker-volumes", "./windmill-test-utils", "./windmill-api-integration-tests", ] @@ -250,6 +251,8 @@ reqwest.workspace = true windmill-queue = { workspace = true, features = ["failpoints"] } windmill-dep-map.workspace = true windmill-test-utils.workspace = true +windmill-worker-volumes.workspace = true +windmill-types.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true @@ -267,6 +270,7 @@ aws-credential-types.workspace = true windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-worker-volumes = { path = "./windmill-worker-volumes" } windmill-dep-map = { path = "./windmill-dep-map" } windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } @@ -439,6 +443,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ "macros", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a71cab586d..0e4b8a1791 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6fd5a2ce908235a17975ad4dbdf0051cd89334f3 +151bc3edfe23c160f4f9b0cfaa708beb36c212f4 \ No newline at end of file diff --git a/backend/migrations/20260226000000_add_volumes.down.sql b/backend/migrations/20260226000000_add_volumes.down.sql new file mode 100644 index 0000000000..33dc3804be --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS volume; diff --git a/backend/migrations/20260226000000_add_volumes.up.sql b/backend/migrations/20260226000000_add_volumes.up.sql new file mode 100644 index 0000000000..00f40c9768 --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.up.sql @@ -0,0 +1,22 @@ +-- Add 'volume' to the asset_kind enum +ALTER TYPE asset_kind ADD VALUE IF NOT EXISTS 'volume'; + +-- Volume metadata table +CREATE TABLE volume ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + file_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(255) NOT NULL, + updated_at TIMESTAMPTZ, + updated_by VARCHAR(255), + description TEXT NOT NULL DEFAULT '', + lease_until TIMESTAMPTZ, + leased_by VARCHAR(255), + last_used_at TIMESTAMPTZ, + extra_perms JSONB NOT NULL DEFAULT '{}', + PRIMARY KEY (workspace_id, name) +); + +CREATE INDEX idx_volume_last_used ON volume(workspace_id, last_used_at); diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 2463995606..44388690e4 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -18,6 +18,7 @@ pub enum AssetKind { Resource, Ducklake, DataTable, + Volume, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -148,4 +149,5 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("$res:", AssetKind::Resource), ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), + ("volume://", AssetKind::Volume), ]; diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index b6414dc24e..4f12d5f42c 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -1,12 +1,12 @@ #![cfg(all(feature = "private", feature = "agent_worker_server"))] -use windmill_test_utils::*; use serde_json::json; use sqlx::{Pool, Postgres}; use windmill_common::{ jobs::{JobPayload, RawCode}, scripts::ScriptLang, }; +use windmill_test_utils::*; fn bun_code(code: &str) -> RawCode { RawCode { @@ -18,8 +18,8 @@ fn bun_code(code: &str) -> RawCode { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: - windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), } } @@ -223,7 +223,10 @@ async fn test_agent_worker_token_and_ping(db: Pool) -> anyhow::Result< .fetch_one(&db) .await?; - assert!(worker_count > 0, "worker ping should be recorded in database"); + assert!( + worker_count > 0, + "worker ping should be recorded in database" + ); // MainLoop ping updates the existing record let resp = http_client @@ -265,3 +268,319 @@ async fn test_agent_worker_multiple_jobs_sequential(db: Pool) -> anyho Ok(()) } + +/// Test the volume HTTP proxy endpoints that agent workers use. +/// +/// Exercises the full volume lifecycle via HTTP: +/// 1. Configure workspace S3 storage (FilesystemStorage) +/// 2. Pre-populate a volume with a file +/// 3. POST /begin — acquire lease, get manifest +/// 4. GET /file/* — download existing file +/// 5. PUT /file/* — upload a new file +/// 6. POST /commit — finalize with stats, release lease +/// 7. Verify DB state and storage +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // 3. POST /begin — acquire lease, get manifest + permissions + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-1", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "begin should succeed, got: {}", + resp.status() + ); + + let begin_body: serde_json::Value = resp.json().await?; + assert!( + begin_body["writable"].as_bool().unwrap(), + "should be writable" + ); + let manifest = begin_body["manifest"].as_object().unwrap(); + assert!( + manifest.contains_key("hello.txt"), + "manifest should contain hello.txt, got: {manifest:?}" + ); + + // 4. GET /file/* — download the existing file + let resp = http + .get(format!("{vol_base}/file/hello.txt")) + .send() + .await?; + assert!( + resp.status().is_success(), + "file download should succeed, got: {}", + resp.status() + ); + let file_bytes = resp.bytes().await?; + assert_eq!( + file_bytes.as_ref(), + b"hello from volume", + "downloaded file content should match" + ); + + // 5. PUT /file/* — upload a new file + let resp = http + .put(format!("{vol_base}/file/output.txt")) + .body(b"written by agent worker".to_vec()) + .send() + .await?; + assert!( + resp.status().is_success(), + "file upload should succeed, got: {}", + resp.status() + ); + + // 6. POST /commit — finalize: report stats, release lease + let resp = http + .post(format!("{vol_base}/commit")) + .json(&json!({ + "worker_name": "test-worker-1", + "deleted_keys": [], + "symlinks": {}, + "file_count": 2, + "size_bytes": 39 + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "commit should succeed, got: {}", + resp.status() + ); + + // 7. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert_eq!(vol_row.file_count, 2, "file_count should be 2"); + assert_eq!(vol_row.size_bytes, 39, "size_bytes should match"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 8. Verify the uploaded file was persisted in storage + let output_path = vol_dir.join("output.txt"); + assert!(output_path.exists(), "output.txt should be in storage"); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Full E2E test: agent worker in HTTP mode runs a Bun script with a volume mount. +/// +/// The worker pulls the job via HTTP, downloads volume files via the server-side +/// volume proxy endpoints, executes the script, and syncs changes back. +#[cfg(all(feature = "parquet", feature = "enterprise"))] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow::Result<()> { + let (_client, port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job, then run worker with HTTP connection (bun tag) + let code = r#"// volume: test-vol /tmp/data +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by agent worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let uuid = RunJob::from(JobPayload::Code(bun_code(code))) + .push(&db) + .await; + let listener = listen_for_completed_jobs(&db).await; + + let conn = testing_http_connection_with_tags( + port, + vec!["bun".into(), "flow".into(), "dependency".into()], + ) + .await; + + in_test_worker(conn, listener.find(&uuid), port).await; + + let result = completed_job(uuid, &db).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 5. Verify the new file was written back to the storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Test the volume release endpoint (error/cancel path). +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // Set up filesystem storage + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // Begin (acquire lease) + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-2", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!(resp.status().is_success(), "begin should succeed"); + + // Verify lease is held + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert_eq!(leased.as_deref(), Some("test-worker-2")); + + // Release without commit (simulating error path) + let resp = http + .post(format!("{vol_base}/release")) + .json(&json!({ "worker_name": "test-worker-2" })) + .send() + .await?; + assert!(resp.status().is_success(), "release should succeed"); + + // Verify lease is cleared + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert!(leased.is_none(), "lease should be released"); + + Ok(()) +} diff --git a/backend/tests/scripts/test_volume_with_claude.ts b/backend/tests/scripts/test_volume_with_claude.ts new file mode 100644 index 0000000000..b6230a19ee --- /dev/null +++ b/backend/tests/scripts/test_volume_with_claude.ts @@ -0,0 +1,102 @@ +// volume: agent-memory .claude +// sandbox + +import Anthropic from "@anthropic-ai/sdk"; +import * as fs from "fs"; +import * as path from "path"; + +type Anthropic = { + api_key: string; + model?: string; +}; + +export async function main(anthropic_resource: Anthropic) { + const claudeDir = ".claude"; + const results: Record = {}; + + // --- Step 1: Verify volume is mounted at the relative path --- + results["volume_exists"] = fs.existsSync(claudeDir); + if (!results["volume_exists"]) { + fs.mkdirSync(claudeDir, { recursive: true }); + } + + const testFile = path.join(claudeDir, "mount-check.txt"); + fs.writeFileSync(testFile, "volume mount verified"); + results["volume_writable"] = fs.readFileSync(testFile, "utf-8") === "volume mount verified"; + + // --- Step 2: Create memory directory structure --- + const memoryDir = path.join(claudeDir, "memory"); + fs.mkdirSync(memoryDir, { recursive: true }); + + const memoryFile = path.join(memoryDir, "MEMORY.md"); + fs.writeFileSync(memoryFile, "# Agent Memory\n\nThis file persists across runs.\n"); + results["memory_file_created"] = fs.existsSync(memoryFile); + + // --- Step 3: Call Claude to generate structured content --- + const client = new Anthropic({ apiKey: anthropic_resource.api_key }); + const model = anthropic_resource.model ?? "claude-sonnet-4-20250514"; + + const response = await client.messages.create({ + model, + max_tokens: 256, + messages: [ + { + role: "user", + content: + 'Return a JSON object with exactly these keys: "greeting" (a short hello), "timestamp" (current ISO date you estimate), "items" (array of 3 random fruit names). Only return the JSON, no markdown.', + }, + ], + }); + + const assistantText = + response.content[0].type === "text" ? response.content[0].text : ""; + results["claude_responded"] = assistantText.length > 0; + results["claude_model"] = response.model; + results["claude_stop_reason"] = response.stop_reason; + + let parsed: Record = {}; + try { + parsed = JSON.parse(assistantText); + results["claude_valid_json"] = true; + results["claude_has_greeting"] = "greeting" in parsed; + results["claude_has_items"] = + Array.isArray(parsed.items) && parsed.items.length === 3; + } catch { + results["claude_valid_json"] = false; + } + + // --- Step 4: Write Claude's response to volume --- + const responsePath = path.join(claudeDir, "claude-response.json"); + fs.writeFileSync(responsePath, JSON.stringify(parsed, null, 2)); + results["response_written"] = fs.existsSync(responsePath); + + // --- Step 5: Read back and verify --- + const readBack = fs.readFileSync(responsePath, "utf-8"); + const readParsed = JSON.parse(readBack); + results["readback_matches"] = + JSON.stringify(readParsed) === JSON.stringify(parsed); + + // --- Step 6: List all volume contents --- + const volumeContents = fs.readdirSync(claudeDir); + results["volume_files"] = volumeContents; + results["volume_file_count"] = volumeContents.length; + + // --- Step 7: Verify memory file persists --- + const memoryContent = fs.readFileSync(memoryFile, "utf-8"); + results["memory_persisted"] = memoryContent.includes("Agent Memory"); + + // --- Summary --- + const allChecks = [ + results["volume_exists"] || true, + results["volume_writable"], + results["claude_responded"], + results["claude_valid_json"], + results["response_written"], + results["readback_matches"], + results["memory_file_created"], + results["memory_persisted"], + ]; + results["all_passed"] = allChecks.every(Boolean); + + return results; +} diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs new file mode 100644 index 0000000000..9df30c5615 --- /dev/null +++ b/backend/tests/volume_tests.rs @@ -0,0 +1,637 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_volume_insert(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "test-volume", + 1024_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT workspace_id, name, size_bytes, created_by, last_used_at + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-volume" + ) + .fetch_one(&db) + .await?; + + assert_eq!(row.workspace_id, "test-workspace"); + assert_eq!(row.name, "test-volume"); + assert_eq!(row.size_bytes, 1024); + assert_eq!(row.created_by, "test-user"); + assert!(row.last_used_at.is_none()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_upsert_size(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 500_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 500); + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 2048_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 2048); + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_last_used(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "used-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_none()); + + sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_nonexistent_noop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "nonexistent-vol" + ) + .execute(&db) + .await?; + + assert_eq!(result.rows_affected(), 0); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_list_multiple(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + for i in 0..5 { + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + format!("vol-{}", i), + (i * 100) as i64, + "test-user" + ) + .execute(&db) + .await?; + } + + let rows = sqlx::query!( + "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "test-workspace" + ) + .fetch_all(&db) + .await?; + + assert_eq!(rows.len(), 5); + assert_eq!(rows[0].name, "vol-0"); + assert_eq!(rows[0].size_bytes, 0); + assert_eq!(rows[4].name, "vol-4"); + assert_eq!(rows[4].size_bytes, 400); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_delete(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "deleteme", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(1)); + + sqlx::query!( + "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_workspace_fk_constraint(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "nonexistent-workspace", + "vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("foreign key"), + "Expected foreign key violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_primary_key_uniqueness(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 200_i64, + "another-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("duplicate key") || err.contains("unique"), + "Expected unique violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_extra_perms(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert volume with default (empty) extra_perms + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "perms-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + // Default extra_perms should be empty object + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + // Set extra_perms via jsonb_set (same pattern as granular_acls.rs) + sqlx::query!( + "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true) + WHERE workspace_id = $3 AND name = $4", + &vec!["u/alice".to_string()], + true, + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + let perms = row.extra_perms.as_object().unwrap(); + assert_eq!(perms.get("u/alice").and_then(|v| v.as_bool()), Some(true)); + + // Remove a permission entry + sqlx::query!( + "UPDATE volume SET extra_perms = extra_perms - $1 + WHERE workspace_id = $2 AND name = $3", + "u/alice", + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + Ok(()) +} + +#[test] +fn test_parse_volume_annotations_python() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"# sandbox +# volume: training-data /tmp/training +# volume: models /opt/models + +def main(): + pass +"#; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 2); + assert_eq!(volumes[0].name, "training-data"); + assert_eq!(volumes[0].target, "/tmp/training"); + assert_eq!(volumes[1].name, "models"); + assert_eq!(volumes[1].target, "/opt/models"); +} + +#[test] +fn test_parse_volume_annotations_typescript() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"// sandbox +// volume: datasets /tmp/datasets + +export async function main() { + return "hello"; +} +"#; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "datasets"); + assert_eq!(volumes[0].target, "/tmp/datasets"); +} + +#[test] +fn test_parse_volume_annotations_no_prefix_match() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "def main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_parse_volume_annotations_empty_script() { + use windmill_worker_volumes::parse_volume_annotations; + + let volumes = parse_volume_annotations("", "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_sandbox_annotation_python() { + use windmill_common::worker::PythonAnnotations; + + let content = "# sandbox\n# volume: data /tmp/data\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_sandbox_annotation_typescript() { + use windmill_common::worker::TypeScriptAnnotations; + + let content = "// sandbox\n// volume: data /tmp/data\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_volume_comment_prefix_selection() { + use windmill_common::scripts::ScriptLang; + + let get_prefix = |lang: &ScriptLang| -> &str { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + } + }; + + assert_eq!(get_prefix(&ScriptLang::Python3), "#"); + assert_eq!(get_prefix(&ScriptLang::Bash), "#"); + assert_eq!(get_prefix(&ScriptLang::Powershell), "#"); + assert_eq!(get_prefix(&ScriptLang::Ansible), "#"); + assert_eq!(get_prefix(&ScriptLang::Ruby), "#"); + assert_eq!(get_prefix(&ScriptLang::Deno), "//"); + assert_eq!(get_prefix(&ScriptLang::Bun), "//"); + assert_eq!(get_prefix(&ScriptLang::Bunnative), "//"); + assert_eq!(get_prefix(&ScriptLang::Nativets), "//"); + assert_eq!(get_prefix(&ScriptLang::Go), "//"); +} + +#[test] +fn test_volume_mount_struct() { + use windmill_worker_volumes::VolumeMount; + + let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() }; + assert_eq!(mount.name, "test-vol"); + assert_eq!(mount.target, "/mnt/data"); +} + +#[test] +fn test_parse_volume_relative_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "agent-memory"); + assert_eq!(volumes[0].target, ".claude"); +} + +#[test] +fn test_parse_volume_relative_nested_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "# volume: data data/models\ndef main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "data"); + assert_eq!(volumes[0].target, "data/models"); +} + +#[cfg(feature = "private")] +#[test] +fn test_volume_nsjail_mount() { + use std::path::Path; + use windmill_worker_volumes::volume_nsjail_mount; + + let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data"); + assert!(result.contains("src: \"/tmp/volumes/data\"")); + assert!(result.contains("dst: \"/mnt/data\"")); + assert!(result.contains("is_bind: true")); + assert!(result.contains("rw: true")); +} + +#[test] +fn test_sync_stats_default() { + use windmill_worker_volumes::SyncStats; + + let stats = SyncStats { new_size_bytes: 0, file_count: 0, uploaded: 0, skipped: 0 }; + assert_eq!(stats.new_size_bytes, 0); + assert_eq!(stats.file_count, 0); + assert_eq!(stats.uploaded, 0); + assert_eq!(stats.skipped, 0); +} + +#[test] +fn test_asset_kind_volume_variant() { + use windmill_types::assets::AssetKind; + + let kind = AssetKind::Volume; + let serialized = serde_json::to_string(&kind).unwrap(); + assert_eq!(serialized, "\"volume\""); + + let deserialized: AssetKind = serde_json::from_str("\"volume\"").unwrap(); + assert!(matches!(deserialized, AssetKind::Volume)); +} + +/// E2E test: run a bun script with volume mount through a SQL-connected worker. +/// Pre-populates the volume in filesystem storage, verifies the script can read +/// files and write new ones, then checks sync-back to storage and DB state. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null, + "volume_storage": "primary" + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file (workspace-namespaced path) + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job and run with SQL-connected worker + let code = r#"// volume: test-vol /tmp/data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let job = JobPayload::Code(RawCode { + hash: None, + content: code.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + + // 5. Verify the new file was written back to storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by sql worker"); + + Ok(()) +} diff --git a/backend/windmill-api-agent-workers/src/lib.rs b/backend/windmill-api-agent-workers/src/lib.rs index d2913a217c..b02a7098b7 100644 --- a/backend/windmill-api-agent-workers/src/lib.rs +++ b/backend/windmill-api-agent-workers/src/lib.rs @@ -51,4 +51,12 @@ impl AgentCache { pub fn new() -> Self { AgentCache {} } + + pub async fn extract_worker_name( + &self, + _token: &str, + _db: &windmill_common::DB, + ) -> Option { + None + } } diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 048a4120f5..da9267419d 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -24,7 +24,7 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 18] = [ +const KINDS: [&str; 19] = [ "script", "group_", "resource", @@ -43,6 +43,7 @@ const KINDS: [&str; 18] = [ "gcp_trigger", "sqs_trigger", "email_trigger", + "volume", ]; pub fn workspaced_service() -> Router { @@ -77,7 +78,7 @@ async fn add_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -89,6 +90,22 @@ async fn add_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -243,6 +260,22 @@ async fn remove_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -250,7 +283,7 @@ async fn remove_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -380,7 +413,11 @@ async fn get_granular_acls( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" { "name" } else { "path" }; + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { + "name" + } else { + "path" + }; let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" )) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 80e497ba97..a85383e496 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -302,6 +302,8 @@ struct LargeFileStorageWithSecondary { large_file_storage: LargeFileStorage, #[serde(default)] secondary_storage: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + volume_storage: Option, } #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 7af8137630..de8935d097 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -70,6 +70,7 @@ windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } windmill-worker = { workspace = true, optional = true } +windmill-worker-volumes.workspace = true windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4e24fd6a4b..86d6084980 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -15198,6 +15198,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] responses: "200": @@ -15243,6 +15244,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -15299,6 +15301,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -17282,7 +17285,90 @@ paths: path: type: string description: The asset path - + + + /w/{workspace}/volumes/list: + get: + summary: List all volumes in the workspace + operationId: listVolumes + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of volumes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Volume" + + /w/{workspace}/volumes/storage: + get: + summary: Get the volume storage name (secondary storage) or null for primary + operationId: getVolumeStorage + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: volume storage name or null + content: + application/json: + schema: + type: string + nullable: true + + /w/{workspace}/volumes/create: + post: + summary: Create a new volume + operationId: createVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: string + responses: + "200": + description: volume created + content: + text/plain: + schema: + type: string + + /w/{workspace}/volumes/delete/{name}: + delete: + summary: Delete a volume (admin only) + operationId: deleteVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: volume deleted + content: + text/plain: + schema: + type: string /mcp/w/{workspace}/list_tools: get: @@ -23997,6 +24083,7 @@ components: - resource - ducklake - datatable + - volume Asset: type: object properties: @@ -24005,6 +24092,38 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + Volume: + type: object + required: + - name + - size_bytes + - file_count + - created_at + - created_by + properties: + name: + type: string + size_bytes: + type: integer + format: int64 + file_count: + type: integer + created_at: + type: string + format: date-time + created_by: + type: string + updated_at: + type: string + format: date-time + nullable: true + last_used_at: + type: string + format: date-time + nullable: true + extra_perms: + type: object + additionalProperties: true ProtectionRuleset: type: object description: A workspace protection rule defining restrictions and bypass permissions diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index d861f338ef..60bd290f66 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -240,11 +240,7 @@ async fn check_database_detailed(db: &DB) -> DatabaseHealth { let check = check_database_with_latency(db).await; let pool = get_pool_stats(db); - DatabaseHealth { - healthy: check.healthy, - latency_ms: check.latency_ms, - pool, - } + DatabaseHealth { healthy: check.healthy, latency_ms: check.latency_ms, pool } } async fn check_worker_count(db: &DB) -> i64 { @@ -295,13 +291,7 @@ async fn check_workers_detailed(db: &DB) -> WorkersHealth { let healthy = active_count > 0; - WorkersHealth { - healthy, - active_count, - worker_groups, - min_version, - versions, - } + WorkersHealth { healthy, active_count, worker_groups, min_version, versions } } async fn check_queue(db: &DB) -> QueueHealth { @@ -333,10 +323,7 @@ fn get_version() -> String { /// Spawn a background task that performs a health check every 10 seconds. /// Updates the cache and prometheus metrics continuously. -pub fn start_health_check_loop( - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { +pub fn start_health_check_loop(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -550,10 +537,7 @@ async fn health_status( } /// Detailed health check - requires DB authentication (always fresh, no caching) -async fn health_detailed( - _authed: ApiAuthed, - Extension(db): Extension, -) -> impl IntoResponse { +async fn health_detailed(_authed: ApiAuthed, Extension(db): Extension) -> impl IntoResponse { let checked_at = Utc::now(); let database = check_database_detailed(&db).await; let readiness = check_readiness(); @@ -564,12 +548,7 @@ async fn health_detailed( status: HealthStatus::Unhealthy, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: None, - queue: None, - readiness, - }, + checks: HealthChecks { database, workers: None, queue: None, readiness }, }; return (StatusCode::SERVICE_UNAVAILABLE, Json(response)); } @@ -587,12 +566,7 @@ async fn health_detailed( status, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: Some(workers), - queue: Some(queue), - readiness, - }, + checks: HealthChecks { database, workers: Some(workers), queue: Some(queue), readiness }, }; let status_code = if status == HealthStatus::Unhealthy { diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 01f5d67bd3..23f20459c3 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -12,15 +12,15 @@ use windmill_types::s3::StorageResourceType; #[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, OptJobAuthed, DB}; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; -#[cfg(not(feature = "private"))] -use windmill_object_store::ObjectStoreResource; -#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(all(feature = "parquet", not(feature = "private")))] +use windmill_common::db::UserDB; #[cfg(not(feature = "private"))] use windmill_common::error; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_common::db::UserDB; +use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; +#[cfg(not(feature = "private"))] +use windmill_object_store::ObjectStoreResource; #[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 528e60b02f..073c1a8aa1 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -170,6 +170,9 @@ pub mod users_ee; mod users_oss; mod utils; mod variables; +#[cfg(feature = "private")] +pub mod volumes_ee; +mod volumes_oss; pub mod webhook_util; mod workspaces; #[cfg(feature = "private")] @@ -248,6 +251,74 @@ type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; +/// Worker name derived from the agent JWT token, used to authenticate volume operations. +/// Defined unconditionally so volume endpoint handlers can reference it regardless of +/// whether agent_worker_server is enabled (the extension is only populated on the agent path). +#[derive(Clone)] +pub struct AgentWorkerName(pub String); + +/// Middleware that injects a synthetic `ApiAuthed` and JWT-derived worker name +/// into request extensions. +/// +/// Used for volume proxy endpoints under the agent_workers path, where the +/// agent JWT auth layer has already validated the request. The volume handlers +/// need `ApiAuthed` to resolve the workspace S3 client, but the agent JWT +/// format is incompatible with the standard auth extractor. +/// +/// The worker name is extracted from the JWT claims rather than trusting +/// self-reported values in request bodies/query params. +#[cfg(feature = "agent_worker_server")] +async fn inject_agent_authed( + request: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut request = request; + + // Extract worker name from agent JWT via AgentCache + // (OSS returns None; EE decodes the JWT and returns the worker name) + { + let extracted = { + let token = request + .headers() + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ").map(|t| t.to_string())); + let cache = request.extensions().get::>().cloned(); + let db = request.extensions().get::().cloned(); + match (token, cache, db) { + (Some(token), Some(cache), Some(db)) => Some((token, cache, db)), + _ => None, + } + }; + + if let Some((token, cache, db)) = extracted { + if let Some(worker_name) = cache.extract_worker_name(&token, &db).await { + request + .extensions_mut() + .insert(AgentWorkerName(worker_name)); + } + } + } + + request + .extensions_mut() + .insert(windmill_api_auth::OptJobAuthed { + authed: ApiAuthed { + email: "agent-worker@windmill.dev".to_string(), + username: "agent-worker".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }, + job_id: None, + }); + next.run(request).await +} + pub async fn run_server( db: DB, job_index_reader: Option, @@ -513,6 +584,7 @@ pub async fn run_server( users::workspaced_service().layer(Extension(argon2.clone())), ) .nest("/variables", variables::workspaced_service()) + .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) @@ -626,7 +698,13 @@ pub async fn run_server( .nest("/w/:workspace_id/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_router.layer(Extension(agent_cache.clone())) + agent_workers_router + .nest( + "/volumes", + volumes_oss::agent_workspaced_service() + .layer(axum::middleware::from_fn(inject_agent_authed)), + ) + .layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index f3ec05348a..9d95ead840 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -369,7 +369,11 @@ async fn route_job( let s3_object = s3_client.get(&path).await; let s3_object = match s3_object { - Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => { + Err( + windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { + .. + }, + ) if trigger.is_static_website => { // fallback to index.html if the file is not found let path = windmill_object_store::object_store_reexports::Path::from(format!( "{}/index.html", diff --git a/backend/windmill-api/src/volumes_oss.rs b/backend/windmill-api/src/volumes_oss.rs new file mode 100644 index 0000000000..26b1c2cd46 --- /dev/null +++ b/backend/windmill-api/src/volumes_oss.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::volumes_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +#[allow(dead_code)] +pub fn agent_workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 669614d64a..2968493ecc 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -72,6 +72,7 @@ pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetK windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, + windmill_parser::asset_parser::AssetKind::Volume => AssetKind::Volume, } } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index ccd9b5d2a0..c0555f5093 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -95,6 +95,51 @@ impl PermsCache { } } +/// Check a user's access level against an `extra_perms` JSONB object. +/// +/// Returns `None` if the user has no matching entry (no access). +/// Returns `Some(true)` if the user (or any of their groups) has write access. +/// Returns `Some(false)` if the user (or any of their groups) has read-only access. +pub fn check_extra_perms( + extra_perms: &serde_json::Map, + username: &str, + groups: &[String], +) -> Option { + // Check direct user permission + let user_key = if username.starts_with("u/") { + username.to_string() + } else { + format!("u/{username}") + }; + if let Some(v) = extra_perms.get(&user_key) { + return Some(v.as_bool().unwrap_or(false)); + } + + // Check group permissions — return highest access level found + let mut found = false; + let mut write = false; + for g in groups { + let key = if g.starts_with("g/") { + g.to_string() + } else { + format!("g/{g}") + }; + if let Some(v) = extra_perms.get(&key) { + found = true; + if v.as_bool().unwrap_or(false) { + write = true; + break; + } + } + } + + if found { + Some(write) + } else { + None + } +} + pub fn has_expired(expiration_time: DateTime, take: Option) -> bool { let now = Utc::now(); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 3195562368..b3876fbcfa 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -354,6 +354,45 @@ impl HttpClient { ))) } } + + pub async fn get_bytes(&self, url: &str) -> anyhow::Result { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .get(format!("{}{}", base_url, url)) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(response.bytes().await?) + } else { + Err(anyhow::anyhow!( + "HTTP agent request GET {} failed {}", + url, + response.status() + )) + } + } + + pub async fn put_bytes(&self, url: &str, bytes: Bytes) -> anyhow::Result<()> { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .put(format!("{}{}", base_url, url)) + .body(bytes) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "HTTP agent request PUT {} failed {}", + url, + response.status() + )) + } + } } #[derive(Clone)] @@ -698,6 +737,7 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub sandbox: bool, } #[annotations("//")] @@ -711,6 +751,7 @@ pub struct TypeScriptAnnotations { pub nodejs: bool, pub native: bool, pub nobundling: bool, + pub sandbox: bool, } #[annotations("--")] @@ -2169,4 +2210,62 @@ mod tests { ); assert_ne!(a, b); } + + #[test] + fn test_python_sandbox_annotation() { + let content = "# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_python_sandbox_annotation_with_other_annotations() { + let content = "# no_cache\n# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.no_cache); + } + + #[test] + fn test_python_no_sandbox_annotation() { + let content = "# no_cache\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation() { + let content = "// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation_with_other_annotations() { + let content = "// npm\n// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.npm); + } + + #[test] + fn test_typescript_no_sandbox_annotation() { + let content = "// npm\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_python_sandbox_no_space() { + let content = "#sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_no_space() { + let content = "//sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } } diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4f27dff2e2..ed22480a9a 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -726,7 +726,7 @@ pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>> ) -> Result> { let token = sqlx::query_scalar!( r#" - SELECT token + SELECT token as "token!" FROM token WHERE token LIKE concat($1::text, '%') LIMIT 1 diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 68da452335..eff7475785 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -830,6 +830,15 @@ pub async fn run_preview_relative_imports( #[cfg(all(feature = "private", feature = "agent_worker_server"))] pub async fn testing_http_connection(port: u16) -> Connection { + testing_http_connection_with_tags( + port, + vec!["flow".into(), "python3".into(), "dependency".into()], + ) + .await +} + +#[cfg(all(feature = "private", feature = "agent_worker_server"))] +pub async fn testing_http_connection_with_tags(port: u16, tags: Vec) -> Connection { let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker"); let agent_token = format!( "{}{}", @@ -837,7 +846,7 @@ pub async fn testing_http_connection(port: u16) -> Connection { windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth { worker_group: "testing-agent".to_owned(), suffix: Some(suffix.clone()), - tags: vec!["flow".into(), "python3".into(), "dependency".into()], + tags, exp: Some(usize::MAX), }) .await diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index cf86d83801..be20cfae3e 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -13,6 +13,7 @@ pub enum AssetKind { Variable, // Deprecated Ducklake, DataTable, + Volume, } #[derive( diff --git a/backend/windmill-worker-volumes/Cargo.toml b/backend/windmill-worker-volumes/Cargo.toml new file mode 100644 index 0000000000..08b9a554b2 --- /dev/null +++ b/backend/windmill-worker-volumes/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "windmill-worker-volumes" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_worker_volumes" +path = "src/lib.rs" + +[features] +enterprise = [] +private = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +object_store.workspace = true +tokio.workspace = true +tracing.workspace = true +bytes.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +regex.workspace = true +lazy_static.workspace = true +md-5.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs new file mode 100644 index 0000000000..a9003a3712 --- /dev/null +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -0,0 +1,544 @@ +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub use volume_oss::*; + +pub use object_store::ObjectStore as DynObjectStore; + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +pub const MAX_VOLUMES_PER_JOB: usize = 10; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileEntry { + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub md5: Option, +} + +pub fn compute_md5_hex(data: &[u8]) -> String { + use md5::{Digest, Md5}; + let result = Md5::digest(data); + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hex = String::with_capacity(32); + for &b in result.iter() { + hex.push(HEX[(b >> 4) as usize] as char); + hex.push(HEX[(b & 0x0f) as usize] as char); + } + hex +} + +/// Extract an MD5 hash from an S3 ETag, if it's a simple (non-multipart) ETag. +pub fn etag_to_md5(e_tag: Option<&str>) -> Option { + let tag = e_tag?.trim_matches('"'); + // Multipart ETags contain a '-' (e.g. "abc123-5"), skip those + if tag.contains('-') || tag.is_empty() { + return None; + } + Some(tag.to_string()) +} + +lazy_static::lazy_static! { + static ref ARGS_INTERPOLATION_RE: regex::Regex = + regex::Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); + static ref VALID_VOLUME_NAME_RE: regex::Regex = + regex::Regex::new(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]$").unwrap(); +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VolumeMount { + pub name: String, + pub target: String, +} + +pub struct VolumeState { + pub mount: VolumeMount, + pub local_dir: PathBuf, + pub manifest: HashMap, + pub symlinks: HashMap, +} + +pub struct DownloadStats { + pub total_files: usize, + pub from_cache: usize, + pub downloaded: usize, +} + +pub struct SyncStats { + pub new_size_bytes: u64, + pub file_count: usize, + pub uploaded: usize, + pub skipped: usize, +} + +pub fn validate_volume_name(name: &str) -> Result<(), String> { + if name.contains("..") { + return Err(format!( + "Volume name '{}' contains '..' which is not allowed", + name + )); + } + if !VALID_VOLUME_NAME_RE.is_match(name) { + return Err(format!( + "Volume name '{}' is invalid. Names must be 2-255 characters, \ + start and end with alphanumeric, and contain only alphanumeric, '.', '_', or '-'", + name + )); + } + Ok(()) +} + +const ALLOWED_ABSOLUTE_PREFIXES: &[&str] = &["/tmp/", "/mnt/", "/opt/", "/home/", "/data/"]; + +pub fn validate_volume_target(target: &str) -> Result<(), String> { + if target.split('/').any(|seg| seg == "..") { + return Err(format!( + "Volume target '{target}' contains '..' segments which is not allowed" + )); + } + if target.starts_with('/') + && !ALLOWED_ABSOLUTE_PREFIXES + .iter() + .any(|p| target.starts_with(p)) + { + return Err(format!( + "Volume target '{target}' must be a relative path or start with one of: {}", + ALLOWED_ABSOLUTE_PREFIXES.join(", ") + )); + } + Ok(()) +} + +pub fn validate_volume_mounts(mounts: &[VolumeMount]) -> Result<(), String> { + if mounts.len() > MAX_VOLUMES_PER_JOB { + return Err(format!( + "Too many volume mounts ({}, max {})", + mounts.len(), + MAX_VOLUMES_PER_JOB + )); + } + let mut seen_names = HashSet::new(); + let mut seen_targets = HashSet::new(); + for v in mounts { + if !seen_names.insert(&v.name) { + return Err(format!("Duplicate volume name: '{}'", v.name)); + } + if !seen_targets.insert(&v.target) { + return Err(format!("Duplicate volume target: '{}'", v.target)); + } + } + Ok(()) +} + +pub fn interpolate_volume_name( + name: &str, + args: Option<&HashMap>>, + workspace_id: &str, +) -> String { + let name = name.replace("$workspace", workspace_id); + if !name.contains("$args[") { + return name; + } + let Some(args) = args else { + return name; + }; + let mut result = name.clone(); + for cap in ARGS_INTERPOLATION_RE.captures_iter(&name) { + let full_match = cap.get(0).unwrap().as_str(); + let arg_name = cap.get(1).unwrap().as_str(); + let arg_value = if arg_name.contains('.') { + let parts: Vec<&str> = arg_name.split('.').collect(); + let root = parts[0]; + let mut value = args + .get(root) + .map(|x| x.get().to_string()) + .unwrap_or_default(); + for part in parts.iter().skip(1) { + if let Ok(obj) = serde_json::from_str::(&value) { + value = obj + .get(part) + .map(|v| v.to_string()) + .unwrap_or_default() + .to_string(); + } else { + value = String::new(); + break; + } + } + value.trim_matches('"').to_string() + } else { + args.get(arg_name) + .map(|x| x.get().trim_matches('"').to_string()) + .unwrap_or_default() + }; + result = result.replace(full_match, &arg_value); + } + result +} + +pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if !trimmed.starts_with(comment_prefix) { + break; + } + let after_prefix = trimmed[comment_prefix.len()..].trim(); + if let Some(rest) = after_prefix.strip_prefix("volume:") { + let rest = rest.trim(); + let mut parts = rest.splitn(2, char::is_whitespace); + if let (Some(name), Some(target)) = (parts.next(), parts.next()) { + let name = name.trim(); + let target = target.trim(); + if !name.is_empty() && !target.is_empty() { + volumes + .push(VolumeMount { name: name.to_string(), target: target.to_string() }); + } + } + } + } + volumes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_single_volume() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_typescript_single_volume() { + let content = "// volume: mydata /tmp/data\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_multiple_volumes() { + let content = "# volume: data1 /tmp/data1\n# volume: data2 /tmp/data2\n# volume: models /opt/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() }, + ] + ); + } + + #[test] + fn parse_mixed_annotations_and_volumes() { + let content = "# sandbox\n# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_no_volumes() { + let content = "# sandbox\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_empty_content() { + let result = parse_volume_annotations("", "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_stops_at_non_comment_line() { + let content = + "# volume: data1 /tmp/data1\ndef main():\n # volume: data2 /tmp/data2\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }] + ); + } + + #[test] + fn parse_skips_blank_lines_in_header() { + let content = + "# volume: data1 /tmp/data1\n\n# volume: data2 /tmp/data2\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + ] + ); + } + + #[test] + fn parse_ignores_malformed_volume_lines() { + let content = + "# volume:\n# volume: onlyname\n# volume: good /tmp/good\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }] + ); + } + + #[test] + fn parse_extra_whitespace() { + let content = "# volume: mydata /tmp/data \ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_target_with_spaces_in_path() { + let content = "// volume: mydata /tmp/my data dir\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { + name: "mydata".to_string(), + target: "/tmp/my data dir".to_string(), + }] + ); + } + + #[test] + fn parse_volume_with_dashes_and_underscores() { + let content = "# volume: my-data_v2 /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn interpolate_workspace() { + let name = "$workspace-data"; + let result = interpolate_volume_name(name, None, "my_ws"); + assert_eq!(result, "my_ws-data"); + } + + #[test] + fn interpolate_args_simple() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"prod\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("data-$args[env]", Some(&args), "ws"); + assert_eq!(result, "data-prod"); + } + + #[test] + fn interpolate_args_and_workspace() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"staging\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("$workspace-$args[env]-cache", Some(&args), "acme"); + assert_eq!(result, "acme-staging-cache"); + } + + #[test] + fn interpolate_no_placeholders() { + let result = interpolate_volume_name("plain-name", None, "ws"); + assert_eq!(result, "plain-name"); + } + + #[test] + fn interpolate_missing_arg() { + let args = HashMap::new(); + let result = interpolate_volume_name("data-$args[missing]", Some(&args), "ws"); + assert_eq!(result, "data-"); + } + + #[test] + fn interpolate_nested_arg() { + let mut args = HashMap::new(); + args.insert( + "config".to_string(), + serde_json::value::RawValue::from_string( + r#"{"env": "prod", "region": "us-east"}"#.to_string(), + ) + .unwrap(), + ); + let result = interpolate_volume_name( + "data-$args[config.env]-$args[config.region]", + Some(&args), + "ws", + ); + assert_eq!(result, "data-prod-us-east"); + } + + #[test] + fn parse_wrong_prefix_returns_empty() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "//"); + assert!(result.is_empty()); + } + + #[test] + fn parse_relative_path() { + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }] + ); + } + + #[test] + fn parse_relative_nested_path() { + let content = "# volume: data data/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }] + ); + } + + #[test] + fn validate_valid_names() { + assert!(validate_volume_name("mydata").is_ok()); + assert!(validate_volume_name("my-data_v2").is_ok()); + assert!(validate_volume_name("acme-staging-cache").is_ok()); + assert!(validate_volume_name("a1").is_ok()); + assert!(validate_volume_name("data.v2").is_ok()); + assert!(validate_volume_name("A0").is_ok()); + } + + #[test] + fn validate_rejects_path_traversal() { + assert!(validate_volume_name("../other-workspace").is_err()); + assert!(validate_volume_name("data/../secrets").is_err()); + assert!(validate_volume_name("a..b").is_err()); + } + + #[test] + fn validate_rejects_special_start_end() { + assert!(validate_volume_name("-data").is_err()); + assert!(validate_volume_name("data-").is_err()); + assert!(validate_volume_name(".data").is_err()); + assert!(validate_volume_name("data.").is_err()); + assert!(validate_volume_name("_data").is_err()); + } + + #[test] + fn validate_rejects_path_separators() { + assert!(validate_volume_name("data/secrets").is_err()); + assert!(validate_volume_name("data\\secrets").is_err()); + } + + #[test] + fn validate_rejects_too_short() { + assert!(validate_volume_name("").is_err()); + assert!(validate_volume_name("a").is_err()); + } + + #[test] + fn validate_rejects_too_long() { + let long_name = format!("a{}a", "b".repeat(254)); + assert!(validate_volume_name(&long_name).is_err()); + } + + #[test] + fn validate_rejects_spaces_and_special() { + assert!(validate_volume_name("my data").is_err()); + assert!(validate_volume_name("my@data").is_err()); + assert!(validate_volume_name("my$data").is_err()); + } + + #[test] + fn validate_target_allows_relative() { + assert!(validate_volume_target("data").is_ok()); + assert!(validate_volume_target("data/models").is_ok()); + assert!(validate_volume_target(".claude").is_ok()); + } + + #[test] + fn validate_target_allows_safe_absolute() { + assert!(validate_volume_target("/tmp/data").is_ok()); + assert!(validate_volume_target("/mnt/data").is_ok()); + assert!(validate_volume_target("/opt/models").is_ok()); + assert!(validate_volume_target("/home/user/data").is_ok()); + assert!(validate_volume_target("/data/cache").is_ok()); + } + + #[test] + fn validate_target_rejects_dangerous_absolute() { + assert!(validate_volume_target("/etc/passwd").is_err()); + assert!(validate_volume_target("/proc/self").is_err()); + assert!(validate_volume_target("/sys/fs").is_err()); + assert!(validate_volume_target("/dev/null").is_err()); + assert!(validate_volume_target("/usr/bin").is_err()); + assert!(validate_volume_target("/var/log").is_err()); + } + + #[test] + fn validate_target_rejects_traversal() { + assert!(validate_volume_target("../../etc").is_err()); + assert!(validate_volume_target("data/../../../etc").is_err()); + assert!(validate_volume_target("/tmp/../etc/passwd").is_err()); + } + + #[test] + fn validate_mounts_rejects_too_many() { + let mounts: Vec = (0..11) + .map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) }) + .collect(); + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_name() { + let mounts = vec![ + VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_target() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_ok() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_ok()); + } +} diff --git a/backend/windmill-worker-volumes/src/volume_oss.rs b/backend/windmill-worker-volumes/src/volume_oss.rs new file mode 100644 index 0000000000..07cd6906d2 --- /dev/null +++ b/backend/windmill-worker-volumes/src/volume_oss.rs @@ -0,0 +1,116 @@ +#[cfg(feature = "private")] +pub use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +use crate::{DownloadStats, SyncStats, VolumeMount, VolumeState}; +#[cfg(not(feature = "private"))] +use object_store::ObjectStore; +#[cfg(not(feature = "private"))] +use std::path::Path; +#[cfg(not(feature = "private"))] +use std::sync::Arc; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn download_volume( + _client: Arc, + _volume: &VolumeMount, + _job_dir: &str, + _workspace_id: &str, +) -> error::Result<(VolumeState, DownloadStats)> { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String { + String::new() +} + +#[cfg(not(feature = "private"))] +pub async fn sync_volume_back( + _client: Arc, + _state: &VolumeState, + _workspace_id: &str, +) -> error::Result { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn walk_dir(dir: &Path) -> std::io::Result> { + let mut result = Vec::new(); + walk_dir_inner(dir, &mut result)?; + Ok(result) +} + +#[cfg(not(feature = "private"))] +fn walk_dir_inner(dir: &Path, result: &mut Vec) -> std::io::Result<()> { + if !dir.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.is_dir() { + walk_dir_inner(&path, result)?; + } else if meta.is_file() { + result.push(path); + } + } + Ok(()) +} + +#[cfg(not(feature = "private"))] +pub fn collect_symlinks(dir: &Path) -> std::collections::HashMap { + let mut symlinks = std::collections::HashMap::new(); + collect_symlinks_inner(dir, dir, &mut symlinks); + symlinks +} + +#[cfg(not(feature = "private"))] +fn collect_symlinks_inner( + base: &Path, + dir: &Path, + symlinks: &mut std::collections::HashMap, +) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.file_type().is_symlink() { + if let Ok(target) = std::fs::read_link(&path) { + let relative = path + .strip_prefix(base) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + symlinks.insert(relative, target.to_string_lossy().to_string()); + } + } else if meta.is_dir() { + collect_symlinks_inner(base, &path, symlinks); + } + } +} + +#[cfg(not(feature = "private"))] +pub fn restore_symlinks(_dir: &Path, _symlinks: &std::collections::HashMap) { + // No-op in OSS build +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5cf0a4c26e..23927753ee 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,10 +10,10 @@ path = "src/lib.rs" [features] default = [] -private = [] +private = ["windmill-worker-volumes/private", "windmill-queue/private"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth @@ -47,6 +47,7 @@ windmill-audit.workspace = true # there isn't really a reason for audit-worth ac windmill-common = { workspace = true, default-features = false } windmill-types.workspace = true windmill-object-store.workspace = true +windmill-worker-volumes.workspace = true windmill-jseval.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 8cbee3dec9..3ba8c73257 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -14,6 +14,18 @@ clone_newnet: false clone_newuser: {CLONE_NEWUSER} clone_newcgroup: false +uidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + +gidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + skip_setsid: true keep_caps: false keep_env: true diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 40f4b9f77e..7939dc77e1 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -22,7 +22,7 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, - NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ @@ -990,6 +990,14 @@ pub async fn handle_bun_job( ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(error::Error::ExecutionErr( + "Script has //sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the //sandbox annotation." + .to_string(), + )); + } + let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = ( maybe_lock.get_lock(), !annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(), @@ -1161,6 +1169,10 @@ pub async fn handle_bun_job( init_logs = format!("\n{}{}", cache_logs, init_logs); } + if annotation.sandbox { + init_logs.push_str("sandbox mode (nsjail)\n"); + } + let write_wrapper_f = async { if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; @@ -1485,7 +1497,7 @@ try {{ append_logs(&job.id, &job.workspace_id, init_logs, conn).await; //do not cache local dependencies - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotation.sandbox { let _ = write_file( job_dir, "run.config.proto", diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1239aa8d7d..846c421024 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -886,7 +886,7 @@ pub async fn cached_result_path( } #[cfg(feature = "parquet")] -async fn get_workspace_s3_resource_path( +pub(crate) async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, @@ -948,7 +948,11 @@ async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { - (StorageResourceType::Filesystem, fs.root_path.clone()) + return Ok(Some( + windmill_object_store::ObjectStoreResource::Filesystem( + windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, + ), + )); } None => { return Ok(None); diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 19831eadd0..6cbf85c47b 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -17,6 +17,7 @@ use crate::{ NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; +use windmill_common::worker::TypeScriptAnnotations; use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL}; @@ -231,8 +232,13 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + let annotations = TypeScriptAnnotations::parse(inner_content); + // let mut start = Instant::now(); - let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + let mut logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + if annotations.sandbox { + logs1.push_str("sandbox mode (nsjail)\n"); + } append_logs(&job.id, &job.workspace_id, logs1, conn).await; let main_override = job.script_entrypoint_override.as_deref(); @@ -451,7 +457,7 @@ try {{ for flag in deno_flags { args.push(flag); } - } else if is_sandboxing_enabled() { + } else if is_sandboxing_enabled() || annotations.sandbox { args.push("--allow-net"); args.push("--allow-sys"); args.push(allow_read.as_str()); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 741458c0e0..f6d752558d 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -70,6 +70,9 @@ mod sanitized_sql_params; mod schema; pub mod sql_utils; mod universal_pkg_installer; +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b289f05380..619087659d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -134,8 +134,8 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, UV_INDEX_STRATEGY, }; use windmill_common::client::AuthedClient; @@ -567,6 +567,14 @@ pub async fn handle_python_job( let annotations = PythonAnnotations::parse(inner_content); + if annotations.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr( + "Script has #sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the #sandbox annotation." + .to_string(), + )); + } + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -605,16 +613,14 @@ pub async fn handle_python_job( } { - append_logs( - &job.id, - &job.workspace_id, - format!( - "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.clone().to_string() - ), - conn, - ) - .await; + let mut logs = format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.clone().to_string() + ); + if annotations.sandbox { + logs.push_str("sandbox mode (nsjail)\n"); + } + append_logs(&job.id, &job.workspace_id, logs, conn).await; } let ( import_loader, @@ -784,7 +790,7 @@ except BaseException as e: #[cfg(windows)] let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); - if is_sandboxing_enabled() { + if is_sandboxing_enabled() || annotations.sandbox { let shared_deps = additional_python_paths .into_iter() .map(|pp| { @@ -828,7 +834,7 @@ mount {{ job.id ); - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotations.sandbox { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) diff --git a/backend/windmill-worker/src/volume_oss.rs b/backend/windmill-worker/src/volume_oss.rs new file mode 100644 index 0000000000..3736e83485 --- /dev/null +++ b/backend/windmill-worker/src/volume_oss.rs @@ -0,0 +1,112 @@ +#[cfg(feature = "private")] +pub(crate) use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct LeaseRenewalGuard(pub Option>); + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +impl Drop for LeaseRenewalGuard { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct VolumeSetupResult { + pub states: Vec, + pub writable: Vec, + pub client: Option>, + pub lease_renewal: LeaseRenewalGuard, +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +#[allow(dead_code)] +pub(crate) fn setup_volume_mount_paths( + _volume: &windmill_worker_volumes::VolumeMount, + _state: &windmill_worker_volumes::VolumeState, + _job_dir: &str, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result<()> { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_sql_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _worker_name: &str, + _job_dir: &str, + _client: &windmill_common::client::AuthedClient, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_http_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _canceled_by: &Option, + _worker_name: &str, + _job_dir: &str, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_sql_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _vol_client: &std::sync::Arc, + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_http_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1497d3ebb8..350af4dab6 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4161,7 +4161,8 @@ pub async fn run_language_executor( job.id ); - let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { + #[allow(unused_mut)] + let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { let folder = if job.script_lang == Some(ScriptLang::Go) { "/go" } else { @@ -4183,7 +4184,8 @@ mount {{ // println!("handle lang job {:?}", SystemTime::now()); - let envs = build_envs(envs.as_ref())?; + #[allow(unused_mut)] + let mut envs = build_envs(envs.as_ref())?; let Some(language) = language else { return Err(Error::ExecutionErr( @@ -4219,6 +4221,106 @@ mount {{ } } + // Volume mount setup (requires workspace S3 storage; CE has file count/size limits) + #[cfg(feature = "parquet")] + let volume_mounts = { + let comment_prefix = match language { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + }; + let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); + let args_ref = job.args.as_ref().map(|a| &**a); + let mut interpolated = Vec::new(); + for mut v in raw_mounts { + v.name = windmill_worker_volumes::interpolate_volume_name( + &v.name, + args_ref, + &job.workspace_id, + ); + if let Err(e) = windmill_worker_volumes::validate_volume_name(&v.name) { + return Err(Error::ExecutionErr(e)); + } + if let Err(e) = windmill_worker_volumes::validate_volume_target(&v.target) { + return Err(Error::ExecutionErr(e)); + } + interpolated.push(v); + } + if let Err(e) = windmill_worker_volumes::validate_volume_mounts(&interpolated) { + return Err(Error::ExecutionErr(e)); + } + interpolated + }; + + #[cfg(feature = "parquet")] + let mut volume_setup = crate::volume_oss::VolumeSetupResult { + states: Vec::new(), + writable: Vec::new(), + client: None, + lease_renewal: crate::volume_oss::LeaseRenewalGuard(None), + }; + + #[cfg(feature = "parquet")] + if !volume_mounts.is_empty() { + let vol_summary: Vec = volume_mounts + .iter() + .map(|v| format!("'{}' -> {}", v.name, v.target)) + .collect(); + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n--- VOLUME MOUNTS ---\nPulling {} volume(s): {}\n", + volume_mounts.len(), + vol_summary.join(", "), + ), + conn, + ) + .await; + + if let Connection::Sql(db) = conn { + volume_setup = crate::volume_oss::setup_volumes_sql_worker( + &volume_mounts, + db, + &job.workspace_id, + job.id, + &job.permissioned_as, + worker_name, + job_dir, + client, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } else if let Connection::Http(http) = conn { + volume_setup = crate::volume_oss::setup_volumes_http_worker( + &volume_mounts, + http, + &job.workspace_id, + job.id, + &job.permissioned_as, + &job.canceled_by, + worker_name, + job_dir, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } + } + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { ScriptLang::Python3 => { @@ -4630,6 +4732,61 @@ mount {{ // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; + // Volume sync-back and lease release + #[cfg(feature = "parquet")] + if !volume_setup.states.is_empty() { + // Stop lease renewal before sync-back + volume_setup.lease_renewal.0.take().map(|h| h.abort()); + + if let Some(ref vol_client) = volume_setup.client { + if let Connection::Sql(db) = conn { + crate::volume_oss::sync_volumes_sql_worker( + &volume_setup.states, + &volume_setup.writable, + vol_client, + db, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + } + + if let Connection::Http(http) = conn { + crate::volume_oss::sync_volumes_http_worker( + &volume_setup.states, + &volume_setup.writable, + http, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + + // Clean up absolute-path symlinks created by setup_volume_mount_paths + if !is_sandboxing_enabled() { + for state in &volume_setup.states { + #[cfg(unix)] + if state.mount.target.starts_with('/') { + let target_path = std::path::Path::new(&state.mount.target); + if target_path + .symlink_metadata() + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) + { + std::fs::remove_file(target_path).ok(); + } + } + } + } + } + tracing::info!( workspace_id = %job.workspace_id, is_ok = result.is_ok(), diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 450dd68399..279c79b780 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7cc4dafa05..88c16aaac0 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index d8429a5ebf..8a79bd7109 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -15,6 +15,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { resourceType: string @@ -149,6 +150,12 @@ }} class="as-json-toggle" /> + {#if resourceType == 'postgresql'} - {#if asset.kind === 's3object'} + {#if asset.kind === 's3object' || asset.kind === 'volume'} Explore {:else if asset.kind === 'resource' || asset.kind === 'ducklake' || asset.kind === 'datatable'} Manage diff --git a/frontend/src/lib/components/FilesetEditor.svelte b/frontend/src/lib/components/FilesetEditor.svelte index 41559d0d8a..39a67f4ad6 100644 --- a/frontend/src/lib/components/FilesetEditor.svelte +++ b/frontend/src/lib/components/FilesetEditor.svelte @@ -46,6 +46,9 @@ } } + // Track the last args we wrote so we can detect external changes. + let lastWrittenArgs: Record = $state(args ?? {}) + // Sync files → args, overlaying current editContent for the active file. // This avoids spreading a new files object on every keystroke. $effect(() => { @@ -58,9 +61,33 @@ newArgs[argKey] = key === currentKey ? currentContent : value } } + lastWrittenArgs = newArgs args = newArgs }) + // Sync args → files when args changes externally (e.g. from AI generation). + $effect(() => { + const currentArgs = args + if (currentArgs === lastWrittenArgs) return + // Check if the args object is actually different + const currentKeys = Object.keys(currentArgs ?? {}).sort().join('\0') + const lastKeys = Object.keys(lastWrittenArgs ?? {}).sort().join('\0') + if (currentKeys === lastKeys) { + const allSame = Object.entries(currentArgs ?? {}).every( + ([k, v]) => lastWrittenArgs[k] === v + ) + if (allSame) return + } + const newFiles = Object.fromEntries( + Object.entries(currentArgs ?? {}).map(([k, v]) => ['/' + k, String(v ?? '')]) + ) + files = newFiles + lastWrittenArgs = currentArgs + const firstFile = Object.keys(newFiles).find((k) => !k.endsWith('/')) + selectedPath = firstFile ?? '/' + editContent = firstFile ? (newFiles[firstFile] ?? '') : '' + }) + function inferLang(filePath: string): string { const ext = filePath.split('.').pop()?.toLowerCase() if (!ext) return 'plaintext' diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 5a5943390e..0ffed949d1 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -24,6 +24,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import Button from './common/button/Button.svelte' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { canSave?: boolean @@ -270,6 +271,13 @@ right: 'As JSON' }} /> + {#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'} {:else} @@ -296,9 +304,17 @@ {#if loadingSchema} {:else if !viewJsonSchema && resourceTypeInfo?.is_fileset} -
- Fileset -
+
+
Fileset
+ +
{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties} {#if resourceTypeInfo?.format_extension} diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index a07ea85d25..55bb25c5ef 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -13,6 +13,7 @@ interface Props { fromWorkspaceSettings?: boolean readOnlyMode: boolean + allowDelete?: boolean initialFileKey?: { s3: string; storage?: string } | undefined selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean @@ -24,6 +25,7 @@ let { fromWorkspaceSettings = false, readOnlyMode, + allowDelete = false, initialFileKey = $bindable(undefined), selectedFileKey = $bindable(undefined), folderOnly = false, @@ -94,6 +96,7 @@ }} {fromWorkspaceSettings} {readOnlyMode} + {allowDelete} bind:initialFileKey bind:selectedFileKey bind:workspaceSettingsInitialized diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 380e3d9a35..74b85bd327 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -81,6 +81,7 @@ count: number } > + allowDelete?: boolean replaceUnauthorizedWarning?: Snippet listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise @@ -102,11 +103,12 @@ folderOnly = false, regexFilter = undefined, hideS3SpecificDetails = false, - rootPath = '', + rootPath: initialRootPath = '', workspaceSettingsInitialized = $bindable(true), storage = $bindable(undefined), uploadModalOpen = $bindable(false), allFilesByKey = $bindable({}), + allowDelete = false, replaceUnauthorizedWarning, listStoredFilesRequest = HelpersService.listStoredFiles, loadFilePreviewRequest = HelpersService.loadFilePreview, @@ -116,6 +118,7 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() + let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) let csvSeparatorChar: string = $state(',') @@ -263,7 +266,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() fileListLoading = false fileInfoLoading = false } @@ -381,7 +384,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) { @@ -424,9 +427,16 @@ export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey) storage = preSelectedFileKey?.storage - if (preSelectedFileKey !== undefined) { + if (preSelectedFileKey !== undefined && preSelectedFileKey.s3.endsWith('/')) { + rootPath = preSelectedFileKey.s3 + filter = '' + selectedFileKey = undefined + } else if (preSelectedFileKey !== undefined) { + rootPath = '' initialFileKey = { ...preSelectedFileKey } selectedFileKey = { ...preSelectedFileKey } + } else { + rootPath = '' } reloadContent() } @@ -461,7 +471,7 @@ if (selectedFileKey !== undefined) { if (allFilesByKey[selectedFileKey.s3] === undefined) { selectedFileKey = { s3: '', storage } - } else { + } else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') { loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } } @@ -518,7 +528,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } else { selectedFileKey = { s3: item_key, @@ -719,8 +729,10 @@ startIcon={{ icon: MoveRight }} iconOnly={true} /> + {/if} + {#if !readOnlyMode || allowDelete} + {#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
{#snippet header()} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 5fe03b03fd..0f1e1d8a6e 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -100,7 +100,7 @@ path: string | undefined lang: Preview['language'] kind?: string | undefined - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' + template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' tag: string | undefined initialArgs?: Record fixedOverflowWidgets?: boolean diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index e5ad54f88b..c8c69106ed 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -35,6 +35,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'email_trigger' + | 'volume' let kind: Kind let path: string = $state('') @@ -53,13 +54,17 @@ let drawer: Drawer | undefined = $state() let own = $state(false) - export async function openDrawer(newPath: string, kind_l: Kind) { + export async function openDrawer(newPath: string, kind_l: Kind, isOwnerOverride?: boolean) { path = newPath kind = kind_l loadAcls() loadGroups() loadUsernames() - loadOwner() + if (isOwnerOverride !== undefined) { + own = isOwnerOverride + } else { + loadOwner() + } drawer?.openDrawer() } diff --git a/frontend/src/lib/components/assets/JobAssetsViewer.svelte b/frontend/src/lib/components/assets/JobAssetsViewer.svelte index 86b9c7eca7..10e2f011f9 100644 --- a/frontend/src/lib/components/assets/JobAssetsViewer.svelte +++ b/frontend/src/lib/components/assets/JobAssetsViewer.svelte @@ -1,5 +1,5 @@ + + (open = false)}> + (open = false)}> + {#if loading} +
+ +
+ {:else if volume} +
+
+
+ Files + {volume.file_count} +
+
+ Size + {displaySize(volume.size_bytes) ?? '0 B'} +
+
+ Created at + {displayDate(volume.created_at)} +
+
+ Created by + {volume.created_by} +
+ {#if volume.last_used_at} +
+ Last used + {displayDate(volume.last_used_at)} +
+ {/if} +
+ +
+ {#if s3FilePicker} + + {/if} + {#if $userStore?.is_admin} + + {/if} +
+
+ {:else} +
+ Volume '{volumeName}' not found. +
+ {/if} + + diff --git a/frontend/src/lib/components/assets/VolumesDrawer.svelte b/frontend/src/lib/components/assets/VolumesDrawer.svelte new file mode 100644 index 0000000000..3d6d861b15 --- /dev/null +++ b/frontend/src/lib/components/assets/VolumesDrawer.svelte @@ -0,0 +1,193 @@ + + + (open = false)}> + (open = false)}> + {#snippet actions()} + + {#snippet trigger()} + + {/snippet} + {#snippet content({ close })} +
+ { + if (e.key === 'Enter' && newVolumeName.trim()) { + createVolume(newVolumeName.trim(), close) + } + } + }} + bind:value={newVolumeName} + /> + +
+ {/snippet} + + {/snippet} + {#if volumes.loading} +
+ +
+ {:else if !volumes.current?.length} +
+ No volumes yet. Create one above or they are auto-created when a job declares a volume annotation. +
+ {:else} +
+ {#each volumes.current as vol (vol.name)} + {@const readable = canReadVolume(vol.created_by, vol.extra_perms)} + {@const writable = canWriteVolume(vol.created_by, vol.extra_perms)} +
+ +
+
+ {vol.name} + } + canWrite={writable} + /> +
+ + {vol.file_count} {vol.file_count === 1 ? 'file' : 'files'} + · {displaySize(vol.size_bytes) ?? '0 B'} + · owner: {vol.created_by.replace(/^u\//, '')} + +
+ {#if vol.last_used_at} + + Used {displayDate(vol.last_used_at)} + + {/if} + {#if writable} + + {/if} + {#if writable} +
+ {/each} +
+ {/if} + + + + refreshKey++} /> diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 2dacedbae3..4dae4c20b1 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -26,6 +26,8 @@ export function formatAsset(asset: Asset): string { return `ducklake://${asset.path}` case 'datatable': return `datatable://${asset.path}` + case 'volume': + return `volume://${asset.path}` } return 'unknown' } @@ -89,6 +91,8 @@ export function formatAssetKind(asset: { return 'Ducklake' case 'datatable': return 'Data table' + case 'volume': + return 'Volume' } } diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index a380a5d753..6c7a052301 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -24,6 +24,7 @@ import JavaIcon from '$lib/components/icons/JavaIcon.svelte' import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte' import RubyIcon from '$lib/components/icons/RubyIcon.svelte' + import ClaudeIcon from '$lib/components/icons/ClaudeIcon.svelte' interface Props { lang: @@ -36,6 +37,7 @@ | 'docker' | 'powershell' | 'bunnative' + | 'claudesandbox' width?: number height?: number scale?: number @@ -45,7 +47,7 @@ let { lang, width = 30, height = 30, scale = 1, size = undefined, ...rest }: Props = $props() - const languageLabel: Record = { + const languageLabel: Record = { python3: 'Python', deno: 'TypeScript', go: 'Go', @@ -68,12 +70,13 @@ csharp: 'C#', nu: 'Nu', java: 'Java', - ruby: 'Ruby' + ruby: 'Ruby', + claudesandbox: 'Claude Sandbox' // for related places search: ADD_NEW_LANG } const langToComponent: Record< - SupportedLanguage | 'pgsql' | 'javascript' | 'fetch' | 'docker' | 'powershell' | 'bunnative', + SupportedLanguage | 'pgsql' | 'javascript' | 'fetch' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox', any > = { go: GoIcon, @@ -103,7 +106,8 @@ nu: NuIcon, java: JavaIcon, ruby: RubyIcon, - duckdb: DuckDbIcon + duckdb: DuckDbIcon, + claudesandbox: TypeScriptIcon // for related places search: ADD_NEW_LANG } @@ -142,4 +146,17 @@ /> {/if} + {#if lang === 'claudesandbox'} +
+ +
+ {/if} diff --git a/frontend/src/lib/components/copilot/ResourceGen.svelte b/frontend/src/lib/components/copilot/ResourceGen.svelte new file mode 100644 index 0000000000..aa18ca0546 --- /dev/null +++ b/frontend/src/lib/components/copilot/ResourceGen.svelte @@ -0,0 +1,186 @@ + + + + {#snippet trigger()} +