starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add sandbox annotations, volume mounts, for AI sandbox starting with claude (#8058)

Browse Source
This commit is contained in:
Ruben Fiszel
2026-03-05 07:19:51 +01:00
committed by GitHub
parent bee50b83d1
commit 5f0ef936d1
114 changed files with 4537 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+6
View File
@@ -262,6 +262,12 @@ COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun
RUN bun install -g windmill-cli \
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
# Install Claude Code CLI (used by claude sandbox scripts)
# The installer puts the binary in ~/.local/bin/claude (symlink to ~/.local/share/claude/versions/*)
# Copy it to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root)
RUN curl -fsSL https://claude.ai/install.sh | bash \
&& cp /root/.local/share/claude/versions/* /usr/bin/claude
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f"
}
backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json → backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json
Generated
+4 -3
View File
@@ -1,16 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT token\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ",
"query": "SELECT group_ FROM usr_to_group WHERE usr = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token",
"name": "group_",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
@@ -18,5 +19,5 @@
false
]
},
"hash": "90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492"
"hash": "015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11"
}
backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT large_file_storage->>'volume_storage' FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e"
}
backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "extra_perms",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4"
}
backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750"
}
backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6"
}
backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e"
}
backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)\n VALUES ($1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET size_bytes = $3, last_used_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"Varchar"
]
},
"nullable": []
},
"hash": "1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab"
}
backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc"
}
backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98"
}
backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Int8",
"Int4",
"Varchar"
]
},
"nullable": []
},
"hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34"
}
backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json
Generated
+76
View File
@@ -0,0 +1,76 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n name as \"name!\",\n size_bytes as \"size_bytes!\",\n file_count as \"file_count!\",\n created_at as \"created_at!\",\n created_by as \"created_by!\",\n updated_at,\n updated_by,\n description as \"description!\",\n last_used_at,\n extra_perms as \"extra_perms!\"\n FROM (\n SELECT\n COALESCE(v.name, a.path) as name,\n COALESCE(v.size_bytes, 0) as size_bytes,\n COALESCE(v.file_count, 0) as file_count,\n COALESCE(v.created_at, a.min_created_at) as created_at,\n COALESCE(v.created_by, 'unknown') as created_by,\n v.updated_at,\n v.updated_by,\n COALESCE(v.description, '') as description,\n v.last_used_at,\n COALESCE(v.extra_perms, '{}'::jsonb) as extra_perms\n FROM (\n SELECT path, MIN(created_at) as min_created_at\n FROM asset\n WHERE workspace_id = $1 AND kind = 'volume'\n GROUP BY path\n ) a\n FULL OUTER JOIN volume v ON v.workspace_id = $1 AND v.name = a.path\n WHERE v.workspace_id = $1 OR a.path IS NOT NULL\n ) combined\n ORDER BY name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "size_bytes!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "file_count!",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "created_at!",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "updated_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "updated_by",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "description!",
"type_info": "Text"
},
{
"ordinal": 8,
"name": "last_used_at",
"type_info": "Timestamptz"
},
{
"ordinal": 9,
"name": "extra_perms!",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
null,
null,
null,
null,
true,
true,
null,
true,
null
]
},
"hash": "40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468"
}
backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2\n AND (lease_until IS NULL OR lease_until < now())\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa"
}
backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by, extra_perms FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "extra_perms",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147"
}
backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) FROM volume WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6"
}
backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Int8",
"Int4",
"Text"
]
},
"nullable": []
},
"hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca"
}
backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f"
}
backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5"
}
backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33"
}
backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"Varchar"
]
},
"nullable": []
},
"hash": "7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b"
}
backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_used_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4"
}
backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET extra_perms = extra_perms - $1\n WHERE workspace_id = $2 AND name = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d"
}
backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now()",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "leased_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a"
}
backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a"
}
backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "size_bytes",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7"
}
backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89"
}
backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de"
}
backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT extra_perms, created_by FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7"
}
backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, 0, $3)\n ON CONFLICT (workspace_id, name) DO NOTHING\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6"
}
backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"Varchar"
]
},
"nullable": []
},
"hash": "ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935"
}
backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "size_bytes",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "last_used_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true
]
},
"hash": "bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb"
}
backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json
Generated
+47
View File
@@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, name, size_bytes, created_by, last_used_at\n FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "size_bytes",
"type_info": "Int8"
},
{
"ordinal": 3,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "last_used_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
true
]
},
"hash": "d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c"
}
backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "leased_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04"
}
backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "size_bytes",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865"
}
backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)\n WHERE workspace_id = $3 AND name = $4",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d"
}
backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT token as \"token!\"\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token!",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06"
}
backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT permissioned_as FROM v2_job WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "permissioned_as",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0"
}
backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json
Generated
+41
View File
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT size_bytes, file_count, leased_by, lease_until\n FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "size_bytes",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "file_count",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "leased_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "lease_until",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
true,
true
]
},
"hash": "f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06"
}
backend/Cargo.lock
Generated
+22
View File
@@ -15798,7 +15798,9 @@ dependencies = [
"windmill-queue",
"windmill-runtime-nativets",
"windmill-test-utils",
"windmill-types",
"windmill-worker",
"windmill-worker-volumes",
"windows-service",
"windows-sys 0.52.0",
]
@@ -15952,6 +15954,7 @@ dependencies = [
"windmill-trigger-websocket",
"windmill-types",
"windmill-worker",
"windmill-worker-volumes",
]
[[package]]
@@ -17499,9 +17502,28 @@ dependencies = [
"windmill-queue",
"windmill-runtime-nativets",
"windmill-types",
"windmill-worker-volumes",
"yaml-rust",
]
[[package]]
name = "windmill-worker-volumes"
version = "1.650.0"
dependencies = [
"bytes",
"futures",
"lazy_static",
"md-5 0.10.6",
"object_store",
"regex",
"serde",
"serde_json",
"tempfile",
"tokio",
"tracing",
"windmill-common",
]
[[package]]
name = "windows"
version = "0.56.0"
backend/Cargo.toml
+5
View File
@@ -70,6 +70,7 @@ members = [
"./parsers/windmill-parser-py-imports",
"./parsers/windmill-sql-datatype-parser-wasm",
"./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu",
"./windmill-worker-volumes",
"./windmill-test-utils",
"./windmill-api-integration-tests",
]
@@ -250,6 +251,8 @@ reqwest.workspace = true
windmill-queue = { workspace = true, features = ["failpoints"] }
windmill-dep-map.workspace = true
windmill-test-utils.workspace = true
windmill-worker-volumes.workspace = true
windmill-types.workspace = true
axum.workspace = true
serde.workspace = true
windmill-api-client.workspace = true
@@ -267,6 +270,7 @@ aws-credential-types.workspace = true
windmill-api = { path = "./windmill-api", default-features = false }
windmill-queue = { path = "./windmill-queue" }
windmill-worker = { path = "./windmill-worker" }
windmill-worker-volumes = { path = "./windmill-worker-volumes" }
windmill-dep-map = { path = "./windmill-dep-map" }
windmill-types = { path = "./windmill-types" }
windmill-common = { path = "./windmill-common", default-features = false }
@@ -439,6 +443,7 @@ base64 = "^0.22.1"
base32 = "^0"
hmac = "0.12.1"
sha2 = "0.10.6"
md-5 = "0.10.6"
sha1 = "0.10.6"
sqlx = { version = "0.8.0", features = [
"macros",
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
6fd5a2ce908235a17975ad4dbdf0051cd89334f3
151bc3edfe23c160f4f9b0cfaa708beb36c212f4
backend/migrations/20260226000000_add_volumes.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS volume;
backend/migrations/20260226000000_add_volumes.up.sql
+22
View File
@@ -0,0 +1,22 @@
-- Add 'volume' to the asset_kind enum
ALTER TYPE asset_kind ADD VALUE IF NOT EXISTS 'volume';
-- Volume metadata table
CREATE TABLE volume (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
name VARCHAR(255) NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
file_count INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
created_by VARCHAR(255) NOT NULL,
updated_at TIMESTAMPTZ,
updated_by VARCHAR(255),
description TEXT NOT NULL DEFAULT '',
lease_until TIMESTAMPTZ,
leased_by VARCHAR(255),
last_used_at TIMESTAMPTZ,
extra_perms JSONB NOT NULL DEFAULT '{}',
PRIMARY KEY (workspace_id, name)
);
CREATE INDEX idx_volume_last_used ON volume(workspace_id, last_used_at);
backend/parsers/windmill-parser/src/asset_parser.rs
+2
View File
@@ -18,6 +18,7 @@ pub enum AssetKind {
Resource,
Ducklake,
DataTable,
Volume,
}
#[derive(Serialize, Debug, PartialEq, Clone)]
@@ -148,4 +149,5 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[
("$res:", AssetKind::Resource),
("ducklake://", AssetKind::Ducklake),
("datatable://", AssetKind::DataTable),
("volume://", AssetKind::Volume),
];
backend/tests/agent_workers.rs
+323 -4
View File
@@ -1,12 +1,12 @@
#![cfg(all(feature = "private", feature = "agent_worker_server"))]
use windmill_test_utils::*;
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_common::{
jobs::{JobPayload, RawCode},
scripts::ScriptLang,
};
use windmill_test_utils::*;
fn bun_code(code: &str) -> RawCode {
RawCode {
@@ -18,8 +18,8 @@ fn bun_code(code: &str) -> RawCode {
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default().into(),
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
}
}
@@ -223,7 +223,10 @@ async fn test_agent_worker_token_and_ping(db: Pool<Postgres>) -> anyhow::Result<
.fetch_one(&db)
.await?;
assert!(worker_count > 0, "worker ping should be recorded in database");
assert!(
worker_count > 0,
"worker ping should be recorded in database"
);
// MainLoop ping updates the existing record
let resp = http_client
@@ -265,3 +268,319 @@ async fn test_agent_worker_multiple_jobs_sequential(db: Pool<Postgres>) -> anyho
Ok(())
}
/// Test the volume HTTP proxy endpoints that agent workers use.
///
/// Exercises the full volume lifecycle via HTTP:
/// 1. Configure workspace S3 storage (FilesystemStorage)
/// 2. Pre-populate a volume with a file
/// 3. POST /begin — acquire lease, get manifest
/// 4. GET /file/* — download existing file
/// 5. PUT /file/* — upload a new file
/// 6. POST /commit — finalize with stats, release lease
/// 7. Verify DB state and storage
#[cfg(feature = "parquet")]
#[sqlx::test(fixtures("base"))]
async fn test_agent_worker_volume_e2e(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _server) = init_client_agent_mode(db.clone()).await;
// 1. Set up filesystem-based object storage in a temp dir
let storage_dir = tempfile::tempdir()?;
let storage_root = storage_dir.path().to_string_lossy().to_string();
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_root,
"public_resource": null,
"advanced_permissions": null
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
lfs_config,
"test-workspace"
)
.execute(&db)
.await?;
// 2. Pre-populate the volume with a file
let vol_dir = storage_dir.path().join("volumes").join("test-vol");
std::fs::create_dir_all(&vol_dir)?;
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
let base = client.baseurl();
let http = client.client();
let vol_base = format!("{base}/w/test-workspace/volumes/test-vol");
// 3. POST /begin — acquire lease, get manifest + permissions
let resp = http
.post(format!("{vol_base}/begin"))
.json(&json!({
"worker_name": "test-worker-1",
"permissioned_as": "u/test-user"
}))
.send()
.await?;
assert!(
resp.status().is_success(),
"begin should succeed, got: {}",
resp.status()
);
let begin_body: serde_json::Value = resp.json().await?;
assert!(
begin_body["writable"].as_bool().unwrap(),
"should be writable"
);
let manifest = begin_body["manifest"].as_object().unwrap();
assert!(
manifest.contains_key("hello.txt"),
"manifest should contain hello.txt, got: {manifest:?}"
);
// 4. GET /file/* — download the existing file
let resp = http
.get(format!("{vol_base}/file/hello.txt"))
.send()
.await?;
assert!(
resp.status().is_success(),
"file download should succeed, got: {}",
resp.status()
);
let file_bytes = resp.bytes().await?;
assert_eq!(
file_bytes.as_ref(),
b"hello from volume",
"downloaded file content should match"
);
// 5. PUT /file/* — upload a new file
let resp = http
.put(format!("{vol_base}/file/output.txt"))
.body(b"written by agent worker".to_vec())
.send()
.await?;
assert!(
resp.status().is_success(),
"file upload should succeed, got: {}",
resp.status()
);
// 6. POST /commit — finalize: report stats, release lease
let resp = http
.post(format!("{vol_base}/commit"))
.json(&json!({
"worker_name": "test-worker-1",
"deleted_keys": [],
"symlinks": {},
"file_count": 2,
"size_bytes": 39
}))
.send()
.await?;
assert!(
resp.status().is_success(),
"commit should succeed, got: {}",
resp.status()
);
// 7. Verify volume DB row was updated
let vol_row = sqlx::query!(
"SELECT size_bytes, file_count, leased_by, lease_until
FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"test-vol"
)
.fetch_optional(&db)
.await?;
let vol_row = vol_row.expect("volume row should exist");
assert_eq!(vol_row.file_count, 2, "file_count should be 2");
assert_eq!(vol_row.size_bytes, 39, "size_bytes should match");
assert!(vol_row.leased_by.is_none(), "lease should be released");
assert!(
vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(),
"lease_until should be cleared or in the past"
);
// 8. Verify the uploaded file was persisted in storage
let output_path = vol_dir.join("output.txt");
assert!(output_path.exists(), "output.txt should be in storage");
let output_content = std::fs::read_to_string(&output_path)?;
assert_eq!(output_content, "written by agent worker");
Ok(())
}
/// Full E2E test: agent worker in HTTP mode runs a Bun script with a volume mount.
///
/// The worker pulls the job via HTTP, downloads volume files via the server-side
/// volume proxy endpoints, executes the script, and syncs changes back.
#[cfg(all(feature = "parquet", feature = "enterprise"))]
#[sqlx::test(fixtures("base"))]
async fn test_agent_worker_volume_http_worker_e2e(db: Pool<Postgres>) -> anyhow::Result<()> {
let (_client, port, _server) = init_client_agent_mode(db.clone()).await;
// 1. Set up filesystem-based object storage in a temp dir
let storage_dir = tempfile::tempdir()?;
let storage_root = storage_dir.path().to_string_lossy().to_string();
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_root,
"public_resource": null,
"advanced_permissions": null
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
lfs_config,
"test-workspace"
)
.execute(&db)
.await?;
// 2. Pre-populate the volume with a file
let vol_dir = storage_dir.path().join("volumes").join("test-vol");
std::fs::create_dir_all(&vol_dir)?;
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
// 3. Push the job, then run worker with HTTP connection (bun tag)
let code = r#"// volume: test-vol /tmp/data
import { readFileSync, writeFileSync, existsSync } from "fs";
export function main() {
const content = readFileSync("/tmp/data/hello.txt", "utf-8");
writeFileSync("/tmp/data/output.txt", "written by agent worker");
return {
read_content: content,
output_exists: existsSync("/tmp/data/output.txt"),
};
}"#;
let uuid = RunJob::from(JobPayload::Code(bun_code(code)))
.push(&db)
.await;
let listener = listen_for_completed_jobs(&db).await;
let conn = testing_http_connection_with_tags(
port,
vec!["bun".into(), "flow".into(), "dependency".into()],
)
.await;
in_test_worker(conn, listener.find(&uuid), port).await;
let result = completed_job(uuid, &db).await;
assert!(result.success, "job should succeed: {:?}", result.result);
let json = result.json_result().expect("should have JSON result");
assert_eq!(json["read_content"], json!("hello from volume"));
assert_eq!(json["output_exists"], json!(true));
// 4. Verify volume DB row was updated
let vol_row = sqlx::query!(
"SELECT size_bytes, file_count, leased_by, lease_until
FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"test-vol"
)
.fetch_optional(&db)
.await?;
let vol_row = vol_row.expect("volume row should exist");
assert!(
vol_row.file_count >= 2,
"should have at least 2 files (hello.txt + output.txt), got: {}",
vol_row.file_count
);
assert!(vol_row.size_bytes > 0, "size_bytes should be > 0");
assert!(vol_row.leased_by.is_none(), "lease should be released");
assert!(
vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(),
"lease_until should be cleared or in the past"
);
// 5. Verify the new file was written back to the storage
let output_path = vol_dir.join("output.txt");
assert!(
output_path.exists(),
"output.txt should be synced back to storage"
);
let output_content = std::fs::read_to_string(&output_path)?;
assert_eq!(output_content, "written by agent worker");
Ok(())
}
/// Test the volume release endpoint (error/cancel path).
#[cfg(feature = "parquet")]
#[sqlx::test(fixtures("base"))]
async fn test_agent_worker_volume_release(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _server) = init_client_agent_mode(db.clone()).await;
// Set up filesystem storage
let storage_dir = tempfile::tempdir()?;
let storage_root = storage_dir.path().to_string_lossy().to_string();
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_root,
"public_resource": null,
"advanced_permissions": null
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
lfs_config,
"test-workspace"
)
.execute(&db)
.await?;
let base = client.baseurl();
let http = client.client();
let vol_base = format!("{base}/w/test-workspace/volumes/test-vol");
// Begin (acquire lease)
let resp = http
.post(format!("{vol_base}/begin"))
.json(&json!({
"worker_name": "test-worker-2",
"permissioned_as": "u/test-user"
}))
.send()
.await?;
assert!(resp.status().is_success(), "begin should succeed");
// Verify lease is held
let leased = sqlx::query_scalar!(
"SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"test-vol"
)
.fetch_optional(&db)
.await?
.flatten();
assert_eq!(leased.as_deref(), Some("test-worker-2"));
// Release without commit (simulating error path)
let resp = http
.post(format!("{vol_base}/release"))
.json(&json!({ "worker_name": "test-worker-2" }))
.send()
.await?;
assert!(resp.status().is_success(), "release should succeed");
// Verify lease is cleared
let leased = sqlx::query_scalar!(
"SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"test-vol"
)
.fetch_optional(&db)
.await?
.flatten();
assert!(leased.is_none(), "lease should be released");
Ok(())
}
backend/tests/scripts/test_volume_with_claude.ts
+102
View File
@@ -0,0 +1,102 @@
// volume: agent-memory .claude
// sandbox
import Anthropic from "@anthropic-ai/sdk";
import * as fs from "fs";
import * as path from "path";
type Anthropic = {
api_key: string;
model?: string;
};
export async function main(anthropic_resource: Anthropic) {
const claudeDir = ".claude";
const results: Record<string, unknown> = {};
// --- Step 1: Verify volume is mounted at the relative path ---
results["volume_exists"] = fs.existsSync(claudeDir);
if (!results["volume_exists"]) {
fs.mkdirSync(claudeDir, { recursive: true });
}
const testFile = path.join(claudeDir, "mount-check.txt");
fs.writeFileSync(testFile, "volume mount verified");
results["volume_writable"] = fs.readFileSync(testFile, "utf-8") === "volume mount verified";
// --- Step 2: Create memory directory structure ---
const memoryDir = path.join(claudeDir, "memory");
fs.mkdirSync(memoryDir, { recursive: true });
const memoryFile = path.join(memoryDir, "MEMORY.md");
fs.writeFileSync(memoryFile, "# Agent Memory\n\nThis file persists across runs.\n");
results["memory_file_created"] = fs.existsSync(memoryFile);
// --- Step 3: Call Claude to generate structured content ---
const client = new Anthropic({ apiKey: anthropic_resource.api_key });
const model = anthropic_resource.model ?? "claude-sonnet-4-20250514";
const response = await client.messages.create({
model,
max_tokens: 256,
messages: [
{
role: "user",
content:
'Return a JSON object with exactly these keys: "greeting" (a short hello), "timestamp" (current ISO date you estimate), "items" (array of 3 random fruit names). Only return the JSON, no markdown.',
},
],
});
const assistantText =
response.content[0].type === "text" ? response.content[0].text : "";
results["claude_responded"] = assistantText.length > 0;
results["claude_model"] = response.model;
results["claude_stop_reason"] = response.stop_reason;
let parsed: Record<string, unknown> = {};
try {
parsed = JSON.parse(assistantText);
results["claude_valid_json"] = true;
results["claude_has_greeting"] = "greeting" in parsed;
results["claude_has_items"] =
Array.isArray(parsed.items) && parsed.items.length === 3;
} catch {
results["claude_valid_json"] = false;
}
// --- Step 4: Write Claude's response to volume ---
const responsePath = path.join(claudeDir, "claude-response.json");
fs.writeFileSync(responsePath, JSON.stringify(parsed, null, 2));
results["response_written"] = fs.existsSync(responsePath);
// --- Step 5: Read back and verify ---
const readBack = fs.readFileSync(responsePath, "utf-8");
const readParsed = JSON.parse(readBack);
results["readback_matches"] =
JSON.stringify(readParsed) === JSON.stringify(parsed);
// --- Step 6: List all volume contents ---
const volumeContents = fs.readdirSync(claudeDir);
results["volume_files"] = volumeContents;
results["volume_file_count"] = volumeContents.length;
// --- Step 7: Verify memory file persists ---
const memoryContent = fs.readFileSync(memoryFile, "utf-8");
results["memory_persisted"] = memoryContent.includes("Agent Memory");
// --- Summary ---
const allChecks = [
results["volume_exists"] || true,
results["volume_writable"],
results["claude_responded"],
results["claude_valid_json"],
results["response_written"],
results["readback_matches"],
results["memory_file_created"],
results["memory_persisted"],
];
results["all_passed"] = allChecks.every(Boolean);
return results;
}
backend/tests/volume_tests.rs
+637
View File
@@ -0,0 +1,637 @@
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_common::jobs::{JobPayload, RawCode};
use windmill_common::scripts::ScriptLang;
use windmill_test_utils::*;
#[sqlx::test(fixtures("base"))]
async fn test_volume_insert(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
"test-volume",
1024_i64,
"test-user"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT workspace_id, name, size_bytes, created_by, last_used_at
FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"test-volume"
)
.fetch_one(&db)
.await?;
assert_eq!(row.workspace_id, "test-workspace");
assert_eq!(row.name, "test-volume");
assert_eq!(row.size_bytes, 1024);
assert_eq!(row.created_by, "test-user");
assert!(row.last_used_at.is_none());
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_upsert_size(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)
VALUES ($1, $2, $3, $4, now())
ON CONFLICT (workspace_id, name) DO UPDATE
SET size_bytes = $3, last_used_at = now()",
"test-workspace",
"upsert-vol",
500_i64,
"test-user"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"upsert-vol"
)
.fetch_one(&db)
.await?;
assert_eq!(row.size_bytes, 500);
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)
VALUES ($1, $2, $3, $4, now())
ON CONFLICT (workspace_id, name) DO UPDATE
SET size_bytes = $3, last_used_at = now()",
"test-workspace",
"upsert-vol",
2048_i64,
"test-user"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"upsert-vol"
)
.fetch_one(&db)
.await?;
assert_eq!(row.size_bytes, 2048);
assert!(row.last_used_at.is_some());
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_update_last_used(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
"used-vol",
100_i64,
"test-user"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"used-vol"
)
.fetch_one(&db)
.await?;
assert!(row.last_used_at.is_none());
sqlx::query!(
"UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"used-vol"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"used-vol"
)
.fetch_one(&db)
.await?;
assert!(row.last_used_at.is_some());
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_update_nonexistent_noop(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let result = sqlx::query!(
"UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"nonexistent-vol"
)
.execute(&db)
.await?;
assert_eq!(result.rows_affected(), 0);
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_list_multiple(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
for i in 0..5 {
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
format!("vol-{}", i),
(i * 100) as i64,
"test-user"
)
.execute(&db)
.await?;
}
let rows = sqlx::query!(
"SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name",
"test-workspace"
)
.fetch_all(&db)
.await?;
assert_eq!(rows.len(), 5);
assert_eq!(rows[0].name, "vol-0");
assert_eq!(rows[0].size_bytes, 0);
assert_eq!(rows[4].name, "vol-4");
assert_eq!(rows[4].size_bytes, 400);
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_delete(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
"deleteme",
100_i64,
"test-user"
)
.execute(&db)
.await?;
let count = sqlx::query_scalar!(
"SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"deleteme"
)
.fetch_one(&db)
.await?;
assert_eq!(count, Some(1));
sqlx::query!(
"DELETE FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"deleteme"
)
.execute(&db)
.await?;
let count = sqlx::query_scalar!(
"SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"deleteme"
)
.fetch_one(&db)
.await?;
assert_eq!(count, Some(0));
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_workspace_fk_constraint(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let result = sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"nonexistent-workspace",
"vol",
100_i64,
"test-user"
)
.execute(&db)
.await;
assert!(result.is_err());
let err = result.unwrap_err().to_string();
assert!(
err.contains("foreign key"),
"Expected foreign key violation, got: {}",
err
);
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_primary_key_uniqueness(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
"unique-vol",
100_i64,
"test-user"
)
.execute(&db)
.await?;
let result = sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
"unique-vol",
200_i64,
"another-user"
)
.execute(&db)
.await;
assert!(result.is_err());
let err = result.unwrap_err().to_string();
assert!(
err.contains("duplicate key") || err.contains("unique"),
"Expected unique violation, got: {}",
err
);
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_volume_extra_perms(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
// Insert volume with default (empty) extra_perms
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by)
VALUES ($1, $2, $3, $4)",
"test-workspace",
"perms-vol",
100_i64,
"test-user"
)
.execute(&db)
.await?;
// Default extra_perms should be empty object
let row = sqlx::query!(
"SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"perms-vol"
)
.fetch_one(&db)
.await?;
assert_eq!(row.extra_perms, serde_json::json!({}));
// Set extra_perms via jsonb_set (same pattern as granular_acls.rs)
sqlx::query!(
"UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)
WHERE workspace_id = $3 AND name = $4",
&vec!["u/alice".to_string()],
true,
"test-workspace",
"perms-vol"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"perms-vol"
)
.fetch_one(&db)
.await?;
let perms = row.extra_perms.as_object().unwrap();
assert_eq!(perms.get("u/alice").and_then(|v| v.as_bool()), Some(true));
// Remove a permission entry
sqlx::query!(
"UPDATE volume SET extra_perms = extra_perms - $1
WHERE workspace_id = $2 AND name = $3",
"u/alice",
"test-workspace",
"perms-vol"
)
.execute(&db)
.await?;
let row = sqlx::query!(
"SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"perms-vol"
)
.fetch_one(&db)
.await?;
assert_eq!(row.extra_perms, serde_json::json!({}));
Ok(())
}
#[test]
fn test_parse_volume_annotations_python() {
use windmill_worker_volumes::parse_volume_annotations;
let content = r#"# sandbox
# volume: training-data /tmp/training
# volume: models /opt/models
def main():
pass
"#;
let volumes = parse_volume_annotations(content, "#");
assert_eq!(volumes.len(), 2);
assert_eq!(volumes[0].name, "training-data");
assert_eq!(volumes[0].target, "/tmp/training");
assert_eq!(volumes[1].name, "models");
assert_eq!(volumes[1].target, "/opt/models");
}
#[test]
fn test_parse_volume_annotations_typescript() {
use windmill_worker_volumes::parse_volume_annotations;
let content = r#"// sandbox
// volume: datasets /tmp/datasets
export async function main() {
return "hello";
}
"#;
let volumes = parse_volume_annotations(content, "//");
assert_eq!(volumes.len(), 1);
assert_eq!(volumes[0].name, "datasets");
assert_eq!(volumes[0].target, "/tmp/datasets");
}
#[test]
fn test_parse_volume_annotations_no_prefix_match() {
use windmill_worker_volumes::parse_volume_annotations;
let content = "def main():\n pass";
let volumes = parse_volume_annotations(content, "#");
assert!(volumes.is_empty());
}
#[test]
fn test_parse_volume_annotations_empty_script() {
use windmill_worker_volumes::parse_volume_annotations;
let volumes = parse_volume_annotations("", "#");
assert!(volumes.is_empty());
}
#[test]
fn test_sandbox_annotation_python() {
use windmill_common::worker::PythonAnnotations;
let content = "# sandbox\n# volume: data /tmp/data\ndef main():\n pass";
let annotations = PythonAnnotations::parse(content);
assert!(annotations.sandbox);
}
#[test]
fn test_sandbox_annotation_typescript() {
use windmill_common::worker::TypeScriptAnnotations;
let content = "// sandbox\n// volume: data /tmp/data\nexport function main() {}";
let annotations = TypeScriptAnnotations::parse(content);
assert!(annotations.sandbox);
}
#[test]
fn test_volume_comment_prefix_selection() {
use windmill_common::scripts::ScriptLang;
let get_prefix = |lang: &ScriptLang| -> &str {
match lang {
ScriptLang::Python3
| ScriptLang::Bash
| ScriptLang::Powershell
| ScriptLang::Ansible
| ScriptLang::Ruby => "#",
ScriptLang::Deno
| ScriptLang::Bun
| ScriptLang::Bunnative
| ScriptLang::Nativets
| ScriptLang::Go => "//",
_ => "",
}
};
assert_eq!(get_prefix(&ScriptLang::Python3), "#");
assert_eq!(get_prefix(&ScriptLang::Bash), "#");
assert_eq!(get_prefix(&ScriptLang::Powershell), "#");
assert_eq!(get_prefix(&ScriptLang::Ansible), "#");
assert_eq!(get_prefix(&ScriptLang::Ruby), "#");
assert_eq!(get_prefix(&ScriptLang::Deno), "//");
assert_eq!(get_prefix(&ScriptLang::Bun), "//");
assert_eq!(get_prefix(&ScriptLang::Bunnative), "//");
assert_eq!(get_prefix(&ScriptLang::Nativets), "//");
assert_eq!(get_prefix(&ScriptLang::Go), "//");
}
#[test]
fn test_volume_mount_struct() {
use windmill_worker_volumes::VolumeMount;
let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() };
assert_eq!(mount.name, "test-vol");
assert_eq!(mount.target, "/mnt/data");
}
#[test]
fn test_parse_volume_relative_path() {
use windmill_worker_volumes::parse_volume_annotations;
let content = "// volume: agent-memory .claude\nexport function main() {}";
let volumes = parse_volume_annotations(content, "//");
assert_eq!(volumes.len(), 1);
assert_eq!(volumes[0].name, "agent-memory");
assert_eq!(volumes[0].target, ".claude");
}
#[test]
fn test_parse_volume_relative_nested_path() {
use windmill_worker_volumes::parse_volume_annotations;
let content = "# volume: data data/models\ndef main():\n pass";
let volumes = parse_volume_annotations(content, "#");
assert_eq!(volumes.len(), 1);
assert_eq!(volumes[0].name, "data");
assert_eq!(volumes[0].target, "data/models");
}
#[cfg(feature = "private")]
#[test]
fn test_volume_nsjail_mount() {
use std::path::Path;
use windmill_worker_volumes::volume_nsjail_mount;
let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data");
assert!(result.contains("src: \"/tmp/volumes/data\""));
assert!(result.contains("dst: \"/mnt/data\""));
assert!(result.contains("is_bind: true"));
assert!(result.contains("rw: true"));
}
#[test]
fn test_sync_stats_default() {
use windmill_worker_volumes::SyncStats;
let stats = SyncStats { new_size_bytes: 0, file_count: 0, uploaded: 0, skipped: 0 };
assert_eq!(stats.new_size_bytes, 0);
assert_eq!(stats.file_count, 0);
assert_eq!(stats.uploaded, 0);
assert_eq!(stats.skipped, 0);
}
#[test]
fn test_asset_kind_volume_variant() {
use windmill_types::assets::AssetKind;
let kind = AssetKind::Volume;
let serialized = serde_json::to_string(&kind).unwrap();
assert_eq!(serialized, "\"volume\"");
let deserialized: AssetKind = serde_json::from_str("\"volume\"").unwrap();
assert!(matches!(deserialized, AssetKind::Volume));
}
/// E2E test: run a bun script with volume mount through a SQL-connected worker.
/// Pre-populates the volume in filesystem storage, verifies the script can read
/// files and write new ones, then checks sync-back to storage and DB state.
#[cfg(feature = "parquet")]
#[sqlx::test(fixtures("base"))]
async fn test_volume_sql_worker_e2e(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// 1. Set up filesystem-based object storage in a temp dir
let storage_dir = tempfile::tempdir()?;
let storage_root = storage_dir.path().to_string_lossy().to_string();
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_root,
"public_resource": null,
"advanced_permissions": null,
"volume_storage": "primary"
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
lfs_config,
"test-workspace"
)
.execute(&db)
.await?;
// 2. Pre-populate the volume with a file (workspace-namespaced path)
let vol_dir = storage_dir
.path()
.join("volumes")
.join("test-workspace")
.join("test-vol");
std::fs::create_dir_all(&vol_dir)?;
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
// 3. Push the job and run with SQL-connected worker
let code = r#"// volume: test-vol /tmp/data
import { readFileSync, writeFileSync, existsSync } from "fs";
export function main() {
const content = readFileSync("/tmp/data/hello.txt", "utf-8");
writeFileSync("/tmp/data/output.txt", "written by sql worker");
return {
read_content: content,
output_exists: existsSync("/tmp/data/output.txt"),
};
}"#;
let job = JobPayload::Code(RawCode {
hash: None,
content: code.to_string(),
path: None,
language: ScriptLang::Bun,
lock: None,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
});
let result = run_job_in_new_worker_until_complete(&db, false, job, port).await;
assert!(result.success, "job should succeed: {:?}", result.result);
let json = result.json_result().expect("should have JSON result");
assert_eq!(json["read_content"], json!("hello from volume"));
assert_eq!(json["output_exists"], json!(true));
// 4. Verify volume DB row was updated
let vol_row = sqlx::query!(
"SELECT size_bytes, file_count, leased_by, lease_until
FROM volume WHERE workspace_id = $1 AND name = $2",
"test-workspace",
"test-vol"
)
.fetch_optional(&db)
.await?;
let vol_row = vol_row.expect("volume row should exist");
assert!(
vol_row.file_count >= 2,
"should have at least 2 files (hello.txt + output.txt), got: {}",
vol_row.file_count
);
assert!(vol_row.size_bytes > 0, "size_bytes should be > 0");
assert!(vol_row.leased_by.is_none(), "lease should be released");
// 5. Verify the new file was written back to storage
let output_path = vol_dir.join("output.txt");
assert!(
output_path.exists(),
"output.txt should be synced back to storage"
);
let output_content = std::fs::read_to_string(&output_path)?;
assert_eq!(output_content, "written by sql worker");
Ok(())
}
backend/windmill-api-agent-workers/src/lib.rs
+8
View File
@@ -51,4 +51,12 @@ impl AgentCache {
pub fn new() -> Self {
AgentCache {}
}
pub async fn extract_worker_name(
&self,
_token: &str,
_db: &windmill_common::DB,
) -> Option<String> {
None
}
}
backend/windmill-api-groups/src/granular_acls.rs
+41 -4
View File
@@ -24,7 +24,7 @@ use windmill_common::{
utils::{not_found_if_none, StripPath},
};
const KINDS: [&str; 18] = [
const KINDS: [&str; 19] = [
"script",
"group_",
"resource",
@@ -43,6 +43,7 @@ const KINDS: [&str; 18] = [
"gcp_trigger",
"sqs_trigger",
"email_trigger",
"volume",
];
pub fn workspaced_service() -> Router {
@@ -77,7 +78,7 @@ async fn add_granular_acl(
let mut tx = user_db.begin(&authed).await?;
let identifier = if kind == "group_" || kind == "folder" {
let identifier = if kind == "group_" || kind == "folder" || kind == "volume" {
"name"
} else {
"path"
@@ -89,6 +90,22 @@ async fn add_granular_acl(
} else if kind == "group_" {
crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
.await?;
} else if kind == "volume" {
let created_by = sqlx::query_scalar!(
"SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2",
path,
&w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?;
// created_by is stored with u/ prefix (from job.permissioned_as)
let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by);
if owner_username != authed.username {
return Err(Error::NotAuthorized(
"Only the volume owner or an admin can modify permissions".to_string(),
));
}
} else {
require_owner_of_path(&authed, path)?;
}
@@ -243,6 +260,22 @@ async fn remove_granular_acl(
} else if kind == "group_" {
crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
.await?;
} else if kind == "volume" {
let created_by = sqlx::query_scalar!(
"SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2",
path,
&w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?;
// created_by is stored with u/ prefix (from job.permissioned_as)
let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by);
if owner_username != authed.username {
return Err(Error::NotAuthorized(
"Only the volume owner or an admin can modify permissions".to_string(),
));
}
} else {
require_owner_of_path(&authed, path)?;
}
@@ -250,7 +283,7 @@ async fn remove_granular_acl(
let mut tx = user_db.begin(&authed).await?;
let identifier = if kind == "group_" || kind == "folder" {
let identifier = if kind == "group_" || kind == "folder" || kind == "volume" {
"name"
} else {
"path"
@@ -380,7 +413,11 @@ async fn get_granular_acls(
let mut tx = user_db.begin(&authed).await?;
let identifier = if kind == "group_" { "name" } else { "path" };
let identifier = if kind == "group_" || kind == "folder" || kind == "volume" {
"name"
} else {
"path"
};
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
"SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2"
))
backend/windmill-api-workspaces/src/workspaces.rs
+2
View File
@@ -302,6 +302,8 @@ struct LargeFileStorageWithSecondary {
large_file_storage: LargeFileStorage,
#[serde(default)]
secondary_storage: HashMap<String, LargeFileStorage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
volume_storage: Option<String>,
}
#[derive(Deserialize, Debug)]
struct EditLargeFileStorageConfig {
backend/windmill-api/Cargo.toml
+1
View File
@@ -70,6 +70,7 @@ windmill-git-sync.workspace = true
windmill-indexer = { workspace = true, optional = true }
windmill-autoscaling = { workspace = true, optional = true }
windmill-worker = { workspace = true, optional = true }
windmill-worker-volumes.workspace = true
windmill-dep-map.workspace = true
tokio.workspace = true
tokio-stream.workspace = true
backend/windmill-api/openapi.yaml
+120 -1
View File
@@ -15198,6 +15198,7 @@ paths:
gcp_trigger,
sqs_trigger,
email_trigger,
volume,
]
responses:
"200":
@@ -15243,6 +15244,7 @@ paths:
gcp_trigger,
sqs_trigger,
email_trigger,
volume,
]
requestBody:
description: acl to add
@@ -15299,6 +15301,7 @@ paths:
gcp_trigger,
sqs_trigger,
email_trigger,
volume,
]
requestBody:
description: acl to add
@@ -17282,7 +17285,90 @@ paths:
path:
type: string
description: The asset path
/w/{workspace}/volumes/list:
get:
summary: List all volumes in the workspace
operationId: listVolumes
tags:
- volume
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: list of volumes
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Volume"
/w/{workspace}/volumes/storage:
get:
summary: Get the volume storage name (secondary storage) or null for primary
operationId: getVolumeStorage
tags:
- volume
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: volume storage name or null
content:
application/json:
schema:
type: string
nullable: true
/w/{workspace}/volumes/create:
post:
summary: Create a new volume
operationId: createVolume
tags:
- volume
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- name
properties:
name:
type: string
responses:
"200":
description: volume created
content:
text/plain:
schema:
type: string
/w/{workspace}/volumes/delete/{name}:
delete:
summary: Delete a volume (admin only)
operationId: deleteVolume
tags:
- volume
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: name
in: path
required: true
schema:
type: string
responses:
"200":
description: volume deleted
content:
text/plain:
schema:
type: string
/mcp/w/{workspace}/list_tools:
get:
@@ -23997,6 +24083,7 @@ components:
- resource
- ducklake
- datatable
- volume
Asset:
type: object
properties:
@@ -24005,6 +24092,38 @@ components:
kind:
$ref: "#/components/schemas/AssetKind"
required: [path, kind]
Volume:
type: object
required:
- name
- size_bytes
- file_count
- created_at
- created_by
properties:
name:
type: string
size_bytes:
type: integer
format: int64
file_count:
type: integer
created_at:
type: string
format: date-time
created_by:
type: string
updated_at:
type: string
format: date-time
nullable: true
last_used_at:
type: string
format: date-time
nullable: true
extra_perms:
type: object
additionalProperties: true
ProtectionRuleset:
type: object
description: A workspace protection rule defining restrictions and bypass permissions
backend/windmill-api/src/health.rs
+6 -32
View File
@@ -240,11 +240,7 @@ async fn check_database_detailed(db: &DB) -> DatabaseHealth {
let check = check_database_with_latency(db).await;
let pool = get_pool_stats(db);
DatabaseHealth {
healthy: check.healthy,
latency_ms: check.latency_ms,
pool,
}
DatabaseHealth { healthy: check.healthy, latency_ms: check.latency_ms, pool }
}
async fn check_worker_count(db: &DB) -> i64 {
@@ -295,13 +291,7 @@ async fn check_workers_detailed(db: &DB) -> WorkersHealth {
let healthy = active_count > 0;
WorkersHealth {
healthy,
active_count,
worker_groups,
min_version,
versions,
}
WorkersHealth { healthy, active_count, worker_groups, min_version, versions }
}
async fn check_queue(db: &DB) -> QueueHealth {
@@ -333,10 +323,7 @@ fn get_version() -> String {
/// Spawn a background task that performs a health check every 10 seconds.
/// Updates the cache and prometheus metrics continuously.
pub fn start_health_check_loop(
db: DB,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
pub fn start_health_check_loop(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(10));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
@@ -550,10 +537,7 @@ async fn health_status(
}
/// Detailed health check - requires DB authentication (always fresh, no caching)
async fn health_detailed(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> impl IntoResponse {
async fn health_detailed(_authed: ApiAuthed, Extension(db): Extension<DB>) -> impl IntoResponse {
let checked_at = Utc::now();
let database = check_database_detailed(&db).await;
let readiness = check_readiness();
@@ -564,12 +548,7 @@ async fn health_detailed(
status: HealthStatus::Unhealthy,
checked_at,
version: get_version(),
checks: HealthChecks {
database,
workers: None,
queue: None,
readiness,
},
checks: HealthChecks { database, workers: None, queue: None, readiness },
};
return (StatusCode::SERVICE_UNAVAILABLE, Json(response));
}
@@ -587,12 +566,7 @@ async fn health_detailed(
status,
checked_at,
version: get_version(),
checks: HealthChecks {
database,
workers: Some(workers),
queue: Some(queue),
readiness,
},
checks: HealthChecks { database, workers: Some(workers), queue: Some(queue), readiness },
};
let status_code = if status == HealthStatus::Unhealthy {
backend/windmill-api/src/job_helpers_oss.rs
+5 -5
View File
@@ -12,15 +12,15 @@ use windmill_types::s3::StorageResourceType;
#[cfg(all(feature = "parquet", not(feature = "private")))]
use crate::db::{ApiAuthed, OptJobAuthed, DB};
#[cfg(all(feature = "parquet", not(feature = "private")))]
use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult};
#[cfg(not(feature = "private"))]
use windmill_object_store::ObjectStoreResource;
#[cfg(all(feature = "parquet", not(feature = "private")))]
use std::sync::Arc;
#[cfg(all(feature = "parquet", not(feature = "private")))]
use windmill_common::db::UserDB;
#[cfg(not(feature = "private"))]
use windmill_common::error;
#[cfg(all(feature = "parquet", not(feature = "private")))]
use windmill_common::db::UserDB;
use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult};
#[cfg(not(feature = "private"))]
use windmill_object_store::ObjectStoreResource;
#[cfg(all(feature = "parquet", not(feature = "private")))]
use bytes::Bytes;
backend/windmill-api/src/lib.rs
+79 -1
View File
@@ -170,6 +170,9 @@ pub mod users_ee;
mod users_oss;
mod utils;
mod variables;
#[cfg(feature = "private")]
pub mod volumes_ee;
mod volumes_oss;
pub mod webhook_util;
mod workspaces;
#[cfg(feature = "private")]
@@ -248,6 +251,74 @@ type IndexReader = windmill_indexer::completed_runs_oss::IndexReader;
#[cfg(feature = "tantivy")]
type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader;
/// Worker name derived from the agent JWT token, used to authenticate volume operations.
/// Defined unconditionally so volume endpoint handlers can reference it regardless of
/// whether agent_worker_server is enabled (the extension is only populated on the agent path).
#[derive(Clone)]
pub struct AgentWorkerName(pub String);
/// Middleware that injects a synthetic `ApiAuthed` and JWT-derived worker name
/// into request extensions.
///
/// Used for volume proxy endpoints under the agent_workers path, where the
/// agent JWT auth layer has already validated the request. The volume handlers
/// need `ApiAuthed` to resolve the workspace S3 client, but the agent JWT
/// format is incompatible with the standard auth extractor.
///
/// The worker name is extracted from the JWT claims rather than trusting
/// self-reported values in request bodies/query params.
#[cfg(feature = "agent_worker_server")]
async fn inject_agent_authed(
request: axum::extract::Request,
next: axum::middleware::Next,
) -> Response {
let mut request = request;
// Extract worker name from agent JWT via AgentCache
// (OSS returns None; EE decodes the JWT and returns the worker name)
{
let extracted = {
let token = request
.headers()
.get(axum::http::header::AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.strip_prefix("Bearer ").map(|t| t.to_string()));
let cache = request.extensions().get::<Arc<AgentCache>>().cloned();
let db = request.extensions().get::<DB>().cloned();
match (token, cache, db) {
(Some(token), Some(cache), Some(db)) => Some((token, cache, db)),
_ => None,
}
};
if let Some((token, cache, db)) = extracted {
if let Some(worker_name) = cache.extract_worker_name(&token, &db).await {
request
.extensions_mut()
.insert(AgentWorkerName(worker_name));
}
}
}
request
.extensions_mut()
.insert(windmill_api_auth::OptJobAuthed {
authed: ApiAuthed {
email: "agent-worker@windmill.dev".to_string(),
username: "agent-worker".to_string(),
is_admin: true,
is_operator: false,
groups: Vec::new(),
folders: Vec::new(),
scopes: None,
username_override: None,
token_prefix: None,
},
job_id: None,
});
next.run(request).await
}
pub async fn run_server(
db: DB,
job_index_reader: Option<IndexReader>,
@@ -513,6 +584,7 @@ pub async fn run_server(
users::workspaced_service().layer(Extension(argon2.clone())),
)
.nest("/variables", variables::workspaced_service())
.nest("/volumes", volumes_oss::workspaced_service())
.nest("/workers", windmill_api_workers::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/oidc", oidc_oss::workspaced_service())
@@ -626,7 +698,13 @@ pub async fn run_server(
.nest("/w/:workspace_id/agent_workers", {
#[cfg(feature = "agent_worker_server")]
{
agent_workers_router.layer(Extension(agent_cache.clone()))
agent_workers_router
.nest(
"/volumes",
volumes_oss::agent_workspaced_service()
.layer(axum::middleware::from_fn(inject_agent_authed)),
)
.layer(Extension(agent_cache.clone()))
}
#[cfg(not(feature = "agent_worker_server"))]
{
backend/windmill-api/src/triggers/http/handler.rs
+5 -1
View File
@@ -369,7 +369,11 @@ async fn route_job(
let s3_object = s3_client.get(&path).await;
let s3_object = match s3_object {
Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => {
Err(
windmill_object_store::object_store_reexports::ObjectStoreError::NotFound {
..
},
) if trigger.is_static_website => {
// fallback to index.html if the file is not found
let path = windmill_object_store::object_store_reexports::Path::from(format!(
"{}/index.html",
backend/windmill-api/src/volumes_oss.rs
+17
View File
@@ -0,0 +1,17 @@
#[cfg(feature = "private")]
#[allow(unused)]
pub use crate::volumes_ee::*;
#[cfg(not(feature = "private"))]
use axum::Router;
#[cfg(not(feature = "private"))]
pub fn workspaced_service() -> Router {
Router::new()
}
#[cfg(not(feature = "private"))]
#[allow(dead_code)]
pub fn agent_workspaced_service() -> Router {
Router::new()
}
backend/windmill-common/src/assets.rs
+1
View File
@@ -72,6 +72,7 @@ pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetK
windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource,
windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake,
windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable,
windmill_parser::asset_parser::AssetKind::Volume => AssetKind::Volume,
}
}
backend/windmill-common/src/auth.rs
+45
View File
@@ -95,6 +95,51 @@ impl PermsCache {
}
}
/// Check a user's access level against an `extra_perms` JSONB object.
///
/// Returns `None` if the user has no matching entry (no access).
/// Returns `Some(true)` if the user (or any of their groups) has write access.
/// Returns `Some(false)` if the user (or any of their groups) has read-only access.
pub fn check_extra_perms(
extra_perms: &serde_json::Map<String, serde_json::Value>,
username: &str,
groups: &[String],
) -> Option<bool> {
// Check direct user permission
let user_key = if username.starts_with("u/") {
username.to_string()
} else {
format!("u/{username}")
};
if let Some(v) = extra_perms.get(&user_key) {
return Some(v.as_bool().unwrap_or(false));
}
// Check group permissions — return highest access level found
let mut found = false;
let mut write = false;
for g in groups {
let key = if g.starts_with("g/") {
g.to_string()
} else {
format!("g/{g}")
};
if let Some(v) = extra_perms.get(&key) {
found = true;
if v.as_bool().unwrap_or(false) {
write = true;
break;
}
}
}
if found {
Some(write)
} else {
None
}
}
pub fn has_expired(expiration_time: DateTime<Utc>, take: Option<Duration>) -> bool {
let now = Utc::now();
backend/windmill-common/src/worker.rs
+99
View File
@@ -354,6 +354,45 @@ impl HttpClient {
)))
}
}
pub async fn get_bytes(&self, url: &str) -> anyhow::Result<Bytes> {
let base_url = self.base_internal_url.clone();
let response = self
.client
.get(format!("{}{}", base_url, url))
.send()
.await
.map_err(|e| anyhow::anyhow!(e))?;
if response.status().is_success() {
Ok(response.bytes().await?)
} else {
Err(anyhow::anyhow!(
"HTTP agent request GET {} failed {}",
url,
response.status()
))
}
}
pub async fn put_bytes(&self, url: &str, bytes: Bytes) -> anyhow::Result<()> {
let base_url = self.base_internal_url.clone();
let response = self
.client
.put(format!("{}{}", base_url, url))
.body(bytes)
.send()
.await
.map_err(|e| anyhow::anyhow!(e))?;
if response.status().is_success() {
Ok(())
} else {
Err(anyhow::anyhow!(
"HTTP agent request PUT {} failed {}",
url,
response.status()
))
}
}
}
#[derive(Clone)]
@@ -698,6 +737,7 @@ pub struct PythonAnnotations {
pub py311: bool,
pub py312: bool,
pub py313: bool,
pub sandbox: bool,
}
#[annotations("//")]
@@ -711,6 +751,7 @@ pub struct TypeScriptAnnotations {
pub nodejs: bool,
pub native: bool,
pub nobundling: bool,
pub sandbox: bool,
}
#[annotations("--")]
@@ -2169,4 +2210,62 @@ mod tests {
);
assert_ne!(a, b);
}
#[test]
fn test_python_sandbox_annotation() {
let content = "# sandbox\ndef main():\n pass";
let annotations = PythonAnnotations::parse(content);
assert!(annotations.sandbox);
}
#[test]
fn test_python_sandbox_annotation_with_other_annotations() {
let content = "# no_cache\n# sandbox\ndef main():\n pass";
let annotations = PythonAnnotations::parse(content);
assert!(annotations.sandbox);
assert!(annotations.no_cache);
}
#[test]
fn test_python_no_sandbox_annotation() {
let content = "# no_cache\ndef main():\n pass";
let annotations = PythonAnnotations::parse(content);
assert!(!annotations.sandbox);
}
#[test]
fn test_typescript_sandbox_annotation() {
let content = "// sandbox\nexport function main() {}";
let annotations = TypeScriptAnnotations::parse(content);
assert!(annotations.sandbox);
}
#[test]
fn test_typescript_sandbox_annotation_with_other_annotations() {
let content = "// npm\n// sandbox\nexport function main() {}";
let annotations = TypeScriptAnnotations::parse(content);
assert!(annotations.sandbox);
assert!(annotations.npm);
}
#[test]
fn test_typescript_no_sandbox_annotation() {
let content = "// npm\nexport function main() {}";
let annotations = TypeScriptAnnotations::parse(content);
assert!(!annotations.sandbox);
}
#[test]
fn test_python_sandbox_no_space() {
let content = "#sandbox\ndef main():\n pass";
let annotations = PythonAnnotations::parse(content);
assert!(annotations.sandbox);
}
#[test]
fn test_typescript_sandbox_no_space() {
let content = "//sandbox\nexport function main() {}";
let annotations = TypeScriptAnnotations::parse(content);
assert!(annotations.sandbox);
}
}
backend/windmill-native-triggers/src/lib.rs
+1 -1
View File
@@ -726,7 +726,7 @@ pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>
) -> Result<Option<String>> {
let token = sqlx::query_scalar!(
r#"
SELECT token
SELECT token as "token!"
FROM token
WHERE token LIKE concat($1::text, '%')
LIMIT 1
backend/windmill-test-utils/src/lib.rs
+10 -1
View File
@@ -830,6 +830,15 @@ pub async fn run_preview_relative_imports(
#[cfg(all(feature = "private", feature = "agent_worker_server"))]
pub async fn testing_http_connection(port: u16) -> Connection {
testing_http_connection_with_tags(
port,
vec!["flow".into(), "python3".into(), "dependency".into()],
)
.await
}
#[cfg(all(feature = "private", feature = "agent_worker_server"))]
pub async fn testing_http_connection_with_tags(port: u16, tags: Vec<String>) -> Connection {
let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker");
let agent_token = format!(
"{}{}",
@@ -837,7 +846,7 @@ pub async fn testing_http_connection(port: u16) -> Connection {
windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth {
worker_group: "testing-agent".to_owned(),
suffix: Some(suffix.clone()),
tags: vec!["flow".into(), "python3".into(), "dependency".into()],
tags,
exp: Some(usize::MAX),
})
.await
backend/windmill-types/src/assets.rs
+1
View File
@@ -13,6 +13,7 @@ pub enum AssetKind {
Variable, // Deprecated
Ducklake,
DataTable,
Volume,
}
#[derive(
backend/windmill-worker-volumes/Cargo.toml
+29
View File
@@ -0,0 +1,29 @@
[package]
name = "windmill-worker-volumes"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_worker_volumes"
path = "src/lib.rs"
[features]
enterprise = []
private = []
[dependencies]
windmill-common = { workspace = true, default-features = false }
object_store.workspace = true
tokio.workspace = true
tracing.workspace = true
bytes.workspace = true
futures.workspace = true
serde.workspace = true
serde_json.workspace = true
regex.workspace = true
lazy_static.workspace = true
md-5.workspace = true
[dev-dependencies]
tempfile.workspace = true
backend/windmill-worker-volumes/src/lib.rs
+544
View File
@@ -0,0 +1,544 @@
#[cfg(feature = "private")]
mod volume_ee;
mod volume_oss;
pub use volume_oss::*;
pub use object_store::ObjectStore as DynObjectStore;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
pub const MAX_VOLUMES_PER_JOB: usize = 10;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileEntry {
pub size: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub md5: Option<String>,
}
pub fn compute_md5_hex(data: &[u8]) -> String {
use md5::{Digest, Md5};
let result = Md5::digest(data);
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut hex = String::with_capacity(32);
for &b in result.iter() {
hex.push(HEX[(b >> 4) as usize] as char);
hex.push(HEX[(b & 0x0f) as usize] as char);
}
hex
}
/// Extract an MD5 hash from an S3 ETag, if it's a simple (non-multipart) ETag.
pub fn etag_to_md5(e_tag: Option<&str>) -> Option<String> {
let tag = e_tag?.trim_matches('"');
// Multipart ETags contain a '-' (e.g. "abc123-5"), skip those
if tag.contains('-') || tag.is_empty() {
return None;
}
Some(tag.to_string())
}
lazy_static::lazy_static! {
static ref ARGS_INTERPOLATION_RE: regex::Regex =
regex::Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap();
static ref VALID_VOLUME_NAME_RE: regex::Regex =
regex::Regex::new(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]$").unwrap();
}
#[derive(Debug, Clone, PartialEq)]
pub struct VolumeMount {
pub name: String,
pub target: String,
}
pub struct VolumeState {
pub mount: VolumeMount,
pub local_dir: PathBuf,
pub manifest: HashMap<String, FileEntry>,
pub symlinks: HashMap<String, String>,
}
pub struct DownloadStats {
pub total_files: usize,
pub from_cache: usize,
pub downloaded: usize,
}
pub struct SyncStats {
pub new_size_bytes: u64,
pub file_count: usize,
pub uploaded: usize,
pub skipped: usize,
}
pub fn validate_volume_name(name: &str) -> Result<(), String> {
if name.contains("..") {
return Err(format!(
"Volume name '{}' contains '..' which is not allowed",
name
));
}
if !VALID_VOLUME_NAME_RE.is_match(name) {
return Err(format!(
"Volume name '{}' is invalid. Names must be 2-255 characters, \
start and end with alphanumeric, and contain only alphanumeric, '.', '_', or '-'",
name
));
}
Ok(())
}
const ALLOWED_ABSOLUTE_PREFIXES: &[&str] = &["/tmp/", "/mnt/", "/opt/", "/home/", "/data/"];
pub fn validate_volume_target(target: &str) -> Result<(), String> {
if target.split('/').any(|seg| seg == "..") {
return Err(format!(
"Volume target '{target}' contains '..' segments which is not allowed"
));
}
if target.starts_with('/')
&& !ALLOWED_ABSOLUTE_PREFIXES
.iter()
.any(|p| target.starts_with(p))
{
return Err(format!(
"Volume target '{target}' must be a relative path or start with one of: {}",
ALLOWED_ABSOLUTE_PREFIXES.join(", ")
));
}
Ok(())
}
pub fn validate_volume_mounts(mounts: &[VolumeMount]) -> Result<(), String> {
if mounts.len() > MAX_VOLUMES_PER_JOB {
return Err(format!(
"Too many volume mounts ({}, max {})",
mounts.len(),
MAX_VOLUMES_PER_JOB
));
}
let mut seen_names = HashSet::new();
let mut seen_targets = HashSet::new();
for v in mounts {
if !seen_names.insert(&v.name) {
return Err(format!("Duplicate volume name: '{}'", v.name));
}
if !seen_targets.insert(&v.target) {
return Err(format!("Duplicate volume target: '{}'", v.target));
}
}
Ok(())
}
pub fn interpolate_volume_name(
name: &str,
args: Option<&HashMap<String, Box<serde_json::value::RawValue>>>,
workspace_id: &str,
) -> String {
let name = name.replace("$workspace", workspace_id);
if !name.contains("$args[") {
return name;
}
let Some(args) = args else {
return name;
};
let mut result = name.clone();
for cap in ARGS_INTERPOLATION_RE.captures_iter(&name) {
let full_match = cap.get(0).unwrap().as_str();
let arg_name = cap.get(1).unwrap().as_str();
let arg_value = if arg_name.contains('.') {
let parts: Vec<&str> = arg_name.split('.').collect();
let root = parts[0];
let mut value = args
.get(root)
.map(|x| x.get().to_string())
.unwrap_or_default();
for part in parts.iter().skip(1) {
if let Ok(obj) = serde_json::from_str::<serde_json::Value>(&value) {
value = obj
.get(part)
.map(|v| v.to_string())
.unwrap_or_default()
.to_string();
} else {
value = String::new();
break;
}
}
value.trim_matches('"').to_string()
} else {
args.get(arg_name)
.map(|x| x.get().trim_matches('"').to_string())
.unwrap_or_default()
};
result = result.replace(full_match, &arg_value);
}
result
}
pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec<VolumeMount> {
let mut volumes = Vec::new();
for line in content.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
if !trimmed.starts_with(comment_prefix) {
break;
}
let after_prefix = trimmed[comment_prefix.len()..].trim();
if let Some(rest) = after_prefix.strip_prefix("volume:") {
let rest = rest.trim();
let mut parts = rest.splitn(2, char::is_whitespace);
if let (Some(name), Some(target)) = (parts.next(), parts.next()) {
let name = name.trim();
let target = target.trim();
if !name.is_empty() && !target.is_empty() {
volumes
.push(VolumeMount { name: name.to_string(), target: target.to_string() });
}
}
}
}
volumes
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_python_single_volume() {
let content = "# volume: mydata /tmp/data\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
);
}
#[test]
fn parse_typescript_single_volume() {
let content = "// volume: mydata /tmp/data\nexport function main() {}";
let result = parse_volume_annotations(content, "//");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
);
}
#[test]
fn parse_multiple_volumes() {
let content = "# volume: data1 /tmp/data1\n# volume: data2 /tmp/data2\n# volume: models /opt/models\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![
VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() },
VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() },
VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() },
]
);
}
#[test]
fn parse_mixed_annotations_and_volumes() {
let content = "# sandbox\n# volume: mydata /tmp/data\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
);
}
#[test]
fn parse_no_volumes() {
let content = "# sandbox\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert!(result.is_empty());
}
#[test]
fn parse_empty_content() {
let result = parse_volume_annotations("", "#");
assert!(result.is_empty());
}
#[test]
fn parse_stops_at_non_comment_line() {
let content =
"# volume: data1 /tmp/data1\ndef main():\n # volume: data2 /tmp/data2\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }]
);
}
#[test]
fn parse_skips_blank_lines_in_header() {
let content =
"# volume: data1 /tmp/data1\n\n# volume: data2 /tmp/data2\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![
VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() },
VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() },
]
);
}
#[test]
fn parse_ignores_malformed_volume_lines() {
let content =
"# volume:\n# volume: onlyname\n# volume: good /tmp/good\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }]
);
}
#[test]
fn parse_extra_whitespace() {
let content = "# volume: mydata /tmp/data \ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
);
}
#[test]
fn parse_target_with_spaces_in_path() {
let content = "// volume: mydata /tmp/my data dir\nexport function main() {}";
let result = parse_volume_annotations(content, "//");
assert_eq!(
result,
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/my data dir".to_string(),
}]
);
}
#[test]
fn parse_volume_with_dashes_and_underscores() {
let content = "# volume: my-data_v2 /tmp/data\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }]
);
}
#[test]
fn interpolate_workspace() {
let name = "$workspace-data";
let result = interpolate_volume_name(name, None, "my_ws");
assert_eq!(result, "my_ws-data");
}
#[test]
fn interpolate_args_simple() {
let mut args = HashMap::new();
args.insert(
"env".to_string(),
serde_json::value::RawValue::from_string("\"prod\"".to_string()).unwrap(),
);
let result = interpolate_volume_name("data-$args[env]", Some(&args), "ws");
assert_eq!(result, "data-prod");
}
#[test]
fn interpolate_args_and_workspace() {
let mut args = HashMap::new();
args.insert(
"env".to_string(),
serde_json::value::RawValue::from_string("\"staging\"".to_string()).unwrap(),
);
let result = interpolate_volume_name("$workspace-$args[env]-cache", Some(&args), "acme");
assert_eq!(result, "acme-staging-cache");
}
#[test]
fn interpolate_no_placeholders() {
let result = interpolate_volume_name("plain-name", None, "ws");
assert_eq!(result, "plain-name");
}
#[test]
fn interpolate_missing_arg() {
let args = HashMap::new();
let result = interpolate_volume_name("data-$args[missing]", Some(&args), "ws");
assert_eq!(result, "data-");
}
#[test]
fn interpolate_nested_arg() {
let mut args = HashMap::new();
args.insert(
"config".to_string(),
serde_json::value::RawValue::from_string(
r#"{"env": "prod", "region": "us-east"}"#.to_string(),
)
.unwrap(),
);
let result = interpolate_volume_name(
"data-$args[config.env]-$args[config.region]",
Some(&args),
"ws",
);
assert_eq!(result, "data-prod-us-east");
}
#[test]
fn parse_wrong_prefix_returns_empty() {
let content = "# volume: mydata /tmp/data\ndef main():\n pass";
let result = parse_volume_annotations(content, "//");
assert!(result.is_empty());
}
#[test]
fn parse_relative_path() {
let content = "// volume: agent-memory .claude\nexport function main() {}";
let result = parse_volume_annotations(content, "//");
assert_eq!(
result,
vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }]
);
}
#[test]
fn parse_relative_nested_path() {
let content = "# volume: data data/models\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }]
);
}
#[test]
fn validate_valid_names() {
assert!(validate_volume_name("mydata").is_ok());
assert!(validate_volume_name("my-data_v2").is_ok());
assert!(validate_volume_name("acme-staging-cache").is_ok());
assert!(validate_volume_name("a1").is_ok());
assert!(validate_volume_name("data.v2").is_ok());
assert!(validate_volume_name("A0").is_ok());
}
#[test]
fn validate_rejects_path_traversal() {
assert!(validate_volume_name("../other-workspace").is_err());
assert!(validate_volume_name("data/../secrets").is_err());
assert!(validate_volume_name("a..b").is_err());
}
#[test]
fn validate_rejects_special_start_end() {
assert!(validate_volume_name("-data").is_err());
assert!(validate_volume_name("data-").is_err());
assert!(validate_volume_name(".data").is_err());
assert!(validate_volume_name("data.").is_err());
assert!(validate_volume_name("_data").is_err());
}
#[test]
fn validate_rejects_path_separators() {
assert!(validate_volume_name("data/secrets").is_err());
assert!(validate_volume_name("data\\secrets").is_err());
}
#[test]
fn validate_rejects_too_short() {
assert!(validate_volume_name("").is_err());
assert!(validate_volume_name("a").is_err());
}
#[test]
fn validate_rejects_too_long() {
let long_name = format!("a{}a", "b".repeat(254));
assert!(validate_volume_name(&long_name).is_err());
}
#[test]
fn validate_rejects_spaces_and_special() {
assert!(validate_volume_name("my data").is_err());
assert!(validate_volume_name("my@data").is_err());
assert!(validate_volume_name("my$data").is_err());
}
#[test]
fn validate_target_allows_relative() {
assert!(validate_volume_target("data").is_ok());
assert!(validate_volume_target("data/models").is_ok());
assert!(validate_volume_target(".claude").is_ok());
}
#[test]
fn validate_target_allows_safe_absolute() {
assert!(validate_volume_target("/tmp/data").is_ok());
assert!(validate_volume_target("/mnt/data").is_ok());
assert!(validate_volume_target("/opt/models").is_ok());
assert!(validate_volume_target("/home/user/data").is_ok());
assert!(validate_volume_target("/data/cache").is_ok());
}
#[test]
fn validate_target_rejects_dangerous_absolute() {
assert!(validate_volume_target("/etc/passwd").is_err());
assert!(validate_volume_target("/proc/self").is_err());
assert!(validate_volume_target("/sys/fs").is_err());
assert!(validate_volume_target("/dev/null").is_err());
assert!(validate_volume_target("/usr/bin").is_err());
assert!(validate_volume_target("/var/log").is_err());
}
#[test]
fn validate_target_rejects_traversal() {
assert!(validate_volume_target("../../etc").is_err());
assert!(validate_volume_target("data/../../../etc").is_err());
assert!(validate_volume_target("/tmp/../etc/passwd").is_err());
}
#[test]
fn validate_mounts_rejects_too_many() {
let mounts: Vec<VolumeMount> = (0..11)
.map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) })
.collect();
assert!(validate_volume_mounts(&mounts).is_err());
}
#[test]
fn validate_mounts_rejects_duplicate_name() {
let mounts = vec![
VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() },
VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() },
];
assert!(validate_volume_mounts(&mounts).is_err());
}
#[test]
fn validate_mounts_rejects_duplicate_target() {
let mounts = vec![
VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() },
VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() },
];
assert!(validate_volume_mounts(&mounts).is_err());
}
#[test]
fn validate_mounts_ok() {
let mounts = vec![
VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() },
VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() },
];
assert!(validate_volume_mounts(&mounts).is_ok());
}
}
backend/windmill-worker-volumes/src/volume_oss.rs
+116
View File
@@ -0,0 +1,116 @@
#[cfg(feature = "private")]
pub use crate::volume_ee::*;
#[cfg(not(feature = "private"))]
use crate::{DownloadStats, SyncStats, VolumeMount, VolumeState};
#[cfg(not(feature = "private"))]
use object_store::ObjectStore;
#[cfg(not(feature = "private"))]
use std::path::Path;
#[cfg(not(feature = "private"))]
use std::sync::Arc;
#[cfg(not(feature = "private"))]
use windmill_common::error;
#[cfg(not(feature = "private"))]
pub async fn download_volume(
_client: Arc<dyn ObjectStore>,
_volume: &VolumeMount,
_job_dir: &str,
_workspace_id: &str,
) -> error::Result<(VolumeState, DownloadStats)> {
Err(error::Error::internal_err(
"Volumes are not available in this build".to_string(),
))
}
#[cfg(not(feature = "private"))]
pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String {
String::new()
}
#[cfg(not(feature = "private"))]
pub async fn sync_volume_back(
_client: Arc<dyn ObjectStore>,
_state: &VolumeState,
_workspace_id: &str,
) -> error::Result<SyncStats> {
Err(error::Error::internal_err(
"Volumes are not available in this build".to_string(),
))
}
#[cfg(not(feature = "private"))]
pub fn walk_dir(dir: &Path) -> std::io::Result<Vec<std::path::PathBuf>> {
let mut result = Vec::new();
walk_dir_inner(dir, &mut result)?;
Ok(result)
}
#[cfg(not(feature = "private"))]
fn walk_dir_inner(dir: &Path, result: &mut Vec<std::path::PathBuf>) -> std::io::Result<()> {
if !dir.is_dir() {
return Ok(());
}
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
let meta = match std::fs::symlink_metadata(&path) {
Ok(m) => m,
Err(_) => continue,
};
if meta.is_dir() {
walk_dir_inner(&path, result)?;
} else if meta.is_file() {
result.push(path);
}
}
Ok(())
}
#[cfg(not(feature = "private"))]
pub fn collect_symlinks(dir: &Path) -> std::collections::HashMap<String, String> {
let mut symlinks = std::collections::HashMap::new();
collect_symlinks_inner(dir, dir, &mut symlinks);
symlinks
}
#[cfg(not(feature = "private"))]
fn collect_symlinks_inner(
base: &Path,
dir: &Path,
symlinks: &mut std::collections::HashMap<String, String>,
) {
let entries = match std::fs::read_dir(dir) {
Ok(e) => e,
Err(_) => return,
};
for entry in entries {
let entry = match entry {
Ok(e) => e,
Err(_) => continue,
};
let path = entry.path();
let meta = match std::fs::symlink_metadata(&path) {
Ok(m) => m,
Err(_) => continue,
};
if meta.file_type().is_symlink() {
if let Ok(target) = std::fs::read_link(&path) {
let relative = path
.strip_prefix(base)
.unwrap_or(&path)
.to_string_lossy()
.replace('\\', "/");
symlinks.insert(relative, target.to_string_lossy().to_string());
}
} else if meta.is_dir() {
collect_symlinks_inner(base, &path, symlinks);
}
}
}
#[cfg(not(feature = "private"))]
pub fn restore_symlinks(_dir: &Path, _symlinks: &std::collections::HashMap<String, String>) {
// No-op in OSS build
}
backend/windmill-worker/Cargo.toml
+3 -2
View File
@@ -10,10 +10,10 @@ path = "src/lib.rs"
[features]
default = []
private = []
private = ["windmill-worker-volumes/private", "windmill-queue/private"]
mcp = ["dep:windmill-mcp"]
prometheus = ["dep:prometheus", "windmill-common/prometheus"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"]
mssql = ["dep:tiberius"]
mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth
mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth
@@ -47,6 +47,7 @@ windmill-audit.workspace = true # there isn't really a reason for audit-worth ac
windmill-common = { workspace = true, default-features = false }
windmill-types.workspace = true
windmill-object-store.workspace = true
windmill-worker-volumes.workspace = true
windmill-jseval.workspace = true
windmill-runtime-nativets = { workspace = true, optional = true }
windmill-mcp = { workspace = true, optional = true }
backend/windmill-worker/nsjail/run.bun.config.proto
+12
View File
@@ -14,6 +14,18 @@ clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
clone_newcgroup: false
uidmap {
inside_id: "1000"
outside_id: ""
count: 1
}
gidmap {
inside_id: "1000"
outside_id: ""
count: 1
}
skip_setsid: true
keep_caps: false
keep_env: true
backend/windmill-worker/src/bun_executor.rs
+14 -2
View File
@@ -22,7 +22,7 @@ use crate::{
handle_child::handle_child,
is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR,
BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH,
NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
};
use windmill_common::{
@@ -990,6 +990,14 @@ pub async fn handle_bun_job(
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
if annotation.sandbox && NSJAIL_AVAILABLE.is_none() {
return Err(error::Error::ExecutionErr(
"Script has //sandbox annotation but nsjail is not available on this worker. \
Please ensure nsjail is installed or remove the //sandbox annotation."
.to_string(),
));
}
let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = (
maybe_lock.get_lock(),
!annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(),
@@ -1161,6 +1169,10 @@ pub async fn handle_bun_job(
init_logs = format!("\n{}{}", cache_logs, init_logs);
}
if annotation.sandbox {
init_logs.push_str("sandbox mode (nsjail)\n");
}
let write_wrapper_f = async {
if !has_bundle_cache && annotation.native {
return Ok(()) as error::Result<()>;
@@ -1485,7 +1497,7 @@ try {{
append_logs(&job.id, &job.workspace_id, init_logs, conn).await;
//do not cache local dependencies
let child = if is_sandboxing_enabled() {
let child = if is_sandboxing_enabled() || annotation.sandbox {
let _ = write_file(
job_dir,
"run.config.proto",
backend/windmill-worker/src/common.rs
+6 -2
View File
@@ -886,7 +886,7 @@ pub async fn cached_result_path(
}
#[cfg(feature = "parquet")]
async fn get_workspace_s3_resource_path(
pub(crate) async fn get_workspace_s3_resource_path(
db: &DB,
client: &AuthedClient,
workspace_id: &str,
@@ -948,7 +948,11 @@ async fn get_workspace_s3_resource_path(
)
}
Some(LargeFileStorage::FilesystemStorage(fs)) => {
(StorageResourceType::Filesystem, fs.root_path.clone())
return Ok(Some(
windmill_object_store::ObjectStoreResource::Filesystem(
windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() },
),
));
}
None => {
return Ok(None);
backend/windmill-worker/src/deno_executor.rs
+8 -2
View File
@@ -17,6 +17,7 @@ use crate::{
NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
};
use windmill_common::client::AuthedClient;
use windmill_common::worker::TypeScriptAnnotations;
use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL};
@@ -231,8 +232,13 @@ pub async fn handle_deno_job(
occupancy_metrics: &mut OccupancyMetrics,
has_stream: &mut bool,
) -> error::Result<Box<RawValue>> {
let annotations = TypeScriptAnnotations::parse(inner_content);
// let mut start = Instant::now();
let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
let mut logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
if annotations.sandbox {
logs1.push_str("sandbox mode (nsjail)\n");
}
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
let main_override = job.script_entrypoint_override.as_deref();
@@ -451,7 +457,7 @@ try {{
for flag in deno_flags {
args.push(flag);
}
} else if is_sandboxing_enabled() {
} else if is_sandboxing_enabled() || annotations.sandbox {
args.push("--allow-net");
args.push("--allow-sys");
args.push(allow_read.as_str());
backend/windmill-worker/src/lib.rs
+3
View File
@@ -70,6 +70,9 @@ mod sanitized_sql_params;
mod schema;
pub mod sql_utils;
mod universal_pkg_installer;
#[cfg(feature = "private")]
mod volume_ee;
mod volume_oss;
mod worker;
mod worker_flow;
mod worker_lockfiles;
backend/windmill-worker/src/python_executor.rs
+20 -14
View File
@@ -134,8 +134,8 @@ use crate::{
handle_child::handle_child,
is_sandboxing_enabled, read_ee_registry,
worker_utils::ping_job_status,
PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR,
PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR,
UV_INDEX_STRATEGY,
};
use windmill_common::client::AuthedClient;
@@ -567,6 +567,14 @@ pub async fn handle_python_job(
let annotations = PythonAnnotations::parse(inner_content);
if annotations.sandbox && NSJAIL_AVAILABLE.is_none() {
return Err(Error::ExecutionErr(
"Script has #sandbox annotation but nsjail is not available on this worker. \
Please ensure nsjail is installed or remove the #sandbox annotation."
.to_string(),
));
}
let (py_version, mut additional_python_paths) = handle_python_deps(
job_dir,
requirements_o,
@@ -605,16 +613,14 @@ pub async fn handle_python_job(
}
{
append_logs(
&job.id,
&job.workspace_id,
format!(
"\n\n--- PYTHON ({}) CODE EXECUTION ---\n",
py_version.clone().to_string()
),
conn,
)
.await;
let mut logs = format!(
"\n\n--- PYTHON ({}) CODE EXECUTION ---\n",
py_version.clone().to_string()
);
if annotations.sandbox {
logs.push_str("sandbox mode (nsjail)\n");
}
append_logs(&job.id, &job.workspace_id, logs, conn).await;
}
let (
import_loader,
@@ -784,7 +790,7 @@ except BaseException as e:
#[cfg(windows)]
let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";");
if is_sandboxing_enabled() {
if is_sandboxing_enabled() || annotations.sandbox {
let shared_deps = additional_python_paths
.into_iter()
.map(|pp| {
@@ -828,7 +834,7 @@ mount {{
job.id
);
let child = if is_sandboxing_enabled() {
let child = if is_sandboxing_enabled() || annotations.sandbox {
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
nsjail_cmd
.current_dir(job_dir)
backend/windmill-worker/src/volume_oss.rs
+112
View File
@@ -0,0 +1,112 @@
#[cfg(feature = "private")]
pub(crate) use crate::volume_ee::*;
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
pub(crate) struct LeaseRenewalGuard(pub Option<tokio::task::JoinHandle<()>>);
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
impl Drop for LeaseRenewalGuard {
fn drop(&mut self) {
if let Some(handle) = self.0.take() {
handle.abort();
}
}
}
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
pub(crate) struct VolumeSetupResult {
pub states: Vec<windmill_worker_volumes::VolumeState>,
pub writable: Vec<bool>,
pub client: Option<std::sync::Arc<dyn windmill_worker_volumes::DynObjectStore>>,
pub lease_renewal: LeaseRenewalGuard,
}
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
#[allow(dead_code)]
pub(crate) fn setup_volume_mount_paths(
_volume: &windmill_worker_volumes::VolumeMount,
_state: &windmill_worker_volumes::VolumeState,
_job_dir: &str,
_language: windmill_common::scripts::ScriptLang,
_envs: &mut std::collections::HashMap<String, String>,
_shared_mount: &mut String,
) -> windmill_common::error::Result<()> {
Err(windmill_common::error::Error::internal_err(
"Volumes are not available in OSS".to_string(),
))
}
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
pub(crate) async fn setup_volumes_sql_worker(
_volume_mounts: &[windmill_worker_volumes::VolumeMount],
_db: &windmill_common::DB,
_workspace_id: &str,
_job_id: uuid::Uuid,
_permissioned_as: &str,
_worker_name: &str,
_job_dir: &str,
_client: &windmill_common::client::AuthedClient,
_conn: &windmill_common::worker::Connection,
_language: windmill_common::scripts::ScriptLang,
_envs: &mut std::collections::HashMap<String, String>,
_shared_mount: &mut String,
) -> windmill_common::error::Result<VolumeSetupResult> {
Err(windmill_common::error::Error::internal_err(
"Volumes are not available in OSS".to_string(),
))
}
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
pub(crate) async fn setup_volumes_http_worker(
_volume_mounts: &[windmill_worker_volumes::VolumeMount],
_http: &windmill_common::worker::HttpClient,
_workspace_id: &str,
_job_id: uuid::Uuid,
_permissioned_as: &str,
_canceled_by: &Option<String>,
_worker_name: &str,
_job_dir: &str,
_conn: &windmill_common::worker::Connection,
_language: windmill_common::scripts::ScriptLang,
_envs: &mut std::collections::HashMap<String, String>,
_shared_mount: &mut String,
) -> windmill_common::error::Result<VolumeSetupResult> {
Err(windmill_common::error::Error::internal_err(
"Volumes are not available in OSS".to_string(),
))
}
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
pub(crate) async fn sync_volumes_sql_worker(
_volume_states: &[windmill_worker_volumes::VolumeState],
_volume_writable: &[bool],
_vol_client: &std::sync::Arc<dyn windmill_worker_volumes::DynObjectStore>,
_db: &windmill_common::DB,
_workspace_id: &str,
_job_id: uuid::Uuid,
_worker_name: &str,
_conn: &windmill_common::worker::Connection,
_job_succeeded: bool,
) {
}
#[cfg(not(feature = "private"))]
#[cfg(feature = "parquet")]
pub(crate) async fn sync_volumes_http_worker(
_volume_states: &[windmill_worker_volumes::VolumeState],
_volume_writable: &[bool],
_http: &windmill_common::worker::HttpClient,
_workspace_id: &str,
_job_id: uuid::Uuid,
_worker_name: &str,
_conn: &windmill_common::worker::Connection,
_job_succeeded: bool,
) {
}
backend/windmill-worker/src/worker.rs
+159 -2
View File
@@ -4161,7 +4161,8 @@ pub async fn run_language_executor(
job.id
);
let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) {
#[allow(unused_mut)]
let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) {
let folder = if job.script_lang == Some(ScriptLang::Go) {
"/go"
} else {
@@ -4183,7 +4184,8 @@ mount {{
// println!("handle lang job {:?}", SystemTime::now());
let envs = build_envs(envs.as_ref())?;
#[allow(unused_mut)]
let mut envs = build_envs(envs.as_ref())?;
let Some(language) = language else {
return Err(Error::ExecutionErr(
@@ -4219,6 +4221,106 @@ mount {{
}
}
// Volume mount setup (requires workspace S3 storage; CE has file count/size limits)
#[cfg(feature = "parquet")]
let volume_mounts = {
let comment_prefix = match language {
ScriptLang::Python3
| ScriptLang::Bash
| ScriptLang::Powershell
| ScriptLang::Ansible
| ScriptLang::Ruby => "#",
ScriptLang::Deno
| ScriptLang::Bun
| ScriptLang::Bunnative
| ScriptLang::Nativets
| ScriptLang::Go => "//",
_ => "",
};
let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix);
let args_ref = job.args.as_ref().map(|a| &**a);
let mut interpolated = Vec::new();
for mut v in raw_mounts {
v.name = windmill_worker_volumes::interpolate_volume_name(
&v.name,
args_ref,
&job.workspace_id,
);
if let Err(e) = windmill_worker_volumes::validate_volume_name(&v.name) {
return Err(Error::ExecutionErr(e));
}
if let Err(e) = windmill_worker_volumes::validate_volume_target(&v.target) {
return Err(Error::ExecutionErr(e));
}
interpolated.push(v);
}
if let Err(e) = windmill_worker_volumes::validate_volume_mounts(&interpolated) {
return Err(Error::ExecutionErr(e));
}
interpolated
};
#[cfg(feature = "parquet")]
let mut volume_setup = crate::volume_oss::VolumeSetupResult {
states: Vec::new(),
writable: Vec::new(),
client: None,
lease_renewal: crate::volume_oss::LeaseRenewalGuard(None),
};
#[cfg(feature = "parquet")]
if !volume_mounts.is_empty() {
let vol_summary: Vec<String> = volume_mounts
.iter()
.map(|v| format!("'{}' -> {}", v.name, v.target))
.collect();
append_logs(
&job.id,
&job.workspace_id,
format!(
"\n--- VOLUME MOUNTS ---\nPulling {} volume(s): {}\n",
volume_mounts.len(),
vol_summary.join(", "),
),
conn,
)
.await;
if let Connection::Sql(db) = conn {
volume_setup = crate::volume_oss::setup_volumes_sql_worker(
&volume_mounts,
db,
&job.workspace_id,
job.id,
&job.permissioned_as,
worker_name,
job_dir,
client,
conn,
language,
&mut envs,
&mut shared_mount,
)
.await?;
} else if let Connection::Http(http) = conn {
volume_setup = crate::volume_oss::setup_volumes_http_worker(
&volume_mounts,
http,
&job.workspace_id,
job.id,
&job.permissioned_as,
&job.canceled_by,
worker_name,
job_dir,
conn,
language,
&mut envs,
&mut shared_mount,
)
.await?;
}
}
// Box::pin all language handlers to prevent large match enum on stack
let result: error::Result<Box<RawValue>> = match language {
ScriptLang::Python3 => {
@@ -4630,6 +4732,61 @@ mount {{
// for related places search: ADD_NEW_LANG
_ => panic!("unreachable, language is not supported: {language:#?}"),
};
// Volume sync-back and lease release
#[cfg(feature = "parquet")]
if !volume_setup.states.is_empty() {
// Stop lease renewal before sync-back
volume_setup.lease_renewal.0.take().map(|h| h.abort());
if let Some(ref vol_client) = volume_setup.client {
if let Connection::Sql(db) = conn {
crate::volume_oss::sync_volumes_sql_worker(
&volume_setup.states,
&volume_setup.writable,
vol_client,
db,
&job.workspace_id,
job.id,
worker_name,
conn,
result.is_ok(),
)
.await;
}
}
if let Connection::Http(http) = conn {
crate::volume_oss::sync_volumes_http_worker(
&volume_setup.states,
&volume_setup.writable,
http,
&job.workspace_id,
job.id,
worker_name,
conn,
result.is_ok(),
)
.await;
}
// Clean up absolute-path symlinks created by setup_volume_mount_paths
if !is_sandboxing_enabled() {
for state in &volume_setup.states {
#[cfg(unix)]
if state.mount.target.starts_with('/') {
let target_path = std::path::Path::new(&state.mount.target);
if target_path
.symlink_metadata()
.map(|m| m.file_type().is_symlink())
.unwrap_or(false)
{
std::fs::remove_file(target_path).ok();
}
}
}
}
}
tracing::info!(
workspace_id = %job.workspace_id,
is_ok = result.is_ok(),
docker/DockerfileSlim
+5
View File
@@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \
&& bun install -g windmill-cli \
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
# Install Claude Code CLI (used by claude sandbox scripts)
# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root)
RUN curl -fsSL https://claude.ai/install.sh | bash \
&& cp /root/.local/share/claude/versions/* /usr/bin/claude
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
docker/DockerfileSlimEe
+5
View File
@@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \
&& bun install -g windmill-cli \
&& ln -s $(bun pm bin -g)/wmill /usr/bin/wmill
# Install Claude Code CLI (used by claude sandbox scripts)
# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root)
RUN curl -fsSL https://claude.ai/install.sh | bash \
&& cp /root/.local/share/claude/versions/* /usr/bin/claude
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
frontend/src/lib/components/ApiConnectForm.svelte
+7
View File
@@ -15,6 +15,7 @@
import GitHubAppIntegration from './GitHubAppIntegration.svelte'
import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte'
import { isCloudHosted } from '$lib/cloud'
import ResourceGen from './copilot/ResourceGen.svelte'
interface Props {
resourceType: string
@@ -149,6 +150,12 @@
}}
class="as-json-toggle"
/>
<ResourceGen
bind:args
resourceType={resourceType}
resourceSchema={notFound ? undefined : schema}
isFileset={resourceTypeInfo?.is_fileset ?? false}
/>
<TestConnection {resourceType} {args} />
{#if resourceType == 'postgresql'}
<Popover
frontend/src/lib/components/EditorBar.svelte
+1 -1
View File
@@ -79,7 +79,7 @@
iconOnly?: boolean
validCode?: boolean
kind?: 'script' | 'trigger' | 'approval'
template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative'
template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox'
collabMode?: boolean
collabLive?: boolean
collabUsers?: { name: string }[]
frontend/src/lib/components/ExploreAssetButton.svelte
+11 -4
View File
@@ -7,6 +7,7 @@
asset.kind === 'ducklake' ||
asset.kind === 'datatable' ||
asset.kind === 's3object' ||
asset.kind === 'volume' ||
(asset.kind === 'resource' && isDbType(_resourceMetadata?.resource_type))
)
}
@@ -17,9 +18,10 @@
import { formatAsset, type Asset } from '$lib/components/assets/lib'
import { Button, ButtonType } from '$lib/components/common'
import S3FilePicker from '$lib/components/S3FilePicker.svelte'
import { globalDbManagerDrawer, userStore } from '$lib/stores'
import { VolumeService } from '$lib/gen'
import { globalDbManagerDrawer, userStore, workspaceStore } from '$lib/stores'
import { isS3Uri } from '$lib/utils'
import { Database, File } from 'lucide-svelte'
import { Database, File, HardDriveIcon } from 'lucide-svelte'
import DucklakeIcon from './icons/DucklakeIcon.svelte'
const {
@@ -66,6 +68,9 @@
})
} else if (asset.kind === 's3object' && isS3Uri(assetUri)) {
s3FilePicker?.open(assetUri)
} else if (asset.kind === 'volume') {
const storage = (await VolumeService.getVolumeStorage({ workspace: $workspaceStore! })) ?? undefined
s3FilePicker?.open({ s3: `volumes/${$workspaceStore}/${asset.path}/`, storage })
} else if (asset.kind === 'ducklake') {
let ducklake = asset.path.split('/')[0]
let specificTable = asset.path.split('/')[1] as string | undefined
@@ -93,9 +98,11 @@
? { icon: Database }
: asset.kind === 'ducklake'
? { icon: DucklakeIcon }
: undefined}
: asset.kind === 'volume'
? { icon: HardDriveIcon }
: undefined}
>
{#if asset.kind === 's3object'}
{#if asset.kind === 's3object' || asset.kind === 'volume'}
<span class:hidden={noText}>Explore</span>
{:else if asset.kind === 'resource' || asset.kind === 'ducklake' || asset.kind === 'datatable'}
<span class:hidden={noText}>Manage</span>
frontend/src/lib/components/FilesetEditor.svelte
+27
View File
@@ -46,6 +46,9 @@
}
}
// Track the last args we wrote so we can detect external changes.
let lastWrittenArgs: Record<string, any> = $state(args ?? {})
// Sync files → args, overlaying current editContent for the active file.
// This avoids spreading a new files object on every keystroke.
$effect(() => {
@@ -58,9 +61,33 @@
newArgs[argKey] = key === currentKey ? currentContent : value
}
}
lastWrittenArgs = newArgs
args = newArgs
})
// Sync args → files when args changes externally (e.g. from AI generation).
$effect(() => {
const currentArgs = args
if (currentArgs === lastWrittenArgs) return
// Check if the args object is actually different
const currentKeys = Object.keys(currentArgs ?? {}).sort().join('\0')
const lastKeys = Object.keys(lastWrittenArgs ?? {}).sort().join('\0')
if (currentKeys === lastKeys) {
const allSame = Object.entries(currentArgs ?? {}).every(
([k, v]) => lastWrittenArgs[k] === v
)
if (allSame) return
}
const newFiles = Object.fromEntries(
Object.entries(currentArgs ?? {}).map(([k, v]) => ['/' + k, String(v ?? '')])
)
files = newFiles
lastWrittenArgs = currentArgs
const firstFile = Object.keys(newFiles).find((k) => !k.endsWith('/'))
selectedPath = firstFile ?? '/'
editContent = firstFile ? (newFiles[firstFile] ?? '') : ''
})
function inferLang(filePath: string): string {
const ext = filePath.split('.').pop()?.toLowerCase()
if (!ext) return 'plaintext'
frontend/src/lib/components/ResourceEditor.svelte
+19 -3
View File
@@ -24,6 +24,7 @@
import GitHubAppIntegration from './GitHubAppIntegration.svelte'
import Button from './common/button/Button.svelte'
import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte'
import ResourceGen from './copilot/ResourceGen.svelte'
interface Props {
canSave?: boolean
@@ -270,6 +271,13 @@
right: 'As JSON'
}}
/>
<ResourceGen
bind:args
resourceType={resource_type}
resourceName={path}
resourceDescription={description}
{resourceSchema}
/>
{#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'}
<TestTriggerConnection kind={resourceToEdit?.resource_type} args={{ connection: args }} />
{:else}
@@ -296,9 +304,17 @@
{#if loadingSchema}
<Skeleton layout={[[4]]} />
{:else if !viewJsonSchema && resourceTypeInfo?.is_fileset}
<h5 class="mt-1 inline-flex items-center gap-4">
Fileset
</h5>
<div class="mt-1 flex items-center gap-2">
<h5 class="inline-flex items-center gap-4">Fileset</h5>
<ResourceGen
bind:args
resourceType={resource_type}
resourceName={path}
resourceDescription={description}
{resourceSchema}
isFileset
/>
</div>
<FilesetEditor bind:args />
{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties}
{#if resourceTypeInfo?.format_extension}
frontend/src/lib/components/S3FilePicker.svelte
+3
View File
@@ -13,6 +13,7 @@
interface Props {
fromWorkspaceSettings?: boolean
readOnlyMode: boolean
allowDelete?: boolean
initialFileKey?: { s3: string; storage?: string } | undefined
selectedFileKey?: { s3: string; storage?: string } | undefined
folderOnly?: boolean
@@ -24,6 +25,7 @@
let {
fromWorkspaceSettings = false,
readOnlyMode,
allowDelete = false,
initialFileKey = $bindable(undefined),
selectedFileKey = $bindable(undefined),
folderOnly = false,
@@ -94,6 +96,7 @@
}}
{fromWorkspaceSettings}
{readOnlyMode}
{allowDelete}
bind:initialFileKey
bind:selectedFileKey
bind:workspaceSettingsInitialized
frontend/src/lib/components/S3FilePickerInner.svelte
+19 -7
View File
@@ -81,6 +81,7 @@
count: number
}
>
allowDelete?: boolean
replaceUnauthorizedWarning?: Snippet
listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise<ListStoredFilesResponse>
loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise<LoadFilePreviewResponse>
@@ -102,11 +103,12 @@
folderOnly = false,
regexFilter = undefined,
hideS3SpecificDetails = false,
rootPath = '',
rootPath: initialRootPath = '',
workspaceSettingsInitialized = $bindable(true),
storage = $bindable(undefined),
uploadModalOpen = $bindable(false),
allFilesByKey = $bindable({}),
allowDelete = false,
replaceUnauthorizedWarning,
listStoredFilesRequest = HelpersService.listStoredFiles,
loadFilePreviewRequest = HelpersService.loadFilePreview,
@@ -116,6 +118,7 @@
testConnectionRequest = HelpersService.datasetStorageTestConnection
}: Props = $props()
let rootPath = $state(initialRootPath)
let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1))
let csvSeparatorChar: string = $state(',')
@@ -263,7 +266,7 @@
}
}
}
displayedFileKeys = displayedFileKeys.sort()
displayedFileKeys = [...new Set(displayedFileKeys)].sort()
fileListLoading = false
fileInfoLoading = false
}
@@ -381,7 +384,7 @@
}
}
}
displayedFileKeys = displayedFileKeys.sort()
displayedFileKeys = [...new Set(displayedFileKeys)].sort()
}
async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) {
@@ -424,9 +427,16 @@
export async function open(_preSelectedFileKey: S3Object | undefined = undefined) {
const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey)
storage = preSelectedFileKey?.storage
if (preSelectedFileKey !== undefined) {
if (preSelectedFileKey !== undefined && preSelectedFileKey.s3.endsWith('/')) {
rootPath = preSelectedFileKey.s3
filter = ''
selectedFileKey = undefined
} else if (preSelectedFileKey !== undefined) {
rootPath = ''
initialFileKey = { ...preSelectedFileKey }
selectedFileKey = { ...preSelectedFileKey }
} else {
rootPath = ''
}
reloadContent()
}
@@ -461,7 +471,7 @@
if (selectedFileKey !== undefined) {
if (allFilesByKey[selectedFileKey.s3] === undefined) {
selectedFileKey = { s3: '', storage }
} else {
} else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') {
loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
}
}
@@ -518,7 +528,7 @@
}
}
}
displayedFileKeys = displayedFileKeys.sort()
displayedFileKeys = [...new Set(displayedFileKeys)].sort()
} else {
selectedFileKey = {
s3: item_key,
@@ -719,8 +729,10 @@
startIcon={{ icon: MoveRight }}
iconOnly={true}
/>
{/if}
{#if !readOnlyMode || allowDelete}
<Button
title="Delete file from S3"
title="Delete file"
variant="default"
on:click={() => {
deletionModalOpen = true
frontend/src/lib/components/ScriptBuilder.svelte
+23 -3
View File
@@ -382,7 +382,7 @@
async function initContent(
language: SupportedLanguage,
kind: Script['kind'] | undefined,
template: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative'
template: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox'
) {
scriptEditor?.disableCollaboration()
const templateScript = await isTemplateScript()
@@ -1159,9 +1159,10 @@
<div class=" grid grid-cols-3 gap-2">
{#each langs as [label, lang] (lang)}
{@const isPicked =
(lang == script.language && template == 'script') ||
(lang == script.language && template != 'bunnative' && template != 'docker' && template != 'claudesandbox') ||
(template == 'bunnative' && lang == 'bunnative') ||
(template == 'docker' && lang == 'docker')}
(template == 'docker' && lang == 'docker') ||
(template == 'claudesandbox' && lang == 'bun')}
<Popover
disablePopup={!enterpriseLangs.includes(lang) || !!$enterpriseLicense}
>
@@ -1194,6 +1195,25 @@
</div>
</Section>
{/if}
<div class="flex items-center gap-2 mt-2">
<span class="text-2xs text-secondary">Template</span>
<Button
size="xs2"
variant="border"
color="light"
startIcon={{
icon: LanguageIcon,
props: { lang: 'claudesandbox', width: 16, height: 16 }
} as ButtonType.Icon}
on:click={() => {
template = 'claudesandbox'
script.language = 'bun'
initContent('bun', script.kind, template)
}}
>
Claude Sandbox
</Button>
</div>
{#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
<Section label="Script kind">
{#snippet header()}
frontend/src/lib/components/ScriptEditor.svelte
+1 -1
View File
@@ -100,7 +100,7 @@
path: string | undefined
lang: Preview['language']
kind?: string | undefined
template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative'
template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox'
tag: string | undefined
initialArgs?: Record<string, any>
fixedOverflowWidgets?: boolean
frontend/src/lib/components/ShareModal.svelte
+7 -2
View File
@@ -35,6 +35,7 @@
| 'postgres_trigger'
| 'gcp_trigger'
| 'email_trigger'
| 'volume'
let kind: Kind
let path: string = $state('')
@@ -53,13 +54,17 @@
let drawer: Drawer | undefined = $state()
let own = $state(false)
export async function openDrawer(newPath: string, kind_l: Kind) {
export async function openDrawer(newPath: string, kind_l: Kind, isOwnerOverride?: boolean) {
path = newPath
kind = kind_l
loadAcls()
loadGroups()
loadUsernames()
loadOwner()
if (isOwnerOverride !== undefined) {
own = isOwnerOverride
} else {
loadOwner()
}
drawer?.openDrawer()
}
frontend/src/lib/components/assets/JobAssetsViewer.svelte
+10 -2
View File
@@ -1,5 +1,5 @@
<script lang="ts">
import { ResourceService, type Job } from '$lib/gen'
import { ResourceService, ScriptService, type Job } from '$lib/gen'
import { inferAssets } from '$lib/infer'
import { workspaceStore } from '$lib/stores'
import { usePromise } from '$lib/svelte5Utils.svelte'
@@ -35,7 +35,15 @@
}
if (job.job_kind === 'script') {
let inferAssetsResult = await inferAssets(job.language!, job.raw_code ?? '')
let code = job.raw_code
if (!code && job.script_hash && $workspaceStore) {
const script = await ScriptService.getScriptByHash({
workspace: $workspaceStore,
hash: job.script_hash
})
code = script.content
}
let inferAssetsResult = await inferAssets(job.language!, code ?? '')
let assets = inferAssetsResult.status === 'ok' ? inferAssetsResult.assets : []
return [...assets, ...parseInputArgsAssets(job.args ?? {})]
}
frontend/src/lib/components/assets/VolumeDetailDrawer.svelte
+107
View File
@@ -0,0 +1,107 @@
<script lang="ts">
import { Drawer, DrawerContent } from '../common'
import { VolumeService, type Volume } from '$lib/gen'
import { workspaceStore, userStore } from '$lib/stores'
import { displayDate, displaySize, sendUserToast } from '$lib/utils'
import { ExternalLink, Loader2, Trash2 } from 'lucide-svelte'
import Button from '../common/button/Button.svelte'
import type S3FilePicker from '../S3FilePicker.svelte'
let { s3FilePicker }: { s3FilePicker?: S3FilePicker } = $props()
let open = $state(false)
let volumeName = $state('')
let loading = $state(false)
let volume: Volume | undefined = $state(undefined)
export function openDrawer(name: string) {
volumeName = name
open = true
loadVolume()
}
async function loadVolume() {
loading = true
try {
const volumes = await VolumeService.listVolumes({ workspace: $workspaceStore! })
volume = volumes.find((v) => v.name === volumeName)
} finally {
loading = false
}
}
async function deleteVolume() {
if (!confirm(`Delete volume '${volumeName}'? This cannot be undone.`)) return
await VolumeService.deleteVolume({ workspace: $workspaceStore!, name: volumeName })
sendUserToast(`Volume '${volumeName}' deleted`)
open = false
}
function exploreFiles() {
open = false
s3FilePicker?.open({ s3: `volumes/${volumeName}/` })
}
</script>
<Drawer {open} size="600px" on:close={() => (open = false)}>
<DrawerContent title="Volume: {volumeName}" on:close={() => (open = false)}>
{#if loading}
<div class="flex items-center justify-center py-8">
<Loader2 size={24} class="animate-spin text-secondary" />
</div>
{:else if volume}
<div class="flex flex-col gap-4">
<div class="flex flex-col gap-2 border rounded-md p-4">
<div class="flex justify-between text-sm">
<span class="text-secondary">Files</span>
<span>{volume.file_count}</span>
</div>
<div class="flex justify-between text-sm">
<span class="text-secondary">Size</span>
<span>{displaySize(volume.size_bytes) ?? '0 B'}</span>
</div>
<div class="flex justify-between text-sm">
<span class="text-secondary">Created at</span>
<span>{displayDate(volume.created_at)}</span>
</div>
<div class="flex justify-between text-sm">
<span class="text-secondary">Created by</span>
<span>{volume.created_by}</span>
</div>
{#if volume.last_used_at}
<div class="flex justify-between text-sm">
<span class="text-secondary">Last used</span>
<span>{displayDate(volume.last_used_at)}</span>
</div>
{/if}
</div>
<div class="flex gap-2">
{#if s3FilePicker}
<Button
variant="border"
startIcon={{ icon: ExternalLink }}
on:click={exploreFiles}
>
Explore files
</Button>
{/if}
{#if $userStore?.is_admin}
<Button
variant="border"
btnClasses="text-red-500 hover:text-red-600"
startIcon={{ icon: Trash2 }}
on:click={deleteVolume}
>
Delete
</Button>
{/if}
</div>
</div>
{:else}
<div class="text-sm text-secondary py-8 text-center">
Volume '{volumeName}' not found.
</div>
{/if}
</DrawerContent>
</Drawer>
frontend/src/lib/components/assets/VolumesDrawer.svelte
+193
View File
@@ -0,0 +1,193 @@
<script lang="ts">
import { Drawer, DrawerContent } from '../common'
import { VolumeService } from '$lib/gen'
import { workspaceStore, userStore } from '$lib/stores'
import { displayDate, displaySize, sendUserToast } from '$lib/utils'
import { File, HardDriveIcon, Loader2, Plus, Shield, Trash2 } from 'lucide-svelte'
import Button from '../common/button/Button.svelte'
import SharedBadge from '../SharedBadge.svelte'
import ShareModal from '../ShareModal.svelte'
import Popover from '../meltComponents/Popover.svelte'
import TextInput from '../text_input/TextInput.svelte'
import { resource } from 'runed'
let { onExplore }: { onExplore?: (volumeName: string) => void } = $props()
let open = $state(false)
let refreshKey = $state(0)
let shareModal: ShareModal | undefined = $state()
let newVolumeName = $state('')
let volumes = resource(
() => (open ? { ws: $workspaceStore, key: refreshKey } : undefined),
(params) => {
if (!params?.ws) return Promise.resolve([])
return VolumeService.listVolumes({ workspace: params.ws })
}
)
export function openDrawer() {
open = true
}
async function createVolume(name: string, close: () => void) {
try {
await VolumeService.createVolume({
workspace: $workspaceStore!,
requestBody: { name }
})
sendUserToast(`Volume '${name}' created`)
newVolumeName = ''
close()
refreshKey++
} catch (e) {
sendUserToast(`Failed to create volume: ${e}`, true)
}
}
async function deleteVolume(name: string) {
if (!confirm(`Delete volume '${name}'? This cannot be undone.`)) return
await VolumeService.deleteVolume({ workspace: $workspaceStore!, name })
sendUserToast(`Volume '${name}' deleted`)
refreshKey++
}
function canReadVolume(
createdBy: string,
extraPerms?: { [key: string]: unknown }
): boolean {
if ($userStore?.is_admin) return true
const username = $userStore?.username
if (username === createdBy || `u/${username}` === createdBy) return true
const perms = extraPerms ?? {}
const keys = Object.keys(perms)
if (keys.length === 0) return true // public
if (`u/${username}` in perms) return true
const pgroups = $userStore?.pgroups ?? []
return pgroups.some((g) => g in perms)
}
function canWriteVolume(
createdBy: string,
extraPerms?: { [key: string]: unknown }
): boolean {
if ($userStore?.is_admin) return true
const username = $userStore?.username
if (username === createdBy || `u/${username}` === createdBy) return true
if (extraPerms?.[`u/${username}`] === true) return true
const pgroups = $userStore?.pgroups ?? []
return pgroups.some((g) => extraPerms?.[g] === true)
}
</script>
<Drawer {open} size="700px" on:close={() => (open = false)}>
<DrawerContent title="Volumes" on:close={() => (open = false)}>
{#snippet actions()}
<Popover
floatingConfig={{ strategy: 'absolute', placement: 'bottom-end' }}
containerClasses="border rounded-lg shadow-lg bg-surface"
>
{#snippet trigger()}
<Button unifiedSize="sm" variant="accent" startIcon={{ icon: Plus }} nonCaptureEvent
>New volume</Button
>
{/snippet}
{#snippet content({ close })}
<div class="flex flex-col gap-2 p-4">
<TextInput
size="md"
inputProps={{
placeholder: 'Volume name',
onkeyup: (e) => {
if (e.key === 'Enter' && newVolumeName.trim()) {
createVolume(newVolumeName.trim(), close)
}
}
}}
bind:value={newVolumeName}
/>
<Button
unifiedSize="sm"
variant="accent"
startIcon={{ icon: Plus }}
disabled={!newVolumeName.trim()}
on:click={() => createVolume(newVolumeName.trim(), close)}
>
Create
</Button>
</div>
{/snippet}
</Popover>
{/snippet}
{#if volumes.loading}
<div class="flex items-center justify-center py-8">
<Loader2 size={24} class="animate-spin text-secondary" />
</div>
{:else if !volumes.current?.length}
<div class="text-sm text-secondary py-8 text-center">
No volumes yet. Create one above or they are auto-created when a job declares a volume annotation.
</div>
{:else}
<div class="flex flex-col divide-y border rounded-md">
{#each volumes.current as vol (vol.name)}
{@const readable = canReadVolume(vol.created_by, vol.extra_perms)}
{@const writable = canWriteVolume(vol.created_by, vol.extra_perms)}
<div class="flex items-center gap-3 px-4 py-3 hover:bg-surface-hover">
<HardDriveIcon size={16} class="text-secondary shrink-0" />
<div class="flex flex-col flex-1 min-w-0">
<div class="flex items-center gap-1.5">
<span class="text-sm font-medium truncate">{vol.name}</span>
<SharedBadge
extraPerms={vol.extra_perms as Record<string, boolean>}
canWrite={writable}
/>
</div>
<span class="text-2xs text-secondary">
{vol.file_count} {vol.file_count === 1 ? 'file' : 'files'}
&middot; {displaySize(vol.size_bytes) ?? '0 B'}
&middot; owner: {vol.created_by.replace(/^u\//, '')}
</span>
</div>
{#if vol.last_used_at}
<span class="text-2xs text-secondary shrink-0">
Used {displayDate(vol.last_used_at)}
</span>
{/if}
{#if writable}
<Button
variant="subtle"
iconOnly
unifiedSize="sm"
endIcon={{ icon: Shield }}
on:click={() =>
shareModal?.openDrawer(vol.name, 'volume', true)}
/>
{/if}
{#if onExplore && readable}
<Button
variant="default"
unifiedSize="sm"
endIcon={{ icon: File }}
on:click={() => onExplore(vol.name)}
>
Explore
</Button>
{/if}
{#if writable}
<Button
variant="subtle"
iconOnly
unifiedSize="sm"
btnClasses="text-red-500 hover:text-red-600"
endIcon={{ icon: Trash2 }}
on:click={() => deleteVolume(vol.name)}
/>
{/if}
</div>
{/each}
</div>
{/if}
</DrawerContent>
</Drawer>
<ShareModal bind:this={shareModal} on:change={() => refreshKey++} />
frontend/src/lib/components/assets/lib.ts
+4
View File
@@ -26,6 +26,8 @@ export function formatAsset(asset: Asset): string {
return `ducklake://${asset.path}`
case 'datatable':
return `datatable://${asset.path}`
case 'volume':
return `volume://${asset.path}`
}
return 'unknown'
}
@@ -89,6 +91,8 @@ export function formatAssetKind(asset: {
return 'Ducklake'
case 'datatable':
return 'Data table'
case 'volume':
return 'Volume'
}
}
frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte
+21 -4
View File
@@ -24,6 +24,7 @@
import JavaIcon from '$lib/components/icons/JavaIcon.svelte'
import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte'
import RubyIcon from '$lib/components/icons/RubyIcon.svelte'
import ClaudeIcon from '$lib/components/icons/ClaudeIcon.svelte'
interface Props {
lang:
@@ -36,6 +37,7 @@
| 'docker'
| 'powershell'
| 'bunnative'
| 'claudesandbox'
width?: number
height?: number
scale?: number
@@ -45,7 +47,7 @@
let { lang, width = 30, height = 30, scale = 1, size = undefined, ...rest }: Props = $props()
const languageLabel: Record<Script['language'] | 'bunnative', String> = {
const languageLabel: Record<Script['language'] | 'bunnative' | 'claudesandbox', String> = {
python3: 'Python',
deno: 'TypeScript',
go: 'Go',
@@ -68,12 +70,13 @@
csharp: 'C#',
nu: 'Nu',
java: 'Java',
ruby: 'Ruby'
ruby: 'Ruby',
claudesandbox: 'Claude Sandbox'
// for related places search: ADD_NEW_LANG
}
const langToComponent: Record<
SupportedLanguage | 'pgsql' | 'javascript' | 'fetch' | 'docker' | 'powershell' | 'bunnative',
SupportedLanguage | 'pgsql' | 'javascript' | 'fetch' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox',
any
> = {
go: GoIcon,
@@ -103,7 +106,8 @@
nu: NuIcon,
java: JavaIcon,
ruby: RubyIcon,
duckdb: DuckDbIcon
duckdb: DuckDbIcon,
claudesandbox: TypeScriptIcon
// for related places search: ADD_NEW_LANG
}
@@ -142,4 +146,17 @@
/>
</div>
{/if}
{#if lang === 'claudesandbox'}
<div
class="absolute -top-1.5 -right-1.5 bg-surface rounded-full flex items-center justify-center"
style={`width: ${width * scale * subIconScale}px; height: ${
height * scale * subIconScale
}px;`}
>
<ClaudeIcon
width={width * scale * (subIconScale - 0.1)}
height={height * scale * (subIconScale - 0.1)}
/>
</div>
{/if}
</div>
frontend/src/lib/components/copilot/ResourceGen.svelte
+186
View File
@@ -0,0 +1,186 @@
<script lang="ts">
import { ExternalLink, Wand2 } from 'lucide-svelte'
import Button from '../common/button/Button.svelte'
import { getNonStreamingCompletion } from './lib'
import Popover from '$lib/components/meltComponents/Popover.svelte'
import { sendUserToast } from '$lib/toast'
import { base } from '$lib/base'
import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'
import { copilotInfo } from '$lib/aiStore'
import type { Schema } from '$lib/common'
interface Props {
args: Record<string, any>
resourceType?: string
resourceName?: string
resourceDescription?: string
resourceSchema?: Schema | undefined
isFileset?: boolean
}
let {
args = $bindable(),
resourceType = '',
resourceName = '',
resourceDescription = '',
resourceSchema = undefined,
isFileset = false
}: Props = $props()
let instructions = $state('')
let instructionsField: HTMLTextAreaElement | undefined = $state(undefined)
let genLoading = $state(false)
let abortController = $state(new AbortController())
function buildSystemPrompt(): string {
let prompt: string
if (isFileset) {
prompt =
'You are a helpful assistant that generates file contents for a Windmill fileset resource. A fileset is a JSON object mapping file paths to their string contents, e.g. {"path/to/file.txt": "file content", "other/file.md": "# Title"}. You MUST return ONLY valid JSON (a flat object with string keys and string values), no markdown fences, no explanation, no extra text.'
} else {
prompt =
'You are a helpful assistant that generates JSON values for Windmill resources. You MUST return ONLY valid JSON, no markdown fences, no explanation, no extra text.'
}
if (resourceType) {
prompt += `\nThe resource type is "${resourceType}".`
}
if (!isFileset && resourceSchema?.properties) {
const schemaDesc = Object.entries(resourceSchema.properties)
.map(([key, prop]: [string, any]) => {
let desc = `- "${key}": type=${prop.type || 'string'}`
if (prop.description) desc += `, description="${prop.description}"`
if (prop.default !== undefined) desc += `, default=${JSON.stringify(prop.default)}`
if (prop.enum) desc += `, enum=${JSON.stringify(prop.enum)}`
return desc
})
.join('\n')
prompt += `\n\nThe resource schema has these properties:\n${schemaDesc}`
if (resourceSchema.required?.length) {
prompt += `\n\nRequired fields: ${resourceSchema.required.join(', ')}`
}
}
return prompt
}
function buildUserPrompt(): string {
let prompt = instructions
if (resourceName) {
prompt += `\nResource name: ${resourceName}`
}
if (resourceDescription) {
prompt += `\nResource description: ${resourceDescription}`
}
return prompt
}
async function generateResource() {
genLoading = true
abortController = new AbortController()
try {
const messages: ChatCompletionMessageParam[] = [
{
role: 'system',
content: buildSystemPrompt()
},
{
role: 'user',
content: buildUserPrompt()
}
]
let response = await getNonStreamingCompletion(messages, abortController)
// Strip markdown fences if present
response = response.trim()
if (response.startsWith('```')) {
response = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '')
}
const parsed = JSON.parse(response)
args = parsed
} catch (err) {
if (!abortController.signal.aborted) {
sendUserToast('Could not generate resource: ' + err, true)
}
} finally {
genLoading = false
}
}
$effect(() => {
instructionsField && setTimeout(() => instructionsField?.focus(), 100)
})
</script>
<Popover floatingConfig={{ strategy: 'absolute', placement: 'bottom-end' }}>
{#snippet trigger()}
<Button
color={genLoading ? 'red' : 'light'}
size="xs"
nonCaptureEvent={!genLoading}
startIcon={{ icon: Wand2 }}
iconOnly
title="AI Assistant"
btnClasses="text-ai bg-violet-100 dark:bg-gray-700"
loading={genLoading}
clickableWhileLoading
on:click={genLoading ? () => abortController?.abort() : () => {}}
/>
{/snippet}
{#snippet content({ close })}
<div class="border rounded-lg shadow-lg p-4 bg-surface">
{#if $copilotInfo.enabled}
<div class="flex flex-col w-80 gap-2">
<textarea
bind:this={instructionsField}
placeholder="Describe the resource values to generate..."
bind:value={instructions}
rows={3}
class="text-xs"
onkeydown={(e) => {
if (e.key === 'Enter' && !e.shiftKey && instructions.length > 0) {
e.preventDefault()
close()
generateResource()
}
}}
/>
<Button
size="xs"
color="light"
variant="contained"
buttonType="button"
btnClasses="text-ai bg-violet-100 dark:bg-gray-700"
title="Generate resource from prompt"
on:click={() => {
close()
generateResource()
}}
disabled={instructions.length == 0}
startIcon={{ icon: Wand2 }}
>
Generate
</Button>
</div>
{:else}
<div class="block text-primary">
<p class="text-sm"
>Enable Windmill AI in the <a
href="{base}/workspace_settings?tab=ai"
target="_blank"
class="inline-flex flex-row items-center gap-1"
>workspace settings <ExternalLink size={16} /></a
></p
>
</div>
{/if}
</div>
{/snippet}
</Popover>
frontend/src/lib/components/flows/content/FlowInputs.svelte
+18
View File
@@ -288,6 +288,24 @@
{/each}
</div>
{#if !failureModule && !preprocessorModule}
<h3 class="pb-2 pt-4">AI Sandbox</h3>
<div class="flex flex-row flex-wrap gap-2">
<FlowScriptPicker
label="Claude Code"
lang="claudesandbox"
on:click={() => {
dispatch('new', {
language: 'bun',
kind,
subkind: 'claudesandbox',
summary
})
}}
/>
</div>
{/if}
<h3 class="mb-2 mt-6"
>Use pre-made <span class="text-blue-500 dark:text-blue-400"
>{kind == 'script' ? 'action' : kind}</span
frontend/src/lib/components/flows/content/FlowInputsQuick.svelte
+21
View File
@@ -491,6 +491,27 @@
{/await}
<div class="pb-1"></div>
{/if}
{#if selectedKind === 'script' && preFilter === 'all' && !selected}
<div class="pb-0 text-2xs font-normal text-secondary ml-2">AI Sandbox</div>
<FlowScriptPickerQuick
eeRestricted={false}
selected={false}
enterpriseLangs={[]}
label="Claude Code"
lang="claudesandbox"
on:click={() => {
dispatch('new', {
kind: selectedKind,
inlineScript: {
language: 'bun',
kind: selectedKind,
subkind: 'claudesandbox',
summary
}
})
}}
/>
{/if}
{#if selectedKind != 'preprocessor' && selectedKind != 'flow'}
{#if (!selected || selected?.kind === 'integrations') && (preFilter === 'hub' || preFilter === 'all')}
{#if !selected && preFilter !== 'hub'}
frontend/src/lib/components/flows/flowStateUtils.svelte.ts
+1 -1
View File
@@ -78,7 +78,7 @@ export async function pickFlow(
export async function createInlineScriptModule(
language: RawScript['language'],
kind: Script['kind'],
subkind: 'pgsql' | 'flow' | undefined,
subkind: 'pgsql' | 'flow' | 'claudesandbox' | undefined,
id: string,
summary?: string
): Promise<[FlowModule, FlowModuleState]> {
frontend/src/lib/components/flows/map/InsertModuleInner.svelte
+45 -19
View File
@@ -31,7 +31,7 @@
}: Props = $props()
let customUi: undefined | FlowBuilderWhitelabelCustomUi = getContext('customUi')
let selectedKind: 'script' | 'trigger' | 'preprocessor' | 'approval' | 'flow' | 'failure' =
let selectedKind: 'script' | 'trigger' | 'preprocessor' | 'approval' | 'flow' | 'failure' | 'aisandbox' =
$state(kind)
let preFilter: 'all' | 'workspace' | 'hub' = $state('all')
let loading = $state(false)
@@ -181,27 +181,53 @@
}}
/>
{/if}
<TopLevelNode
label="AI Sandbox"
selected={selectedKind === 'aisandbox'}
onSelect={() => {
selectedKind = 'aisandbox'
}}
/>
{/if}
</div>
{/if}
<FlowInputsQuick
{selectedKind}
bind:loading
filter={funcDesc}
{disableAi}
{funcDesc}
{kind}
bind:owners
on:close={() => {
dispatch('close')
}}
on:new
on:pickScript
on:pickFlow
{preFilter}
{displayPath}
refreshCount={refreshCount.val}
/>
{#if selectedKind === 'aisandbox'}
<div class="h-full overflow-auto grow min-w-0 p-2 gap-1 flex flex-col">
<TopLevelNode
label="Claude Code"
onSelect={() => {
dispatch('close')
dispatch('new', {
kind: 'script',
inlineScript: {
language: 'bun',
kind: 'script',
subkind: 'claudesandbox',
}
})
}}
/>
</div>
{:else}
<FlowInputsQuick
{selectedKind}
bind:loading
filter={funcDesc}
{disableAi}
{funcDesc}
{kind}
bind:owners
on:close={() => {
dispatch('close')
}}
on:new
on:pickScript
on:pickFlow
{preFilter}
{displayPath}
refreshCount={refreshCount.val}
/>
{/if}
</div>
</div>
frontend/src/lib/components/flows/pickers/FlowScriptPicker.svelte
+4 -1
View File
@@ -8,7 +8,7 @@
interface Props {
disabled?: boolean
label: string
lang?: SupportedLanguage | 'docker' | 'javascript' | undefined
lang?: SupportedLanguage | 'docker' | 'javascript' | 'claudesandbox' | undefined
id?: string | undefined
}
@@ -32,6 +32,9 @@
<LanguageIcon {lang} />
{/if}
<span class="text-xs">{label}</span>
{#if lang === 'claudesandbox'}
<span class="text-primary !text-xs">(new)</span>
{/if}
</div>
</Button>
{#snippet text()}
frontend/src/lib/components/flows/pickers/FlowScriptPickerQuick.svelte
+4 -1
View File
@@ -6,7 +6,7 @@
import { createEventDispatcher } from 'svelte'
export let label: string
export let lang: SupportedLanguage | 'docker' | 'javascript' | undefined = undefined
export let lang: SupportedLanguage | 'docker' | 'javascript' | 'claudesandbox' | undefined = undefined
export let selected = false
export let eeRestricted: boolean
export let enterpriseLangs: string[] = []
@@ -46,6 +46,9 @@
{/if}
<span class="grow truncate text-left {eeRestricted ? 'text-disabled' : ''}">
{label}{#if eeRestricted}&nbsp;(EE){/if}
{#if lang === 'claudesandbox'}
<span class="text-primary !text-xs">(new)</span>
{/if}
</span>
{#if selected}
<kbd class="!text-xs">&crarr;</kbd>

Some files were not shown because too many files have changed in this diff Show More