mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
feat: mount volumes read-only without the exclusive lease (#12410)
* feat: mount volumes read-only without the exclusive lease Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvvmmTkuELF5qpS2Hs3 * fix: gate lease-free volume reads on the fleet version Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvvmmTkuELF5qpS2Hs3 * fix: keep held volumes counted by cache eviction, restore MIN_VERSION in tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvvmmTkuELF5qpS2Hs3 * chore: update ee-repo-ref to 38904055ab0d104006278322299bdbcea32c88d1 This commit updates the EE repository reference after PR #980 was merged in windmill-ee-private. Previous ee-repo-ref: 1fe25fa2530efdba4f8a3bf5dbfbb6f85e616ce7 New ee-repo-ref: 38904055ab0d104006278322299bdbcea32c88d1 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
windmill-internal-app[bot]
parent
164528563d
commit
70711bec68
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume SET generation = generation + 1 - generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()\n RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "159f2030de3ba169925236c8c23720e820a132958cd64483cba8c8e40bf2a563"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2",
|
||||
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -14,5 +14,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34"
|
||||
"hash": "19c481a0d6ecd4c0dfab6973e4732c2146953253ba6062ab05b34b6b35e41f66"
|
||||
}
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume SET generation = generation + 1 - generation % 2\n WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3\n RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "2ae1bfc683bbefadf8777b06898fcf48ebbdff22e059a33bc9188f38460f964c"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume SET generation = 2 WHERE workspace_id = 'test-workspace' AND name = 'test-vol'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "2d67831f591efa5c37047f18a7dd8d33d862bba924ca098a1423e782df88fb1b"
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT extra_perms, created_by, generation, coalesce(lease_until > now(), false) AS \"write_live!\"\n FROM volume WHERE workspace_id = $1 AND name = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "extra_perms",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "generation",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "write_live!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "2eb0de1f862785cc61aa5cc66641b2774bb0851593a55571ee9e7c2479c2919a"
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT j.permissioned_as, v.extra_perms, v.created_by, v.generation\n FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN volume v ON v.workspace_id = j.workspace_id AND v.name = $3\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.worker = $4",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "permissioned_as",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "extra_perms",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "generation",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "423dc0716c7a6f63e311a069f67158960cd9df94fa0540f299fa31cd41a82c5a"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
|
||||
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4,\n generation = volume.generation + volume.generation % 2\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -21,5 +21,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6"
|
||||
"hash": "54549a1aca624587a1f91c00f54fdcc887172d6a4c2081af6f2fafe249c5c29a"
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT leased_by, generation FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "leased_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "generation",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "8338b741d782c6c4273b1c98c7e62763959428e37f83ff376cd9018d11ea21ed"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = 'test-workspace'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Jsonb"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "85a9967ab419bd1f09a2be7751ff8d15cb8431bf05fb34a7b99d03a2a86e48ca"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
|
||||
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -12,5 +12,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33"
|
||||
"hash": "85ac712e057207d1db48303925b2ca08479830f6de789cbc64661509fccba9ce"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)\n VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + interval '1 hour', 'other-worker', 1)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a26528bf01138bfad0c4989c04fd74e3c0aacd9913b7524edcf69f58115875fc"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
|
||||
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -12,5 +12,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98"
|
||||
"hash": "b0350f074557a476882697046c3573d6ac532b374d9b2470322f70b875eaec87"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
|
||||
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -12,5 +12,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89"
|
||||
"hash": "b6d05e743b558e82bca080ab119aef7949ceae08e71a0191ce185420e037913d"
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT leased_by FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "leased_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "cd5cdad3449dd5f58c88d3a7b99158da707d8f23513c155deb430b702d932263"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)\n VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + make_interval(secs => $1), 'other-worker', $2)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Float8",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "dfe0a2a6e4ab88d5341415bdc19a6982a22cd119e5e1dbe2b6149883e71a0066"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT generation FROM volume WHERE workspace_id = $1 AND name = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "generation",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "e0838b093a54f1f9735452f62068612d76144ac859fc904d882a58d990de952d"
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT generation, coalesce(lease_until > now(), false) AS \"write_live!\"\n FROM volume WHERE workspace_id = $1 AND name = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "generation",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "write_live!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "e938951c0c6ddd268db7a0dc69484446d4e869951188a0add41036d4fd095298"
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4,\n generation = volume.generation + volume.generation % 2\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "e99555474694f563991558f8f1447c2eeec0ece06b69d730c6b3d3dffcafeb26"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5",
|
||||
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL,\n generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -14,5 +14,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca"
|
||||
"hash": "f5946a3968df859f668d6197562c10024ffc24f108ca48d5b3fa98e489d931ea"
|
||||
}
|
||||
Generated
+1
@@ -17671,6 +17671,7 @@ name = "windmill-worker-volumes"
|
||||
version = "1.871.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fs4",
|
||||
"futures",
|
||||
"lazy_static",
|
||||
"md-5 0.10.6",
|
||||
|
||||
@@ -1 +1 @@
|
||||
540d78337de9da0ec56f59c911f9feac5d0df608
|
||||
38904055ab0d104006278322299bdbcea32c88d1
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE volume DROP COLUMN IF EXISTS generation;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Odd while a writer is changing the volume's objects, even otherwise. Read-only mounts take no
|
||||
-- lease and use it to detect a write that overlapped their pull.
|
||||
ALTER TABLE volume ADD COLUMN IF NOT EXISTS generation BIGINT NOT NULL DEFAULT 0;
|
||||
@@ -528,6 +528,146 @@ export function main() {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Every test that sets the fleet's minimum version holds this: tests share the process.
|
||||
#[cfg(all(feature = "parquet", feature = "enterprise"))]
|
||||
static FLEET_VERSION: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[cfg(all(feature = "parquet", feature = "enterprise"))]
|
||||
struct OnDrop<F: FnMut()>(F);
|
||||
|
||||
#[cfg(all(feature = "parquet", feature = "enterprise"))]
|
||||
impl<F: FnMut()> Drop for OnDrop<F> {
|
||||
fn drop(&mut self) {
|
||||
(self.0)()
|
||||
}
|
||||
}
|
||||
|
||||
/// An agent worker's `volume:ro` pull takes no lease, waits for a write in progress to close,
|
||||
/// and writes nothing back.
|
||||
#[cfg(all(feature = "parquet", feature = "enterprise"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_agent_worker_read_only_volume(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let leased_by = run_agent_read_only_volume(db, true).await?;
|
||||
assert_eq!(leased_by.as_deref(), Some("other-worker"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Until every worker marks its writes, a `volume:ro` mount pulls under the lease, and holding
|
||||
/// it must not make the mount writable.
|
||||
#[cfg(all(feature = "parquet", feature = "enterprise"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_agent_worker_read_only_volume_leased(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let leased_by = run_agent_read_only_volume(db, false).await?;
|
||||
assert_eq!(leased_by, None, "the lease should be released");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Another worker holds the volume and rewrites `hello.txt` after 3s: under a write window
|
||||
/// (odd generation, lease held for an hour) when the fleet supports lease-free pulls, or as an
|
||||
/// older worker would, with no window and a lease that lapses after 4s. Either way the
|
||||
/// `volume:ro` job must read the rewritten file and leave nothing behind. Returns `leased_by`.
|
||||
#[cfg(all(feature = "parquet", feature = "enterprise"))]
|
||||
async fn run_agent_read_only_volume(
|
||||
db: Pool<Postgres>,
|
||||
fleet_supports_lease_free: bool,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
use windmill_common::min_version::{MIN_VERSION, MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES};
|
||||
let _fleet = FLEET_VERSION.lock().await;
|
||||
// Put back for the rest of the binary, panics included
|
||||
let before = MIN_VERSION.load_full();
|
||||
let _restore = OnDrop(move || MIN_VERSION.store(before.clone()));
|
||||
let mut fleet = MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES.version().clone();
|
||||
if !fleet_supports_lease_free {
|
||||
fleet.minor -= 1;
|
||||
}
|
||||
MIN_VERSION.store(std::sync::Arc::new(fleet));
|
||||
|
||||
let (_client, port, _server) = init_client_agent_mode(db.clone()).await;
|
||||
|
||||
let storage_dir = tempfile::tempdir()?;
|
||||
let lfs_config = json!({
|
||||
"type": "FilesystemStorage",
|
||||
"root_path": storage_dir.path().to_string_lossy(),
|
||||
"public_resource": null,
|
||||
"advanced_permissions": null,
|
||||
"volume_storage": "primary"
|
||||
});
|
||||
sqlx::query!(
|
||||
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = 'test-workspace'",
|
||||
lfs_config,
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let vol_dir = storage_dir.path().join("volumes/test-workspace/test-vol");
|
||||
std::fs::create_dir_all(&vol_dir)?;
|
||||
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
|
||||
|
||||
let (lease_secs, generation) = if fleet_supports_lease_free {
|
||||
(3600, 1)
|
||||
} else {
|
||||
(4, 0)
|
||||
};
|
||||
sqlx::query!(
|
||||
"INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)
|
||||
VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + make_interval(secs => $1), 'other-worker', $2)",
|
||||
lease_secs as f64,
|
||||
generation
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let db_c = db.clone();
|
||||
let hello = vol_dir.join("hello.txt");
|
||||
let writer = tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
|
||||
std::fs::write(&hello, b"hello after the write").unwrap();
|
||||
if fleet_supports_lease_free {
|
||||
sqlx::query!(
|
||||
"UPDATE volume SET generation = 2 WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
|
||||
)
|
||||
.execute(&db_c)
|
||||
.await?;
|
||||
}
|
||||
Ok::<_, sqlx::Error>(())
|
||||
});
|
||||
|
||||
let code = r#"// volume:ro test-vol data
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
|
||||
export function main() {
|
||||
const content = readFileSync("data/hello.txt", "utf-8");
|
||||
writeFileSync("data/output.txt", "discarded");
|
||||
return { read_content: content };
|
||||
}"#;
|
||||
let uuid = RunJob::from(JobPayload::Code(bun_code(code)))
|
||||
.push(&db)
|
||||
.await;
|
||||
let listener = listen_for_completed_jobs(&db).await;
|
||||
let conn = testing_http_connection_with_tags(
|
||||
port,
|
||||
vec!["bun".into(), "flow".into(), "dependency".into()],
|
||||
)
|
||||
.await;
|
||||
in_test_worker(conn, listener.find(&uuid), port).await;
|
||||
writer.await??;
|
||||
|
||||
let result = completed_job(uuid, &db).await;
|
||||
assert!(result.success, "job should succeed: {:?}", result.result);
|
||||
assert_eq!(
|
||||
result.json_result().unwrap()["read_content"],
|
||||
json!("hello after the write"),
|
||||
"the pull must wait for the other worker's write"
|
||||
);
|
||||
assert!(
|
||||
!vol_dir.join("output.txt").exists(),
|
||||
"a read-only mount must not write back"
|
||||
);
|
||||
Ok(sqlx::query_scalar!(
|
||||
"SELECT leased_by FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Test the volume release endpoint (error/cancel path).
|
||||
#[cfg(feature = "parquet")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
|
||||
+147
-48
@@ -468,7 +468,11 @@ fn test_volume_comment_prefix_selection() {
|
||||
fn test_volume_mount_struct() {
|
||||
use windmill_worker_volumes::VolumeMount;
|
||||
|
||||
let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() };
|
||||
let mount = VolumeMount {
|
||||
name: "test-vol".to_string(),
|
||||
target: "/mnt/data".to_string(),
|
||||
read_only: false,
|
||||
};
|
||||
assert_eq!(mount.name, "test-vol");
|
||||
assert_eq!(mount.target, "/mnt/data");
|
||||
}
|
||||
@@ -501,11 +505,13 @@ fn test_volume_nsjail_mount() {
|
||||
use std::path::Path;
|
||||
use windmill_worker_volumes::volume_nsjail_mount;
|
||||
|
||||
let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data");
|
||||
let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data", false);
|
||||
assert!(result.contains("src: \"/tmp/volumes/data\""));
|
||||
assert!(result.contains("dst: \"/mnt/data\""));
|
||||
assert!(result.contains("is_bind: true"));
|
||||
assert!(result.contains("rw: true"));
|
||||
let read_only = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data", true);
|
||||
assert!(read_only.contains("rw: false"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -599,53 +605,10 @@ async fn run_volume_sql_worker_e2e(
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
// 1. Set up filesystem-based object storage in a temp dir
|
||||
let storage_dir = tempfile::tempdir()?;
|
||||
let storage_root = storage_dir.path().to_string_lossy().to_string();
|
||||
let (_storage_dir, vol_dir) = setup_filesystem_volume(&db).await?;
|
||||
|
||||
let lfs_config = json!({
|
||||
"type": "FilesystemStorage",
|
||||
"root_path": storage_root,
|
||||
"public_resource": null,
|
||||
"advanced_permissions": null,
|
||||
"volume_storage": "primary"
|
||||
});
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
|
||||
lfs_config,
|
||||
"test-workspace"
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
// 2. Pre-populate the volume with a file (workspace-namespaced path)
|
||||
let vol_dir = storage_dir
|
||||
.path()
|
||||
.join("volumes")
|
||||
.join("test-workspace")
|
||||
.join("test-vol");
|
||||
std::fs::create_dir_all(&vol_dir)?;
|
||||
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
|
||||
|
||||
// 3. Push the job and run with SQL-connected worker
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content: code.to_string(),
|
||||
path: None,
|
||||
language,
|
||||
lock: None,
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
modules: None,
|
||||
tag: None,
|
||||
});
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, false, job, port).await;
|
||||
let result =
|
||||
run_job_in_new_worker_until_complete(&db, false, raw_code_job(language, code), port).await;
|
||||
|
||||
assert!(result.success, "job should succeed: {:?}", result.result);
|
||||
let json = result.json_result().expect("should have JSON result");
|
||||
@@ -682,3 +645,139 @@ async fn run_volume_sql_worker_e2e(
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A `volume:ro` mount takes no lease: it runs while another worker holds the volume, waits
|
||||
/// only for that worker's write in progress (odd generation) to close, and writes nothing back.
|
||||
#[cfg(all(feature = "parquet", feature = "private"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_read_only_volume_ignores_lease_and_waits_for_write(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
use windmill_common::min_version::{MIN_VERSION, MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES};
|
||||
struct OnDrop<F: FnMut()>(F);
|
||||
impl<F: FnMut()> Drop for OnDrop<F> {
|
||||
fn drop(&mut self) {
|
||||
(self.0)()
|
||||
}
|
||||
}
|
||||
// Put back for the rest of the binary, panics included
|
||||
let before = MIN_VERSION.load_full();
|
||||
let _restore = OnDrop(move || MIN_VERSION.store(before.clone()));
|
||||
MIN_VERSION.store(std::sync::Arc::new(
|
||||
MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES.version().clone(),
|
||||
));
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
let (_storage_dir, vol_dir) = setup_filesystem_volume(&db).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)
|
||||
VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + interval '1 hour', 'other-worker', 1)"
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let db_c = db.clone();
|
||||
let hello = vol_dir.join("hello.txt");
|
||||
let writer = tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
|
||||
// A different length: the cache cannot tell same-size rewrites apart on a store
|
||||
// whose etags are not content hashes.
|
||||
std::fs::write(&hello, b"hello after the write").unwrap();
|
||||
sqlx::query!(
|
||||
"UPDATE volume SET generation = 2 WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
|
||||
)
|
||||
.execute(&db_c)
|
||||
.await
|
||||
});
|
||||
|
||||
let code = r#"// volume:ro test-vol data
|
||||
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
|
||||
export function main() {
|
||||
const content = readFileSync("data/hello.txt", "utf-8");
|
||||
writeFileSync("data/output.txt", "discarded");
|
||||
return { read_content: content };
|
||||
}"#;
|
||||
let result =
|
||||
run_job_in_new_worker_until_complete(&db, false, raw_code_job(ScriptLang::Bun, code), port)
|
||||
.await;
|
||||
writer.await??;
|
||||
|
||||
assert!(result.success, "job should succeed: {:?}", result.result);
|
||||
assert_eq!(
|
||||
result.json_result().unwrap()["read_content"],
|
||||
json!("hello after the write"),
|
||||
"the pull must wait for the write in progress"
|
||||
);
|
||||
assert!(
|
||||
!vol_dir.join("output.txt").exists(),
|
||||
"a read-only mount must not write back"
|
||||
);
|
||||
let row = sqlx::query!(
|
||||
"SELECT leased_by, generation FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?;
|
||||
assert_eq!(row.leased_by.as_deref(), Some("other-worker"));
|
||||
assert_eq!(row.generation, 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
fn raw_code_job(language: ScriptLang, code: &str) -> JobPayload {
|
||||
JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content: code.to_string(),
|
||||
path: None,
|
||||
language,
|
||||
lock: None,
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
modules: None,
|
||||
tag: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Filesystem object storage as the workspace's volume storage, holding `test-vol` with
|
||||
/// `hello.txt`. Returns the storage dir (deleted on drop) and the volume's object dir.
|
||||
#[cfg(feature = "parquet")]
|
||||
async fn setup_filesystem_volume(
|
||||
db: &Pool<Postgres>,
|
||||
) -> anyhow::Result<(tempfile::TempDir, std::path::PathBuf)> {
|
||||
// 1. Set up filesystem-based object storage in a temp dir
|
||||
let storage_dir = tempfile::tempdir()?;
|
||||
let storage_root = storage_dir.path().to_string_lossy().to_string();
|
||||
|
||||
let lfs_config = json!({
|
||||
"type": "FilesystemStorage",
|
||||
"root_path": storage_root,
|
||||
"public_resource": null,
|
||||
"advanced_permissions": null,
|
||||
"volume_storage": "primary"
|
||||
});
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
|
||||
lfs_config,
|
||||
"test-workspace"
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
|
||||
// 2. Pre-populate the volume with a file (workspace-namespaced path)
|
||||
let vol_dir = storage_dir
|
||||
.path()
|
||||
.join("volumes")
|
||||
.join("test-workspace")
|
||||
.join("test-vol");
|
||||
std::fs::create_dir_all(&vol_dir)?;
|
||||
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
|
||||
|
||||
Ok((storage_dir, vol_dir))
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> {
|
||||
|
||||
/// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): `<prefix>
|
||||
/// volume: <path>` lines in the leading comment block, each an `rw` volume
|
||||
/// asset. Scanning stops at the first non-comment line; blank lines and PHP's
|
||||
/// asset (`r` for `volume:ro <path>`). Scanning stops at the first non-comment line; blank lines and PHP's
|
||||
/// opening tag line are skipped whole (the tag may carry code, so an annotation
|
||||
/// must sit on its own line below it), exactly like the frontend.
|
||||
fn parse_volume_annotations(content: &str, prefix: &str) -> Vec<AssetWithAltAccessType> {
|
||||
@@ -105,11 +105,16 @@ fn parse_volume_annotations(content: &str, prefix: &str) -> Vec<AssetWithAltAcce
|
||||
};
|
||||
let after = after.trim();
|
||||
if let Some(rest) = after.strip_prefix("volume:") {
|
||||
// `volume:ro <name>`; with a space after the colon, `ro` is the volume's name
|
||||
let (access_type, rest) = match rest.strip_prefix("ro") {
|
||||
Some(r) if r.starts_with(char::is_whitespace) => (AssetUsageAccessType::R, r),
|
||||
_ => (AssetUsageAccessType::RW, rest),
|
||||
};
|
||||
if let Some(path) = rest.trim().split_whitespace().next() {
|
||||
volumes.push(AssetWithAltAccessType {
|
||||
path: path.to_string(),
|
||||
kind: AssetKind::Volume,
|
||||
access_type: Some(AssetUsageAccessType::RW),
|
||||
access_type: Some(access_type),
|
||||
alt_access_type: None,
|
||||
columns: None,
|
||||
});
|
||||
@@ -265,6 +270,24 @@ mod tests {
|
||||
assert_eq!(vols[0].access_type, Some(AssetUsageAccessType::RW));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_volume_annotation_is_a_read() {
|
||||
let content = "// volume:ro models /opt/models\n// volume: ro /tmp/ro\nconsole.log(1)\n";
|
||||
let got = effective_script_assets(&ScriptLang::Bun, content, None).unwrap();
|
||||
let vols: Vec<_> = got
|
||||
.iter()
|
||||
.filter(|a| a.kind == AssetKind::Volume)
|
||||
.map(|a| (a.path.as_str(), a.access_type))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
vols,
|
||||
vec![
|
||||
("models", Some(AssetUsageAccessType::R)),
|
||||
("ro", Some(AssetUsageAccessType::RW))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn php_volume_annotations_survive_the_open_tag() {
|
||||
let content = "<?php\n\n// volume: my_vol\nfunction main() {}\n";
|
||||
|
||||
@@ -39,6 +39,11 @@ pub const MIN_VERSION_SUPPORTS_VARIABLE_VALUE_WIDENING: VC =
|
||||
// to accept one once this is met. Only worker versions are seen here, not servers. Must name
|
||||
// the release this ships in.
|
||||
pub const MIN_VERSION_SUPPORTS_ED25519_LICENSE: VC = vc(1, 838, 0, "Ed25519 license keys");
|
||||
// `volume:ro` mounts pull without the volume's lease and rely on every writer marking its
|
||||
// write in `volume.generation`. A worker that predates it writes without that mark, so a
|
||||
// lease-free pull overlapping its write would keep a mix of old and new files; until the
|
||||
// whole fleet has it, read-only mounts take the lease. Must name the release this ships in.
|
||||
pub const MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES: VC = vc(1, 872, 0, "Lease-free volume reads");
|
||||
pub const MIN_VERSION_SUPPORTS_NODE_DEBOUNCING: VC = vc(1, 658, 0, "Flow node debouncing");
|
||||
pub const MIN_VERSION_SUPPORTS_TOKEN_HASH: VC = vc(1, 659, 0, "Token hash storage");
|
||||
pub const MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: VC = vc(1, 602, 0, "Sync jobs debouncing");
|
||||
|
||||
@@ -655,6 +655,18 @@ pub static DEDICATED_WORKER_RELOAD: std::sync::LazyLock<tokio::sync::Notify> =
|
||||
std::sync::LazyLock::new(tokio::sync::Notify::new);
|
||||
|
||||
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
|
||||
/// The server's error message, which callers match on to tell a retryable refusal (a volume
|
||||
/// leased or being written by another worker) from a failure.
|
||||
async fn agent_error_body(response: reqwest::Response) -> String {
|
||||
response
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.chars()
|
||||
.take(1000)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HttpClient {
|
||||
pub client: ClientWithMiddleware,
|
||||
@@ -693,11 +705,12 @@ impl HttpClient {
|
||||
if status.is_success() {
|
||||
Ok(response.json().await?)
|
||||
} else {
|
||||
Err(anyhow::anyhow!(format!(
|
||||
"HTTP agent request POST {} failed {}",
|
||||
Err(anyhow::anyhow!(
|
||||
"HTTP agent request POST {} failed {}: {}",
|
||||
url,
|
||||
response.status()
|
||||
)))
|
||||
status,
|
||||
agent_error_body(response).await
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -713,11 +726,12 @@ impl HttpClient {
|
||||
if status.is_success() {
|
||||
Ok(response.json().await?)
|
||||
} else {
|
||||
Err(anyhow::anyhow!(format!(
|
||||
"HTTP agent request GET {} failed {}",
|
||||
Err(anyhow::anyhow!(
|
||||
"HTTP agent request GET {} failed {}: {}",
|
||||
url,
|
||||
response.status()
|
||||
)))
|
||||
status,
|
||||
agent_error_body(response).await
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3962,7 +3976,10 @@ mod tests {
|
||||
#[test]
|
||||
fn dedicated_worker_patterns_are_script_folders() {
|
||||
assert_eq!(dedicated_worker_pattern_prefix("f/dw/*"), Some("f/dw/"));
|
||||
assert_eq!(dedicated_worker_pattern_prefix("u/alice/*"), Some("u/alice/"));
|
||||
assert_eq!(
|
||||
dedicated_worker_pattern_prefix("u/alice/*"),
|
||||
Some("u/alice/")
|
||||
);
|
||||
assert_eq!(dedicated_worker_pattern_prefix("f/dw/script"), None);
|
||||
assert_eq!(dedicated_worker_pattern_prefix("f/dw*"), None);
|
||||
assert_eq!(dedicated_worker_pattern_prefix("/*"), None);
|
||||
|
||||
@@ -26,6 +26,7 @@ serde_json.workspace = true
|
||||
regex.workspace = true
|
||||
lazy_static.workspace = true
|
||||
md-5.workspace = true
|
||||
fs4.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -52,6 +52,8 @@ lazy_static::lazy_static! {
|
||||
pub struct VolumeMount {
|
||||
pub name: String,
|
||||
pub target: String,
|
||||
/// `volume:ro`: pulled without the volume's lease and never synced back.
|
||||
pub read_only: bool,
|
||||
}
|
||||
|
||||
/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while
|
||||
@@ -220,6 +222,15 @@ fn is_php_open_tag(trimmed_line: &str) -> bool {
|
||||
.is_some_and(|p| p.eq_ignore_ascii_case("<?php"))
|
||||
}
|
||||
|
||||
/// `ro` glued to `volume:` (`volume:ro <name> <target>`) marks a read-only mount. With a space
|
||||
/// after the colon, `ro` is a volume's name: `volume: ro /tmp/x` mounts the volume `ro`.
|
||||
pub fn split_read_only_flag(after_colon: &str) -> (bool, &str) {
|
||||
match after_colon.strip_prefix("ro") {
|
||||
Some(rest) if rest.starts_with(char::is_whitespace) => (true, rest),
|
||||
_ => (false, after_colon),
|
||||
}
|
||||
}
|
||||
|
||||
/// Scans the leading comment block for `<prefix> volume: <name> <target>` lines,
|
||||
/// stopping at the first line that is neither blank nor a comment. A PHP script
|
||||
/// opens with `<?php`, which is not a comment, so that line is skipped like a blank
|
||||
@@ -238,14 +249,17 @@ pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec<Volu
|
||||
}
|
||||
let after_prefix = trimmed[comment_prefix.len()..].trim();
|
||||
if let Some(rest) = after_prefix.strip_prefix("volume:") {
|
||||
let rest = rest.trim();
|
||||
let mut parts = rest.splitn(2, char::is_whitespace);
|
||||
let (read_only, rest) = split_read_only_flag(rest);
|
||||
let mut parts = rest.trim().splitn(2, char::is_whitespace);
|
||||
if let (Some(name), Some(target)) = (parts.next(), parts.next()) {
|
||||
let name = name.trim();
|
||||
let target = target.trim();
|
||||
if !name.is_empty() && !target.is_empty() {
|
||||
volumes
|
||||
.push(VolumeMount { name: name.to_string(), target: target.to_string() });
|
||||
volumes.push(VolumeMount {
|
||||
name: name.to_string(),
|
||||
target: target.to_string(),
|
||||
read_only,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -263,7 +277,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "mydata".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -273,7 +291,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "//");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "mydata".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -284,7 +306,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "//");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "mydata".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -295,9 +321,21 @@ mod tests {
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() },
|
||||
VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() },
|
||||
VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() },
|
||||
VolumeMount {
|
||||
name: "data1".to_string(),
|
||||
target: "/tmp/data1".to_string(),
|
||||
read_only: false
|
||||
},
|
||||
VolumeMount {
|
||||
name: "data2".to_string(),
|
||||
target: "/tmp/data2".to_string(),
|
||||
read_only: false
|
||||
},
|
||||
VolumeMount {
|
||||
name: "models".to_string(),
|
||||
target: "/opt/models".to_string(),
|
||||
read_only: false
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -308,7 +346,37 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "mydata".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_read_only_volume() {
|
||||
let content = "# volume:ro models /opt/models\n# volume: ro /tmp/ro\n# volume:robots /tmp/r\ndef main():\n pass";
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
VolumeMount {
|
||||
name: "models".to_string(),
|
||||
target: "/opt/models".to_string(),
|
||||
read_only: true,
|
||||
},
|
||||
VolumeMount {
|
||||
name: "ro".to_string(),
|
||||
target: "/tmp/ro".to_string(),
|
||||
read_only: false
|
||||
},
|
||||
VolumeMount {
|
||||
name: "robots".to_string(),
|
||||
target: "/tmp/r".to_string(),
|
||||
read_only: false,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -332,7 +400,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "data1".to_string(),
|
||||
target: "/tmp/data1".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -344,8 +416,16 @@ mod tests {
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() },
|
||||
VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() },
|
||||
VolumeMount {
|
||||
name: "data1".to_string(),
|
||||
target: "/tmp/data1".to_string(),
|
||||
read_only: false
|
||||
},
|
||||
VolumeMount {
|
||||
name: "data2".to_string(),
|
||||
target: "/tmp/data2".to_string(),
|
||||
read_only: false
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -357,7 +437,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "good".to_string(),
|
||||
target: "/tmp/good".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -367,7 +451,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "mydata".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -380,6 +468,7 @@ mod tests {
|
||||
vec![VolumeMount {
|
||||
name: "mydata".to_string(),
|
||||
target: "/tmp/my data dir".to_string(),
|
||||
read_only: false,
|
||||
}]
|
||||
);
|
||||
}
|
||||
@@ -390,7 +479,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "my-data_v2".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -467,7 +560,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "//");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "agent-memory".to_string(),
|
||||
target: ".claude".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -477,7 +574,11 @@ mod tests {
|
||||
let result = parse_volume_annotations(content, "#");
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }]
|
||||
vec![VolumeMount {
|
||||
name: "data".to_string(),
|
||||
target: "data/models".to_string(),
|
||||
read_only: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -568,7 +669,11 @@ mod tests {
|
||||
#[test]
|
||||
fn validate_mounts_rejects_too_many() {
|
||||
let mounts: Vec<VolumeMount> = (0..11)
|
||||
.map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) })
|
||||
.map(|i| VolumeMount {
|
||||
name: format!("v{:02}", i),
|
||||
target: format!("t{}", i),
|
||||
read_only: false,
|
||||
})
|
||||
.collect();
|
||||
assert!(validate_volume_mounts(&mounts).is_err());
|
||||
}
|
||||
@@ -576,8 +681,16 @@ mod tests {
|
||||
#[test]
|
||||
fn validate_mounts_rejects_duplicate_name() {
|
||||
let mounts = vec![
|
||||
VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() },
|
||||
VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() },
|
||||
VolumeMount {
|
||||
name: "data".to_string(),
|
||||
target: "/tmp/a".to_string(),
|
||||
read_only: false,
|
||||
},
|
||||
VolumeMount {
|
||||
name: "data".to_string(),
|
||||
target: "/tmp/b".to_string(),
|
||||
read_only: false,
|
||||
},
|
||||
];
|
||||
assert!(validate_volume_mounts(&mounts).is_err());
|
||||
}
|
||||
@@ -585,8 +698,16 @@ mod tests {
|
||||
#[test]
|
||||
fn validate_mounts_rejects_duplicate_target() {
|
||||
let mounts = vec![
|
||||
VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() },
|
||||
VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() },
|
||||
VolumeMount {
|
||||
name: "v1".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false,
|
||||
},
|
||||
VolumeMount {
|
||||
name: "v2".to_string(),
|
||||
target: "/tmp/data".to_string(),
|
||||
read_only: false,
|
||||
},
|
||||
];
|
||||
assert!(validate_volume_mounts(&mounts).is_err());
|
||||
}
|
||||
@@ -594,8 +715,8 @@ mod tests {
|
||||
#[test]
|
||||
fn validate_mounts_ok() {
|
||||
let mounts = vec![
|
||||
VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() },
|
||||
VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() },
|
||||
VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string(), read_only: false },
|
||||
VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string(), read_only: false },
|
||||
];
|
||||
assert!(validate_volume_mounts(&mounts).is_ok());
|
||||
}
|
||||
|
||||
@@ -25,15 +25,16 @@ pub async fn download_volume(
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String {
|
||||
pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str, _read_only: bool) -> String {
|
||||
String::new()
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub async fn sync_volume_back(
|
||||
pub async fn sync_volume_back<F: std::future::Future<Output = error::Result<()>>>(
|
||||
_client: Arc<dyn ObjectStore>,
|
||||
_state: &VolumeState,
|
||||
_workspace_id: &str,
|
||||
_open_write_window: F,
|
||||
) -> error::Result<SyncStats> {
|
||||
Err(error::Error::internal_err(
|
||||
"Volumes are not available in this build".to_string(),
|
||||
|
||||
@@ -6972,7 +6972,10 @@ mount {{
|
||||
if !volume_mounts.is_empty() {
|
||||
let vol_summary: Vec<String> = volume_mounts
|
||||
.iter()
|
||||
.map(|v| format!("'{}' -> {}", v.name, v.target))
|
||||
.map(|v| {
|
||||
let mode = if v.read_only { " (read-only)" } else { "" };
|
||||
format!("'{}' -> {}{mode}", v.name, v.target)
|
||||
})
|
||||
.collect();
|
||||
append_logs(
|
||||
&job.id,
|
||||
|
||||
@@ -219,9 +219,10 @@ function parseVolumeAnnotations(code: string, commentPrefix: string): AssetWithA
|
||||
if (!trimmed || trimmed.slice(0, 5).toLowerCase() === '<?php') continue
|
||||
if (!trimmed.startsWith(commentPrefix)) break
|
||||
const after = trimmed.slice(commentPrefix.length).trim()
|
||||
const match = after.match(/^volume:\s*(\S+)/)
|
||||
// `volume:ro <name>`; with a space after the colon, `ro` is the volume's name
|
||||
const match = after.match(/^volume:(ro(?=\s))?\s*(\S+)/)
|
||||
if (match) {
|
||||
volumes.push({ kind: 'volume', path: match[1], access_type: 'rw' })
|
||||
volumes.push({ kind: 'volume', path: match[2], access_type: match[1] ? 'r' : 'rw' })
|
||||
}
|
||||
}
|
||||
return volumes
|
||||
|
||||
Reference in New Issue
Block a user