feat: mount volumes read-only without the exclusive lease (#12410)

* feat: mount volumes read-only without the exclusive lease

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PjsYvvmmTkuELF5qpS2Hs3

* fix: gate lease-free volume reads on the fleet version

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PjsYvvmmTkuELF5qpS2Hs3

* fix: keep held volumes counted by cache eviction, restore MIN_VERSION in tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PjsYvvmmTkuELF5qpS2Hs3

* chore: update ee-repo-ref to 38904055ab0d104006278322299bdbcea32c88d1

This commit updates the EE repository reference after PR #980 was merged in windmill-ee-private.

Previous ee-repo-ref: 1fe25fa2530efdba4f8a3bf5dbfbb6f85e616ce7

New ee-repo-ref: 38904055ab0d104006278322299bdbcea32c88d1

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
Ruben Fiszel
2026-11-13 07:55:54 +01:00
committed by GitHub
co-authored by Claude Opus 5.5 windmill-internal-app[bot]
parent 164528563d
commit 70711bec68
34 changed files with 827 additions and 128 deletions
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET generation = generation + 1 - generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "159f2030de3ba169925236c8c23720e820a132958cd64483cba8c8e40bf2a563"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2",
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [],
"parameters": {
@@ -14,5 +14,5 @@
},
"nullable": []
},
"hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34"
"hash": "19c481a0d6ecd4c0dfab6973e4732c2146953253ba6062ab05b34b6b35e41f66"
}
@@ -1,25 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc"
}
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET generation = generation + 1 - generation % 2\n WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "2ae1bfc683bbefadf8777b06898fcf48ebbdff22e059a33bc9188f38460f964c"
}
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET generation = 2 WHERE workspace_id = 'test-workspace' AND name = 'test-vol'",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "2d67831f591efa5c37047f18a7dd8d33d862bba924ca098a1423e782df88fb1b"
}
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT extra_perms, created_by, generation, coalesce(lease_until > now(), false) AS \"write_live!\"\n FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "generation",
"type_info": "Int8"
},
{
"ordinal": 3,
"name": "write_live!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
null
]
},
"hash": "2eb0de1f862785cc61aa5cc66641b2774bb0851593a55571ee9e7c2479c2919a"
}
@@ -0,0 +1,43 @@
{
"db_name": "PostgreSQL",
"query": "SELECT j.permissioned_as, v.extra_perms, v.created_by, v.generation\n FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN volume v ON v.workspace_id = j.workspace_id AND v.name = $3\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.worker = $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "permissioned_as",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "generation",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false
]
},
"hash": "423dc0716c7a6f63e311a069f67158960cd9df94fa0540f299fa31cd41a82c5a"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4,\n generation = volume.generation + volume.generation % 2\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
"describe": {
"columns": [
{
@@ -21,5 +21,5 @@
false
]
},
"hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6"
"hash": "54549a1aca624587a1f91c00f54fdcc887172d6a4c2081af6f2fafe249c5c29a"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT leased_by, generation FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "leased_by",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "generation",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
true,
false
]
},
"hash": "8338b741d782c6c4273b1c98c7e62763959428e37f83ff376cd9018d11ea21ed"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = 'test-workspace'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "85a9967ab419bd1f09a2be7751ff8d15cb8431bf05fb34a7b99d03a2a86e48ca"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33"
"hash": "85ac712e057207d1db48303925b2ca08479830f6de789cbc64661509fccba9ce"
}
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)\n VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + interval '1 hour', 'other-worker', 1)",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a26528bf01138bfad0c4989c04fd74e3c0aacd9913b7524edcf69f58115875fc"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98"
"hash": "b0350f074557a476882697046c3573d6ac532b374d9b2470322f70b875eaec87"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"query": "UPDATE volume SET lease_until = NULL, leased_by = NULL, generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89"
"hash": "b6d05e743b558e82bca080ab119aef7949ceae08e71a0191ce185420e037913d"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT leased_by FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "leased_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
true
]
},
"hash": "cd5cdad3449dd5f58c88d3a7b99158da707d8f23513c155deb430b702d932263"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)\n VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + make_interval(secs => $1), 'other-worker', $2)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Float8",
"Int8"
]
},
"nullable": []
},
"hash": "dfe0a2a6e4ab88d5341415bdc19a6982a22cd119e5e1dbe2b6149883e71a0066"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT generation FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "generation",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "e0838b093a54f1f9735452f62068612d76144ac859fc904d882a58d990de952d"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT generation, coalesce(lease_until > now(), false) AS \"write_live!\"\n FROM volume WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "generation",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "write_live!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "e938951c0c6ddd268db7a0dc69484446d4e869951188a0add41036d4fd095298"
}
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4,\n generation = volume.generation + volume.generation % 2\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "e99555474694f563991558f8f1447c2eeec0ece06b69d730c6b3d3dffcafeb26"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5",
"query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL,\n generation = generation + generation % 2\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5",
"describe": {
"columns": [],
"parameters": {
@@ -14,5 +14,5 @@
},
"nullable": []
},
"hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca"
"hash": "f5946a3968df859f668d6197562c10024ffc24f108ca48d5b3fa98e489d931ea"
}
+1
View File
@@ -17671,6 +17671,7 @@ name = "windmill-worker-volumes"
version = "1.871.1"
dependencies = [
"bytes",
"fs4",
"futures",
"lazy_static",
"md-5 0.10.6",
+1 -1
View File
@@ -1 +1 @@
540d78337de9da0ec56f59c911f9feac5d0df608
38904055ab0d104006278322299bdbcea32c88d1
@@ -0,0 +1 @@
ALTER TABLE volume DROP COLUMN IF EXISTS generation;
@@ -0,0 +1,3 @@
-- Odd while a writer is changing the volume's objects, even otherwise. Read-only mounts take no
-- lease and use it to detect a write that overlapped their pull.
ALTER TABLE volume ADD COLUMN IF NOT EXISTS generation BIGINT NOT NULL DEFAULT 0;
+140
View File
@@ -528,6 +528,146 @@ export function main() {
Ok(())
}
/// Every test that sets the fleet's minimum version holds this: tests share the process.
#[cfg(all(feature = "parquet", feature = "enterprise"))]
static FLEET_VERSION: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[cfg(all(feature = "parquet", feature = "enterprise"))]
struct OnDrop<F: FnMut()>(F);
#[cfg(all(feature = "parquet", feature = "enterprise"))]
impl<F: FnMut()> Drop for OnDrop<F> {
fn drop(&mut self) {
(self.0)()
}
}
/// An agent worker's `volume:ro` pull takes no lease, waits for a write in progress to close,
/// and writes nothing back.
#[cfg(all(feature = "parquet", feature = "enterprise"))]
#[sqlx::test(fixtures("base"))]
async fn test_agent_worker_read_only_volume(db: Pool<Postgres>) -> anyhow::Result<()> {
let leased_by = run_agent_read_only_volume(db, true).await?;
assert_eq!(leased_by.as_deref(), Some("other-worker"));
Ok(())
}
/// Until every worker marks its writes, a `volume:ro` mount pulls under the lease, and holding
/// it must not make the mount writable.
#[cfg(all(feature = "parquet", feature = "enterprise"))]
#[sqlx::test(fixtures("base"))]
async fn test_agent_worker_read_only_volume_leased(db: Pool<Postgres>) -> anyhow::Result<()> {
let leased_by = run_agent_read_only_volume(db, false).await?;
assert_eq!(leased_by, None, "the lease should be released");
Ok(())
}
/// Another worker holds the volume and rewrites `hello.txt` after 3s: under a write window
/// (odd generation, lease held for an hour) when the fleet supports lease-free pulls, or as an
/// older worker would, with no window and a lease that lapses after 4s. Either way the
/// `volume:ro` job must read the rewritten file and leave nothing behind. Returns `leased_by`.
#[cfg(all(feature = "parquet", feature = "enterprise"))]
async fn run_agent_read_only_volume(
db: Pool<Postgres>,
fleet_supports_lease_free: bool,
) -> anyhow::Result<Option<String>> {
use windmill_common::min_version::{MIN_VERSION, MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES};
let _fleet = FLEET_VERSION.lock().await;
// Put back for the rest of the binary, panics included
let before = MIN_VERSION.load_full();
let _restore = OnDrop(move || MIN_VERSION.store(before.clone()));
let mut fleet = MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES.version().clone();
if !fleet_supports_lease_free {
fleet.minor -= 1;
}
MIN_VERSION.store(std::sync::Arc::new(fleet));
let (_client, port, _server) = init_client_agent_mode(db.clone()).await;
let storage_dir = tempfile::tempdir()?;
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_dir.path().to_string_lossy(),
"public_resource": null,
"advanced_permissions": null,
"volume_storage": "primary"
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = 'test-workspace'",
lfs_config,
)
.execute(&db)
.await?;
let vol_dir = storage_dir.path().join("volumes/test-workspace/test-vol");
std::fs::create_dir_all(&vol_dir)?;
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
let (lease_secs, generation) = if fleet_supports_lease_free {
(3600, 1)
} else {
(4, 0)
};
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)
VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + make_interval(secs => $1), 'other-worker', $2)",
lease_secs as f64,
generation
)
.execute(&db)
.await?;
let db_c = db.clone();
let hello = vol_dir.join("hello.txt");
let writer = tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
std::fs::write(&hello, b"hello after the write").unwrap();
if fleet_supports_lease_free {
sqlx::query!(
"UPDATE volume SET generation = 2 WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
)
.execute(&db_c)
.await?;
}
Ok::<_, sqlx::Error>(())
});
let code = r#"// volume:ro test-vol data
import { readFileSync, writeFileSync } from "fs";
export function main() {
const content = readFileSync("data/hello.txt", "utf-8");
writeFileSync("data/output.txt", "discarded");
return { read_content: content };
}"#;
let uuid = RunJob::from(JobPayload::Code(bun_code(code)))
.push(&db)
.await;
let listener = listen_for_completed_jobs(&db).await;
let conn = testing_http_connection_with_tags(
port,
vec!["bun".into(), "flow".into(), "dependency".into()],
)
.await;
in_test_worker(conn, listener.find(&uuid), port).await;
writer.await??;
let result = completed_job(uuid, &db).await;
assert!(result.success, "job should succeed: {:?}", result.result);
assert_eq!(
result.json_result().unwrap()["read_content"],
json!("hello after the write"),
"the pull must wait for the other worker's write"
);
assert!(
!vol_dir.join("output.txt").exists(),
"a read-only mount must not write back"
);
Ok(sqlx::query_scalar!(
"SELECT leased_by FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
)
.fetch_one(&db)
.await?)
}
/// Test the volume release endpoint (error/cancel path).
#[cfg(feature = "parquet")]
#[sqlx::test(fixtures("base"))]
+147 -48
View File
@@ -468,7 +468,11 @@ fn test_volume_comment_prefix_selection() {
fn test_volume_mount_struct() {
use windmill_worker_volumes::VolumeMount;
let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() };
let mount = VolumeMount {
name: "test-vol".to_string(),
target: "/mnt/data".to_string(),
read_only: false,
};
assert_eq!(mount.name, "test-vol");
assert_eq!(mount.target, "/mnt/data");
}
@@ -501,11 +505,13 @@ fn test_volume_nsjail_mount() {
use std::path::Path;
use windmill_worker_volumes::volume_nsjail_mount;
let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data");
let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data", false);
assert!(result.contains("src: \"/tmp/volumes/data\""));
assert!(result.contains("dst: \"/mnt/data\""));
assert!(result.contains("is_bind: true"));
assert!(result.contains("rw: true"));
let read_only = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data", true);
assert!(read_only.contains("rw: false"));
}
#[test]
@@ -599,53 +605,10 @@ async fn run_volume_sql_worker_e2e(
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// 1. Set up filesystem-based object storage in a temp dir
let storage_dir = tempfile::tempdir()?;
let storage_root = storage_dir.path().to_string_lossy().to_string();
let (_storage_dir, vol_dir) = setup_filesystem_volume(&db).await?;
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_root,
"public_resource": null,
"advanced_permissions": null,
"volume_storage": "primary"
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
lfs_config,
"test-workspace"
)
.execute(&db)
.await?;
// 2. Pre-populate the volume with a file (workspace-namespaced path)
let vol_dir = storage_dir
.path()
.join("volumes")
.join("test-workspace")
.join("test-vol");
std::fs::create_dir_all(&vol_dir)?;
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
// 3. Push the job and run with SQL-connected worker
let job = JobPayload::Code(RawCode {
hash: None,
content: code.to_string(),
path: None,
language,
lock: None,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
modules: None,
tag: None,
});
let result = run_job_in_new_worker_until_complete(&db, false, job, port).await;
let result =
run_job_in_new_worker_until_complete(&db, false, raw_code_job(language, code), port).await;
assert!(result.success, "job should succeed: {:?}", result.result);
let json = result.json_result().expect("should have JSON result");
@@ -682,3 +645,139 @@ async fn run_volume_sql_worker_e2e(
Ok(())
}
/// A `volume:ro` mount takes no lease: it runs while another worker holds the volume, waits
/// only for that worker's write in progress (odd generation) to close, and writes nothing back.
#[cfg(all(feature = "parquet", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_read_only_volume_ignores_lease_and_waits_for_write(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::min_version::{MIN_VERSION, MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES};
struct OnDrop<F: FnMut()>(F);
impl<F: FnMut()> Drop for OnDrop<F> {
fn drop(&mut self) {
(self.0)()
}
}
// Put back for the rest of the binary, panics included
let before = MIN_VERSION.load_full();
let _restore = OnDrop(move || MIN_VERSION.store(before.clone()));
MIN_VERSION.store(std::sync::Arc::new(
MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES.version().clone(),
));
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let (_storage_dir, vol_dir) = setup_filesystem_volume(&db).await?;
sqlx::query!(
"INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by, generation)
VALUES ('test-workspace', 'test-vol', 0, 'test-user', now() + interval '1 hour', 'other-worker', 1)"
)
.execute(&db)
.await?;
let db_c = db.clone();
let hello = vol_dir.join("hello.txt");
let writer = tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
// A different length: the cache cannot tell same-size rewrites apart on a store
// whose etags are not content hashes.
std::fs::write(&hello, b"hello after the write").unwrap();
sqlx::query!(
"UPDATE volume SET generation = 2 WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
)
.execute(&db_c)
.await
});
let code = r#"// volume:ro test-vol data
import { readFileSync, writeFileSync } from "fs";
export function main() {
const content = readFileSync("data/hello.txt", "utf-8");
writeFileSync("data/output.txt", "discarded");
return { read_content: content };
}"#;
let result =
run_job_in_new_worker_until_complete(&db, false, raw_code_job(ScriptLang::Bun, code), port)
.await;
writer.await??;
assert!(result.success, "job should succeed: {:?}", result.result);
assert_eq!(
result.json_result().unwrap()["read_content"],
json!("hello after the write"),
"the pull must wait for the write in progress"
);
assert!(
!vol_dir.join("output.txt").exists(),
"a read-only mount must not write back"
);
let row = sqlx::query!(
"SELECT leased_by, generation FROM volume WHERE workspace_id = 'test-workspace' AND name = 'test-vol'"
)
.fetch_one(&db)
.await?;
assert_eq!(row.leased_by.as_deref(), Some("other-worker"));
assert_eq!(row.generation, 2);
Ok(())
}
#[cfg(feature = "parquet")]
fn raw_code_job(language: ScriptLang, code: &str) -> JobPayload {
JobPayload::Code(RawCode {
hash: None,
content: code.to_string(),
path: None,
language,
lock: None,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
modules: None,
tag: None,
})
}
/// Filesystem object storage as the workspace's volume storage, holding `test-vol` with
/// `hello.txt`. Returns the storage dir (deleted on drop) and the volume's object dir.
#[cfg(feature = "parquet")]
async fn setup_filesystem_volume(
db: &Pool<Postgres>,
) -> anyhow::Result<(tempfile::TempDir, std::path::PathBuf)> {
// 1. Set up filesystem-based object storage in a temp dir
let storage_dir = tempfile::tempdir()?;
let storage_root = storage_dir.path().to_string_lossy().to_string();
let lfs_config = json!({
"type": "FilesystemStorage",
"root_path": storage_root,
"public_resource": null,
"advanced_permissions": null,
"volume_storage": "primary"
});
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
lfs_config,
"test-workspace"
)
.execute(db)
.await?;
// 2. Pre-populate the volume with a file (workspace-namespaced path)
let vol_dir = storage_dir
.path()
.join("volumes")
.join("test-workspace")
.join("test-vol");
std::fs::create_dir_all(&vol_dir)?;
std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?;
Ok((storage_dir, vol_dir))
}
@@ -86,7 +86,7 @@ fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> {
/// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): `<prefix>
/// volume: <path>` lines in the leading comment block, each an `rw` volume
/// asset. Scanning stops at the first non-comment line; blank lines and PHP's
/// asset (`r` for `volume:ro <path>`). Scanning stops at the first non-comment line; blank lines and PHP's
/// opening tag line are skipped whole (the tag may carry code, so an annotation
/// must sit on its own line below it), exactly like the frontend.
fn parse_volume_annotations(content: &str, prefix: &str) -> Vec<AssetWithAltAccessType> {
@@ -105,11 +105,16 @@ fn parse_volume_annotations(content: &str, prefix: &str) -> Vec<AssetWithAltAcce
};
let after = after.trim();
if let Some(rest) = after.strip_prefix("volume:") {
// `volume:ro <name>`; with a space after the colon, `ro` is the volume's name
let (access_type, rest) = match rest.strip_prefix("ro") {
Some(r) if r.starts_with(char::is_whitespace) => (AssetUsageAccessType::R, r),
_ => (AssetUsageAccessType::RW, rest),
};
if let Some(path) = rest.trim().split_whitespace().next() {
volumes.push(AssetWithAltAccessType {
path: path.to_string(),
kind: AssetKind::Volume,
access_type: Some(AssetUsageAccessType::RW),
access_type: Some(access_type),
alt_access_type: None,
columns: None,
});
@@ -265,6 +270,24 @@ mod tests {
assert_eq!(vols[0].access_type, Some(AssetUsageAccessType::RW));
}
#[test]
fn read_only_volume_annotation_is_a_read() {
let content = "// volume:ro models /opt/models\n// volume: ro /tmp/ro\nconsole.log(1)\n";
let got = effective_script_assets(&ScriptLang::Bun, content, None).unwrap();
let vols: Vec<_> = got
.iter()
.filter(|a| a.kind == AssetKind::Volume)
.map(|a| (a.path.as_str(), a.access_type))
.collect();
assert_eq!(
vols,
vec![
("models", Some(AssetUsageAccessType::R)),
("ro", Some(AssetUsageAccessType::RW))
]
);
}
#[test]
fn php_volume_annotations_survive_the_open_tag() {
let content = "<?php\n\n// volume: my_vol\nfunction main() {}\n";
@@ -39,6 +39,11 @@ pub const MIN_VERSION_SUPPORTS_VARIABLE_VALUE_WIDENING: VC =
// to accept one once this is met. Only worker versions are seen here, not servers. Must name
// the release this ships in.
pub const MIN_VERSION_SUPPORTS_ED25519_LICENSE: VC = vc(1, 838, 0, "Ed25519 license keys");
// `volume:ro` mounts pull without the volume's lease and rely on every writer marking its
// write in `volume.generation`. A worker that predates it writes without that mark, so a
// lease-free pull overlapping its write would keep a mix of old and new files; until the
// whole fleet has it, read-only mounts take the lease. Must name the release this ships in.
pub const MIN_VERSION_SUPPORTS_LEASE_FREE_VOLUMES: VC = vc(1, 872, 0, "Lease-free volume reads");
pub const MIN_VERSION_SUPPORTS_NODE_DEBOUNCING: VC = vc(1, 658, 0, "Flow node debouncing");
pub const MIN_VERSION_SUPPORTS_TOKEN_HASH: VC = vc(1, 659, 0, "Token hash storage");
pub const MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: VC = vc(1, 602, 0, "Sync jobs debouncing");
+26 -9
View File
@@ -655,6 +655,18 @@ pub static DEDICATED_WORKER_RELOAD: std::sync::LazyLock<tokio::sync::Notify> =
std::sync::LazyLock::new(tokio::sync::Notify::new);
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
/// The server's error message, which callers match on to tell a retryable refusal (a volume
/// leased or being written by another worker) from a failure.
async fn agent_error_body(response: reqwest::Response) -> String {
response
.text()
.await
.unwrap_or_default()
.chars()
.take(1000)
.collect()
}
#[derive(Clone)]
pub struct HttpClient {
pub client: ClientWithMiddleware,
@@ -693,11 +705,12 @@ impl HttpClient {
if status.is_success() {
Ok(response.json().await?)
} else {
Err(anyhow::anyhow!(format!(
"HTTP agent request POST {} failed {}",
Err(anyhow::anyhow!(
"HTTP agent request POST {} failed {}: {}",
url,
response.status()
)))
status,
agent_error_body(response).await
))
}
}
@@ -713,11 +726,12 @@ impl HttpClient {
if status.is_success() {
Ok(response.json().await?)
} else {
Err(anyhow::anyhow!(format!(
"HTTP agent request GET {} failed {}",
Err(anyhow::anyhow!(
"HTTP agent request GET {} failed {}: {}",
url,
response.status()
)))
status,
agent_error_body(response).await
))
}
}
@@ -3962,7 +3976,10 @@ mod tests {
#[test]
fn dedicated_worker_patterns_are_script_folders() {
assert_eq!(dedicated_worker_pattern_prefix("f/dw/*"), Some("f/dw/"));
assert_eq!(dedicated_worker_pattern_prefix("u/alice/*"), Some("u/alice/"));
assert_eq!(
dedicated_worker_pattern_prefix("u/alice/*"),
Some("u/alice/")
);
assert_eq!(dedicated_worker_pattern_prefix("f/dw/script"), None);
assert_eq!(dedicated_worker_pattern_prefix("f/dw*"), None);
assert_eq!(dedicated_worker_pattern_prefix("/*"), None);
@@ -26,6 +26,7 @@ serde_json.workspace = true
regex.workspace = true
lazy_static.workspace = true
md-5.workspace = true
fs4.workspace = true
[dev-dependencies]
tempfile.workspace = true
+147 -26
View File
@@ -52,6 +52,8 @@ lazy_static::lazy_static! {
pub struct VolumeMount {
pub name: String,
pub target: String,
/// `volume:ro`: pulled without the volume's lease and never synced back.
pub read_only: bool,
}
/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while
@@ -220,6 +222,15 @@ fn is_php_open_tag(trimmed_line: &str) -> bool {
.is_some_and(|p| p.eq_ignore_ascii_case("<?php"))
}
/// `ro` glued to `volume:` (`volume:ro <name> <target>`) marks a read-only mount. With a space
/// after the colon, `ro` is a volume's name: `volume: ro /tmp/x` mounts the volume `ro`.
pub fn split_read_only_flag(after_colon: &str) -> (bool, &str) {
match after_colon.strip_prefix("ro") {
Some(rest) if rest.starts_with(char::is_whitespace) => (true, rest),
_ => (false, after_colon),
}
}
/// Scans the leading comment block for `<prefix> volume: <name> <target>` lines,
/// stopping at the first line that is neither blank nor a comment. A PHP script
/// opens with `<?php`, which is not a comment, so that line is skipped like a blank
@@ -238,14 +249,17 @@ pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec<Volu
}
let after_prefix = trimmed[comment_prefix.len()..].trim();
if let Some(rest) = after_prefix.strip_prefix("volume:") {
let rest = rest.trim();
let mut parts = rest.splitn(2, char::is_whitespace);
let (read_only, rest) = split_read_only_flag(rest);
let mut parts = rest.trim().splitn(2, char::is_whitespace);
if let (Some(name), Some(target)) = (parts.next(), parts.next()) {
let name = name.trim();
let target = target.trim();
if !name.is_empty() && !target.is_empty() {
volumes
.push(VolumeMount { name: name.to_string(), target: target.to_string() });
volumes.push(VolumeMount {
name: name.to_string(),
target: target.to_string(),
read_only,
});
}
}
}
@@ -263,7 +277,11 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/data".to_string(),
read_only: false
}]
);
}
@@ -273,7 +291,11 @@ mod tests {
let result = parse_volume_annotations(content, "//");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/data".to_string(),
read_only: false
}]
);
}
@@ -284,7 +306,11 @@ mod tests {
let result = parse_volume_annotations(content, "//");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/data".to_string(),
read_only: false
}]
);
}
@@ -295,9 +321,21 @@ mod tests {
assert_eq!(
result,
vec![
VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() },
VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() },
VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() },
VolumeMount {
name: "data1".to_string(),
target: "/tmp/data1".to_string(),
read_only: false
},
VolumeMount {
name: "data2".to_string(),
target: "/tmp/data2".to_string(),
read_only: false
},
VolumeMount {
name: "models".to_string(),
target: "/opt/models".to_string(),
read_only: false
},
]
);
}
@@ -308,7 +346,37 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/data".to_string(),
read_only: false
}]
);
}
#[test]
fn parse_read_only_volume() {
let content = "# volume:ro models /opt/models\n# volume: ro /tmp/ro\n# volume:robots /tmp/r\ndef main():\n pass";
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![
VolumeMount {
name: "models".to_string(),
target: "/opt/models".to_string(),
read_only: true,
},
VolumeMount {
name: "ro".to_string(),
target: "/tmp/ro".to_string(),
read_only: false
},
VolumeMount {
name: "robots".to_string(),
target: "/tmp/r".to_string(),
read_only: false,
},
]
);
}
@@ -332,7 +400,11 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }]
vec![VolumeMount {
name: "data1".to_string(),
target: "/tmp/data1".to_string(),
read_only: false
}]
);
}
@@ -344,8 +416,16 @@ mod tests {
assert_eq!(
result,
vec![
VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() },
VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() },
VolumeMount {
name: "data1".to_string(),
target: "/tmp/data1".to_string(),
read_only: false
},
VolumeMount {
name: "data2".to_string(),
target: "/tmp/data2".to_string(),
read_only: false
},
]
);
}
@@ -357,7 +437,11 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }]
vec![VolumeMount {
name: "good".to_string(),
target: "/tmp/good".to_string(),
read_only: false
}]
);
}
@@ -367,7 +451,11 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }]
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/data".to_string(),
read_only: false
}]
);
}
@@ -380,6 +468,7 @@ mod tests {
vec![VolumeMount {
name: "mydata".to_string(),
target: "/tmp/my data dir".to_string(),
read_only: false,
}]
);
}
@@ -390,7 +479,11 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }]
vec![VolumeMount {
name: "my-data_v2".to_string(),
target: "/tmp/data".to_string(),
read_only: false
}]
);
}
@@ -467,7 +560,11 @@ mod tests {
let result = parse_volume_annotations(content, "//");
assert_eq!(
result,
vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }]
vec![VolumeMount {
name: "agent-memory".to_string(),
target: ".claude".to_string(),
read_only: false
}]
);
}
@@ -477,7 +574,11 @@ mod tests {
let result = parse_volume_annotations(content, "#");
assert_eq!(
result,
vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }]
vec![VolumeMount {
name: "data".to_string(),
target: "data/models".to_string(),
read_only: false
}]
);
}
@@ -568,7 +669,11 @@ mod tests {
#[test]
fn validate_mounts_rejects_too_many() {
let mounts: Vec<VolumeMount> = (0..11)
.map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) })
.map(|i| VolumeMount {
name: format!("v{:02}", i),
target: format!("t{}", i),
read_only: false,
})
.collect();
assert!(validate_volume_mounts(&mounts).is_err());
}
@@ -576,8 +681,16 @@ mod tests {
#[test]
fn validate_mounts_rejects_duplicate_name() {
let mounts = vec![
VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() },
VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() },
VolumeMount {
name: "data".to_string(),
target: "/tmp/a".to_string(),
read_only: false,
},
VolumeMount {
name: "data".to_string(),
target: "/tmp/b".to_string(),
read_only: false,
},
];
assert!(validate_volume_mounts(&mounts).is_err());
}
@@ -585,8 +698,16 @@ mod tests {
#[test]
fn validate_mounts_rejects_duplicate_target() {
let mounts = vec![
VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() },
VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() },
VolumeMount {
name: "v1".to_string(),
target: "/tmp/data".to_string(),
read_only: false,
},
VolumeMount {
name: "v2".to_string(),
target: "/tmp/data".to_string(),
read_only: false,
},
];
assert!(validate_volume_mounts(&mounts).is_err());
}
@@ -594,8 +715,8 @@ mod tests {
#[test]
fn validate_mounts_ok() {
let mounts = vec![
VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() },
VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() },
VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string(), read_only: false },
VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string(), read_only: false },
];
assert!(validate_volume_mounts(&mounts).is_ok());
}
@@ -25,15 +25,16 @@ pub async fn download_volume(
}
#[cfg(not(feature = "private"))]
pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String {
pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str, _read_only: bool) -> String {
String::new()
}
#[cfg(not(feature = "private"))]
pub async fn sync_volume_back(
pub async fn sync_volume_back<F: std::future::Future<Output = error::Result<()>>>(
_client: Arc<dyn ObjectStore>,
_state: &VolumeState,
_workspace_id: &str,
_open_write_window: F,
) -> error::Result<SyncStats> {
Err(error::Error::internal_err(
"Volumes are not available in this build".to_string(),
+4 -1
View File
@@ -6972,7 +6972,10 @@ mount {{
if !volume_mounts.is_empty() {
let vol_summary: Vec<String> = volume_mounts
.iter()
.map(|v| format!("'{}' -> {}", v.name, v.target))
.map(|v| {
let mode = if v.read_only { " (read-only)" } else { "" };
format!("'{}' -> {}{mode}", v.name, v.target)
})
.collect();
append_logs(
&job.id,
+3 -2
View File
@@ -219,9 +219,10 @@ function parseVolumeAnnotations(code: string, commentPrefix: string): AssetWithA
if (!trimmed || trimmed.slice(0, 5).toLowerCase() === '<?php') continue
if (!trimmed.startsWith(commentPrefix)) break
const after = trimmed.slice(commentPrefix.length).trim()
const match = after.match(/^volume:\s*(\S+)/)
// `volume:ro <name>`; with a space after the colon, `ro` is the volume's name
const match = after.match(/^volume:(ro(?=\s))?\s*(\S+)/)
if (match) {
volumes.push({ kind: 'volume', path: match[1], access_type: 'rw' })
volumes.push({ kind: 'volume', path: match[2], access_type: match[1] ? 'r' : 'rw' })
}
}
return volumes