feat(backend): add sandbox snapshots & volumes for nsjail

Add support for custom rootfs snapshots and persistent volumes in nsjail
sandboxes, enabling users to run scripts with pre-built environments and
shared persistent storage.

- DB migration: sandbox_snapshot + sandbox_volume tables
- API: CRUD endpoints under /api/w/{ws}/sandbox/
- Common: annotation parser for # sandbox: and # volume: directives
- Worker: S3-backed snapshot cache, overlayfs mount, volume sync
- Executors: nsjail config post-processing to swap rootfs when overlay active
- Tests: 28 tests covering parsing, config generation, and end-to-end flows

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend/Cargo.lock

@@ -17361,6 +17361,7 @@ dependencies = [
  "derive_more 1.0.0",
  "dotenv",
  "eventsource-stream",
+ "flate2",
  "flume",
  "futures",
  "gcp_auth",
@@ -17402,6 +17403,7 @@ dependencies = [
  "sha2 0.10.9",
  "sqlx",
  "tar",
+ "tempfile",
  "tiberius",
  "tokio",
  "tokio-postgres 0.7.13",

backend/migrations/20260217000000_sandbox_snapshots_volumes.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS sandbox_volume;
+DROP TABLE IF EXISTS sandbox_snapshot;

backend/migrations/20260217000000_sandbox_snapshots_volumes.up.sql

@@ -0,0 +1,32 @@
+-- Add sandbox_snapshot and sandbox_volume tables
+
+CREATE TABLE sandbox_snapshot (
+    workspace_id  VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    name          VARCHAR(255) NOT NULL,
+    tag           VARCHAR(255) NOT NULL DEFAULT 'latest',
+    s3_key        TEXT NOT NULL,
+    content_hash  VARCHAR(64) NOT NULL DEFAULT '',
+    docker_image  TEXT NOT NULL,
+    setup_script  TEXT,
+    size_bytes    BIGINT,
+    status        VARCHAR(20) NOT NULL DEFAULT 'pending',
+    build_error   TEXT,
+    build_job_id  UUID,
+    created_by    VARCHAR(255) NOT NULL,
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at    TIMESTAMPTZ NOT NULL DEFAULT now(),
+    extra_perms   JSONB NOT NULL DEFAULT '{}'::jsonb,
+    PRIMARY KEY (workspace_id, name, tag)
+);
+
+CREATE TABLE sandbox_volume (
+    workspace_id  VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    name          VARCHAR(255) NOT NULL,
+    s3_key        TEXT NOT NULL,
+    size_bytes    BIGINT,
+    created_by    VARCHAR(255) NOT NULL,
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at    TIMESTAMPTZ NOT NULL DEFAULT now(),
+    extra_perms   JSONB NOT NULL DEFAULT '{}'::jsonb,
+    PRIMARY KEY (workspace_id, name)
+);

backend/windmill-api/src/lib.rs

@@ -131,6 +131,7 @@ pub mod oidc_ee;
 mod oidc_oss;
 mod raw_apps;
 mod resources;
+mod sandbox;
 #[cfg(feature = "private")]
 pub mod saml_ee;
 mod saml_oss;
@@ -509,6 +510,7 @@ pub async fn run_server(
                         .nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service())
                         .nest("/raw_apps", raw_apps::workspaced_service())
                         .nest("/resources", resources::workspaced_service())
+                        .nest("/sandbox", sandbox::workspaced_service())
                         .nest("/schedules", windmill_api_schedule::workspaced_service())
                         .nest("/scripts", scripts::workspaced_service())
                         .nest(

backend/windmill-api/src/sandbox.rs

@@ -0,0 +1,303 @@
+use axum::{
+    extract::{Extension, Path, Query},
+    routing::{delete, get, post},
+    Json, Router,
+};
+use serde::Deserialize;
+use windmill_common::{
+    db::DB,
+    error::{self, JsonResult},
+    sandbox::{SandboxSnapshot, SandboxVolume},
+    utils::require_admin,
+};
+
+use crate::db::ApiAuthed;
+
+#[cfg(feature = "parquet")]
+async fn delete_s3_object(s3_key: &str) {
+    if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
+        let path = object_store::path::Path::from(s3_key);
+        if let Err(e) = os.delete(&path).await {
+            tracing::warn!("Failed to delete S3 object {s3_key}: {e}");
+        }
+    }
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn delete_s3_object(_s3_key: &str) {}
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/snapshots", get(list_snapshots).post(create_snapshot))
+        .route("/snapshots/:name/:tag", get(get_snapshot).delete(delete_snapshot))
+        .route("/snapshots/:name", delete(delete_snapshot_all_tags))
+        .route("/snapshots/:name/:tag/rebuild", post(rebuild_snapshot))
+        .route("/volumes", get(list_volumes).post(create_volume))
+        .route("/volumes/:name", get(get_volume).delete(delete_volume))
+}
+
+#[derive(Deserialize)]
+struct ListSnapshotsQuery {
+    name: Option<String>,
+}
+
+async fn list_snapshots(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Query(query): Query<ListSnapshotsQuery>,
+) -> JsonResult<Vec<SandboxSnapshot>> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let rows = if let Some(name) = query.name {
+        sqlx::query_as!(
+            SandboxSnapshot,
+            "SELECT workspace_id, name, tag, s3_key, content_hash, docker_image, setup_script, \
+             size_bytes, status, build_error, build_job_id, created_by, created_at, updated_at, \
+             extra_perms \
+             FROM sandbox_snapshot WHERE workspace_id = $1 AND name = $2 ORDER BY created_at DESC",
+            &w_id,
+            &name,
+        )
+        .fetch_all(&db)
+        .await?
+    } else {
+        sqlx::query_as!(
+            SandboxSnapshot,
+            "SELECT workspace_id, name, tag, s3_key, content_hash, docker_image, setup_script, \
+             size_bytes, status, build_error, build_job_id, created_by, created_at, updated_at, \
+             extra_perms \
+             FROM sandbox_snapshot WHERE workspace_id = $1 ORDER BY created_at DESC",
+            &w_id,
+        )
+        .fetch_all(&db)
+        .await?
+    };
+    Ok(Json(rows))
+}
+
+#[derive(Deserialize)]
+struct CreateSnapshot {
+    name: String,
+    #[serde(default = "default_tag")]
+    tag: String,
+    docker_image: String,
+    setup_script: Option<String>,
+}
+
+fn default_tag() -> String {
+    "latest".to_string()
+}
+
+async fn create_snapshot(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(body): Json<CreateSnapshot>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let s3_key = format!(
+        "sandbox/snapshots/{}/{}/{}.tar.gz",
+        w_id, body.name, body.tag
+    );
+    sqlx::query!(
+        "INSERT INTO sandbox_snapshot \
+         (workspace_id, name, tag, s3_key, docker_image, setup_script, created_by, status) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending') \
+         ON CONFLICT (workspace_id, name, tag) DO UPDATE SET \
+         docker_image = $5, setup_script = $6, status = 'pending', \
+         build_error = NULL, updated_at = now()",
+        &w_id,
+        &body.name,
+        &body.tag,
+        &s3_key,
+        &body.docker_image,
+        body.setup_script.as_deref(),
+        &authed.username,
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!(
+        "Snapshot {}:{} created with status 'pending'",
+        body.name, body.tag
+    ))
+}
+
+async fn get_snapshot(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name, tag)): Path<(String, String, String)>,
+) -> JsonResult<SandboxSnapshot> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let row = sqlx::query_as!(
+        SandboxSnapshot,
+        "SELECT workspace_id, name, tag, s3_key, content_hash, docker_image, setup_script, \
+         size_bytes, status, build_error, build_job_id, created_by, created_at, updated_at, \
+         extra_perms \
+         FROM sandbox_snapshot WHERE workspace_id = $1 AND name = $2 AND tag = $3",
+        &w_id,
+        &name,
+        &tag,
+    )
+    .fetch_optional(&db)
+    .await?;
+    let row = windmill_common::utils::not_found_if_none(row, "sandbox_snapshot", &name)?;
+    Ok(Json(row))
+}
+
+async fn delete_snapshot(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name, tag)): Path<(String, String, String)>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+
+    let row = sqlx::query!(
+        "DELETE FROM sandbox_snapshot WHERE workspace_id = $1 AND name = $2 AND tag = $3 \
+         RETURNING s3_key",
+        &w_id,
+        &name,
+        &tag,
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    if let Some(row) = row {
+        delete_s3_object(&row.s3_key).await;
+    }
+
+    Ok(format!("Deleted snapshot {}:{}", name, tag))
+}
+
+async fn delete_snapshot_all_tags(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+
+    let rows = sqlx::query!(
+        "DELETE FROM sandbox_snapshot WHERE workspace_id = $1 AND name = $2 RETURNING s3_key",
+        &w_id,
+        &name,
+    )
+    .fetch_all(&db)
+    .await?;
+
+    for row in &rows {
+        delete_s3_object(&row.s3_key).await;
+    }
+
+    Ok(format!(
+        "Deleted {} tag(s) for snapshot {}",
+        rows.len(),
+        name
+    ))
+}
+
+async fn rebuild_snapshot(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name, tag)): Path<(String, String, String)>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+    sqlx::query!(
+        "UPDATE sandbox_snapshot SET status = 'pending', build_error = NULL, updated_at = now() \
+         WHERE workspace_id = $1 AND name = $2 AND tag = $3",
+        &w_id,
+        &name,
+        &tag,
+    )
+    .execute(&db)
+    .await?;
+    Ok(format!("Rebuild queued for snapshot {}:{}", name, tag))
+}
+
+// --- Volumes ---
+
+async fn list_volumes(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+) -> JsonResult<Vec<SandboxVolume>> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let rows = sqlx::query_as!(
+        SandboxVolume,
+        "SELECT workspace_id, name, s3_key, size_bytes, created_by, created_at, updated_at, \
+         extra_perms \
+         FROM sandbox_volume WHERE workspace_id = $1 ORDER BY created_at DESC",
+        &w_id,
+    )
+    .fetch_all(&db)
+    .await?;
+    Ok(Json(rows))
+}
+
+#[derive(Deserialize)]
+struct CreateVolume {
+    name: String,
+}
+
+async fn create_volume(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(body): Json<CreateVolume>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let s3_key = format!("sandbox/volumes/{}/{}.tar.gz", w_id, body.name);
+    sqlx::query!(
+        "INSERT INTO sandbox_volume (workspace_id, name, s3_key, created_by) \
+         VALUES ($1, $2, $3, $4) \
+         ON CONFLICT (workspace_id, name) DO NOTHING",
+        &w_id,
+        &body.name,
+        &s3_key,
+        &authed.username,
+    )
+    .execute(&db)
+    .await?;
+    Ok(format!("Volume {} created", body.name))
+}
+
+async fn get_volume(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<SandboxVolume> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let row = sqlx::query_as!(
+        SandboxVolume,
+        "SELECT workspace_id, name, s3_key, size_bytes, created_by, created_at, updated_at, \
+         extra_perms \
+         FROM sandbox_volume WHERE workspace_id = $1 AND name = $2",
+        &w_id,
+        &name,
+    )
+    .fetch_optional(&db)
+    .await?;
+    let row = windmill_common::utils::not_found_if_none(row, "sandbox_volume", &name)?;
+    Ok(Json(row))
+}
+
+async fn delete_volume(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+
+    let row = sqlx::query!(
+        "DELETE FROM sandbox_volume WHERE workspace_id = $1 AND name = $2 RETURNING s3_key",
+        &w_id,
+        &name,
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    if let Some(row) = row {
+        delete_s3_object(&row.s3_key).await;
+    }
+
+    Ok(format!("Deleted volume {}", name))
+}

backend/windmill-common/src/lib.rs

@@ -84,6 +84,7 @@ pub mod queue;
 pub mod result_stream;
 pub mod runnable_settings;
 pub mod s3_helpers;
+pub mod sandbox;
 pub mod schedule;
 pub mod schema;
 pub mod scripts;

backend/windmill-common/src/sandbox.rs

@@ -0,0 +1,250 @@
+use std::collections::HashMap;
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(Debug, Clone, Default)]
+pub struct SandboxConfig {
+    pub snapshot: Option<SnapshotRef>,
+    pub volumes: HashMap<String, String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SnapshotRef {
+    pub name: String,
+    pub tag: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, sqlx::FromRow)]
+pub struct SandboxSnapshot {
+    pub workspace_id: String,
+    pub name: String,
+    pub tag: String,
+    pub s3_key: String,
+    pub content_hash: String,
+    pub docker_image: String,
+    pub setup_script: Option<String>,
+    pub size_bytes: Option<i64>,
+    pub status: String,
+    pub build_error: Option<String>,
+    pub build_job_id: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    pub extra_perms: serde_json::Value,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, sqlx::FromRow)]
+pub struct SandboxVolume {
+    pub workspace_id: String,
+    pub name: String,
+    pub s3_key: String,
+    pub size_bytes: Option<i64>,
+    pub created_by: String,
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    pub extra_perms: serde_json::Value,
+}
+
+/// Parse sandbox annotations from script content.
+///
+/// Supports comment styles: `#` (Python/bash) and `//` (TS/Go/Rust).
+///
+/// ```text
+/// # sandbox: python-env:latest
+/// # volume: data:/workspace/data
+/// # volume: models:/workspace/models
+///
+/// // sandbox: node-env:v2
+/// // volume: cache:/tmp/cache
+/// ```
+///
+/// Tag defaults to `"latest"` if omitted: `# sandbox: python-env`
+pub fn parse_sandbox_config(code: &str) -> SandboxConfig {
+    let mut config = SandboxConfig::default();
+    for line in code.lines() {
+        let trimmed = line.trim();
+        let content = trimmed
+            .strip_prefix('#')
+            .or_else(|| trimmed.strip_prefix("//"))
+            .map(|s| s.trim());
+        let Some(content) = content else {
+            continue;
+        };
+        if let Some(spec) = content.strip_prefix("sandbox:").map(|s| s.trim()) {
+            let (name, tag) = spec
+                .split_once(':')
+                .map(|(n, t)| (n.to_string(), t.to_string()))
+                .unwrap_or_else(|| (spec.to_string(), "latest".to_string()));
+            config.snapshot = Some(SnapshotRef { name, tag });
+        } else if let Some(spec) = content.strip_prefix("volume:").map(|s| s.trim()) {
+            if let Some((name, path)) = spec.split_once(':') {
+                config.volumes.insert(name.to_string(), path.to_string());
+            }
+        }
+    }
+    config
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_python_annotations() {
+        let code = "# sandbox: python-env:latest\n\
+                     # volume: data:/workspace/data\n\
+                     # volume: models:/workspace/models\n\
+                     def main():\n    pass\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "python-env");
+        assert_eq!(snap.tag, "latest");
+        assert_eq!(config.volumes.len(), 2);
+        assert_eq!(config.volumes["data"], "/workspace/data");
+        assert_eq!(config.volumes["models"], "/workspace/models");
+    }
+
+    #[test]
+    fn test_parse_ts_annotations() {
+        let code = "// sandbox: node-env:v2\n\
+                     // volume: cache:/tmp/cache\n\
+                     export async function main() {}\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "node-env");
+        assert_eq!(snap.tag, "v2");
+        assert_eq!(config.volumes["cache"], "/tmp/cache");
+    }
+
+    #[test]
+    fn test_parse_default_tag() {
+        let code = "# sandbox: myenv\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "myenv");
+        assert_eq!(snap.tag, "latest");
+    }
+
+    #[test]
+    fn test_parse_no_annotations() {
+        let code = "def main():\n    pass\n";
+        let config = parse_sandbox_config(code);
+        assert!(config.snapshot.is_none());
+        assert!(config.volumes.is_empty());
+    }
+
+    #[test]
+    fn test_parse_volumes_only() {
+        let code = "# volume: data:/mnt/data\ndef main(): pass\n";
+        let config = parse_sandbox_config(code);
+        assert!(config.snapshot.is_none());
+        assert_eq!(config.volumes.len(), 1);
+        assert_eq!(config.volumes["data"], "/mnt/data");
+    }
+
+    #[test]
+    fn test_parse_empty_code() {
+        let config = parse_sandbox_config("");
+        assert!(config.snapshot.is_none());
+        assert!(config.volumes.is_empty());
+    }
+
+    #[test]
+    fn test_parse_leading_whitespace() {
+        let code = "  # sandbox: indented-env:v1\n  # volume: cache:/mnt/cache\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "indented-env");
+        assert_eq!(snap.tag, "v1");
+        assert_eq!(config.volumes["cache"], "/mnt/cache");
+    }
+
+    #[test]
+    fn test_parse_mixed_comment_styles() {
+        let code = "# sandbox: py-env:v3\n// volume: data:/mnt/data\n# volume: logs:/var/log\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "py-env");
+        assert_eq!(snap.tag, "v3");
+        assert_eq!(config.volumes.len(), 2);
+        assert_eq!(config.volumes["data"], "/mnt/data");
+        assert_eq!(config.volumes["logs"], "/var/log");
+    }
+
+    #[test]
+    fn test_parse_last_sandbox_wins() {
+        let code = "# sandbox: first:v1\n# sandbox: second:v2\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "second");
+        assert_eq!(snap.tag, "v2");
+    }
+
+    #[test]
+    fn test_parse_volume_path_with_nested_colons() {
+        // "volume: name:/some/path" — split_once(':') splits on first colon only
+        let code = "# volume: db:/opt/db:data\n";
+        let config = parse_sandbox_config(code);
+        // name = "db", path = "/opt/db:data" (everything after first colon)
+        assert_eq!(config.volumes["db"], "/opt/db:data");
+    }
+
+    #[test]
+    fn test_parse_annotations_among_code() {
+        let code = "import os\n\
+                     # sandbox: ml-env:gpu\n\
+                     # volume: models:/workspace/models\n\
+                     \n\
+                     def main():\n\
+                         # This is a regular comment, not an annotation\n\
+                         return os.getenv('PATH')\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "ml-env");
+        assert_eq!(snap.tag, "gpu");
+        assert_eq!(config.volumes.len(), 1);
+        assert_eq!(config.volumes["models"], "/workspace/models");
+    }
+
+    #[test]
+    fn test_parse_go_style() {
+        let code = "// sandbox: go-env:1.22\n// volume: cache:/tmp/go-cache\npackage main\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "go-env");
+        assert_eq!(snap.tag, "1.22");
+        assert_eq!(config.volumes["cache"], "/tmp/go-cache");
+    }
+
+    #[test]
+    fn test_parse_bash_style_snapshot_with_multiple_volumes() {
+        let code = "#!/bin/bash\n\
+                     # sandbox: custom-runtime\n\
+                     # volume: input:/mnt/input\n\
+                     # volume: output:/mnt/output\n\
+                     # volume: scratch:/tmp/scratch\n\
+                     echo 'hello'\n";
+        let config = parse_sandbox_config(code);
+        let snap = config.snapshot.unwrap();
+        assert_eq!(snap.name, "custom-runtime");
+        assert_eq!(snap.tag, "latest");
+        assert_eq!(config.volumes.len(), 3);
+        assert_eq!(config.volumes["input"], "/mnt/input");
+        assert_eq!(config.volumes["output"], "/mnt/output");
+        assert_eq!(config.volumes["scratch"], "/tmp/scratch");
+    }
+
+    #[test]
+    fn test_parse_ignores_unrelated_comments() {
+        let code = "# This is a normal comment\n\
+                     # Another comment\n\
+                     // Yet another\n\
+                     # sandboxing is cool (not a directive)\n\
+                     # volume: data:/mnt/data\n";
+        let config = parse_sandbox_config(code);
+        assert!(config.snapshot.is_none());
+        assert_eq!(config.volumes.len(), 1);
+    }
+}

backend/windmill-worker/Cargo.toml

@@ -119,6 +119,8 @@ hex.workspace = true
 tiberius = { workspace = true, optional = true }
 tokio-util = { workspace = true, optional = true }
 tar.workspace = true
+flate2.workspace = true
+tempfile.workspace = true
 object_store = { workspace = true, optional = true}
 convert_case.workspace = true
 yaml-rust.workspace = true

backend/windmill-worker/src/ansible_executor.rs

@@ -1180,9 +1180,7 @@ mount {{
                 )
             })
             .join("\n");
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT
                 .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR)
                 .replace("{JOB_DIR}", job_dir)
@@ -1194,7 +1192,8 @@ mount {{
                     "{ADDITIONAL_PYTHON_PATHS}",
                     additional_python_paths_folders.as_str(),
                 ),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
     } else {
         reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
     }

backend/windmill-worker/src/bash_executor.rs

@@ -187,16 +187,15 @@ exit $exit_status
     // Use nsjail if globally enabled OR if script has #sandbox annotation
     let nsjail = (is_sandboxing_enabled() || annotation.sandbox) && is_regular_job;
     let child = if nsjail {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_BASH_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
                 .replace("{SHARED_MOUNT}", shared_mount)
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut cmd_args = vec![
             "--config",
             "run.config.proto",

backend/windmill-worker/src/bun_executor.rs

@@ -1451,9 +1451,7 @@ try {{
 
     //do not cache local dependencies
     let child = if is_sandboxing_enabled() {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_BUN_CONTENT
                 .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" })
                 .replace("{JOB_DIR}", job_dir)
@@ -1471,7 +1469,8 @@ try {{
                 )
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
 
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         let args = if annotation.nodejs {

backend/windmill-worker/src/csharp_executor.rs

@@ -584,9 +584,7 @@ pub async fn handle_csharp_job(
         get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
 
     let child = if is_sandboxing_enabled() {
-        write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_CSHARP_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CACHE_DIR}", CSHARP_CACHE_DIR)
@@ -595,7 +593,8 @@ pub async fn handle_csharp_job(
                 .replace("{SHARED_MOUNT}", shared_mount)
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL),
-        )?;
+        );
+        write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         nsjail_cmd
             .current_dir(job_dir)

backend/windmill-worker/src/go_executor.rs

@@ -339,16 +339,15 @@ func Run(req Req) (interface{{}}, error){{
         get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
 
     let child = if is_sandboxing_enabled() {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_GO_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
                 .replace("{SHARED_MOUNT}", shared_mount)
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         nsjail_cmd
             .current_dir(job_dir)

backend/windmill-worker/src/java_executor.rs

@@ -598,16 +598,14 @@ async fn run<'a>(
         )
         .await;
 
-        write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_JAVA_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CACHE_DIR}", JAVA_CACHE_DIR)
                 .replace("{SHARED_MOUNT}", &shared_mount)
-                // .replace("{CACHED_TARGET}", &shared_mount)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
-        )?;
+        );
+        write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut cmd = Command::new(NSJAIL_PATH.as_str());
         cmd.env_clear()
             .current_dir(job_dir)

backend/windmill-worker/src/lib.rs

@@ -66,6 +66,7 @@ mod python_versions;
 pub mod result_processor;
 #[cfg(feature = "rust")]
 mod rust_executor;
+mod sandbox_setup;
 mod sanitized_sql_params;
 mod schema;
 pub mod sql_utils;

backend/windmill-worker/src/nu_executor.rs

@@ -245,9 +245,7 @@ async fn run<'a>(
         )
         .await;
 
-        write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_NU_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{NU_PATH}", &NU_PATH)
@@ -255,7 +253,8 @@ async fn run<'a>(
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL),
-        )?;
+        );
+        write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         nsjail_cmd
             .env_clear()

backend/windmill-worker/src/php_executor.rs

@@ -294,14 +294,13 @@ try {{
     let (reserved_variables, _) = tokio::try_join!(reserved_variables_args_out_f, write_wrapper_f)?;
 
     let child = if is_sandboxing_enabled() {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_PHP_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
                 .replace("{SHARED_MOUNT}", shared_mount),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
 
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         let args = vec![

backend/windmill-worker/src/pwsh_executor.rs

@@ -518,15 +518,14 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
 
     let nsjail = is_sandboxing_enabled() && is_regular_job;
     let child = if nsjail {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
                 .replace("{SHARED_MOUNT}", shared_mount)
                 .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let cmd_args = vec![
             "--config",
             "run.config.proto",

backend/windmill-worker/src/python_executor.rs

@@ -800,9 +800,7 @@ mount {{
                 )
             })
             .join("\n");
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR)
@@ -817,7 +815,8 @@ mount {{
                 )
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
     } else {
         reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
     }

backend/windmill-worker/src/ruby_executor.rs

@@ -793,9 +793,7 @@ mount {{
             })
             .join("\n");
 
-        write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_RUBY_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{SHARED_MOUNT}", &shared_mount)
@@ -803,7 +801,8 @@ mount {{
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
-        )?;
+        );
+        write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut cmd = Command::new(NSJAIL_PATH.as_str());
         cmd.env_clear()
             .current_dir(job_dir)

backend/windmill-worker/src/rust_executor.rs

@@ -663,9 +663,7 @@ pub async fn handle_rust_job(
     append_logs(&job.id, &job.workspace_id, logs2, conn).await;
 
     let child = if is_sandboxing_enabled() {
-        let _ = write_file(
-            job_dir,
-            "run.config.proto",
+        let nsjail_config = crate::sandbox_setup::finalize_nsjail_config(
             &NSJAIL_CONFIG_RUN_RUST_CONTENT
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CACHE_DIR}", RUST_CACHE_DIR)
@@ -674,7 +672,8 @@ pub async fn handle_rust_job(
                 .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH)
                 .replace("#{DEV}", DEV_CONF_NSJAIL)
                 .replace("{SHARED_MOUNT}", shared_mount),
-        )?;
+        );
+        let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?;
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         nsjail_cmd
             .current_dir(job_dir)

backend/windmill-worker/src/worker.rs

@@ -4147,7 +4147,7 @@ pub async fn run_language_executor(
         job.id
     );
 
-    let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) {
+    let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) {
         let folder = if job.script_lang == Some(ScriptLang::Go) {
             "/go"
         } else {
@@ -4167,7 +4167,45 @@ mount {{
         "".to_string()
     };
 
-    // println!("handle lang job {:?}",  SystemTime::now());
+    let sandbox_config = windmill_common::sandbox::parse_sandbox_config(&code);
+    let sandbox_setup = if sandbox_config.snapshot.is_some() || !sandbox_config.volumes.is_empty() {
+        let db = conn.as_sql().ok_or_else(|| {
+            Error::ExecutionErr("Sandbox features require SQL connection".to_string())
+        })?;
+        let mut setup = crate::sandbox_setup::SandboxSetupState::default();
+
+        if let Some(ref snap) = sandbox_config.snapshot {
+            let snapshot_path = crate::sandbox_setup::ensure_snapshot_cached(
+                &job.workspace_id, &snap.name, &snap.tag, db,
+            )
+            .await?;
+            setup.overlay =
+                Some(crate::sandbox_setup::mount_overlay(&snapshot_path, job_dir).await?);
+        }
+
+        for (vol_name, mount_path) in &sandbox_config.volumes {
+            let local_dir = format!("{job_dir}/volumes/{vol_name}");
+            std::fs::create_dir_all(&local_dir).map_err(|e| {
+                Error::ExecutionErr(format!("Failed to create volume dir: {e}"))
+            })?;
+            crate::sandbox_setup::download_volume(
+                &job.workspace_id,
+                vol_name,
+                std::path::Path::new(&local_dir),
+                db,
+            )
+            .await?;
+            setup.volume_mounts.insert(
+                vol_name.clone(),
+                (std::path::PathBuf::from(&local_dir), mount_path.clone()),
+            );
+        }
+
+        shared_mount.push_str(&crate::sandbox_setup::build_sandbox_mounts(&setup));
+        Some(setup)
+    } else {
+        None
+    };
 
     let envs = build_envs(envs.as_ref())?;
 
@@ -4623,7 +4661,26 @@ mount {{
         &lang_str,
         job.id
     );
-    // println!("handled job: {:?}",  SystemTime::now());
+
+    if let Some(ref setup) = sandbox_setup {
+        if let Some(db) = conn.as_sql() {
+            for (vol_name, (local_dir, _)) in &setup.volume_mounts {
+                if let Err(e) = crate::sandbox_setup::upload_volume(
+                    &job.workspace_id,
+                    vol_name,
+                    local_dir,
+                    db,
+                )
+                .await
+                {
+                    tracing::error!("Failed to upload volume {vol_name}: {e}");
+                }
+            }
+        }
+        if let Some(ref overlay) = setup.overlay {
+            crate::sandbox_setup::unmount_overlay(overlay).await.ok();
+        }
+    }
 
     result
 }