feat: isolate fork object storage under a per-fork prefix (#11829)

* feat: route the python SDK's S3 client settings through the S3 proxy Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: drop the boto3 config key and expose the proxied bucket to polars Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: proxy S3 settings only for server-signed tokens and keep tls verify Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat: isolate fork object storage under a per-fork prefix with read-through Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: guard fork storage isolation against traversal, shared lakes and orphaning Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: serialize fork purges with same-id creation and guard shared middle forks Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: drop fork storage mappings on every replica and lock fork deletion against child creation Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref to 455681e55eb7cdfcfdb7cd79cbae75b333e07943 This commit updates the EE repository reference after PR #893 was merged in windmill-ee-private. Previous ee-repo-ref: 93f26e3cf816e4eb97b7d4dbbb5fc97fdf9821af New ee-repo-ref: 455681e55eb7cdfcfdb7cd79cbae75b333e07943 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
2026-10-27 00:02:35 +00:00 · 2026-10-16 22:21:38 +02:00
parent e0138a7374
commit 9375805fdf
26 changed files with 1391 additions and 18 deletions
@@ -54,6 +54,23 @@ pub(crate) async fn change_workspace_id(
        require_admin(authed.is_admin, &authed.username)?;
    }

+    // An isolating fork's objects live under a prefix derived from its id: under a new id it
+    // would see none of them, and descendants would lose them from their read-through.
+    let isolates_storage = sqlx::query_scalar!(
+        "SELECT EXISTS (SELECT 1 FROM fork_storage_namespace WHERE workspace_id = $1)",
+        &old_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+    if isolates_storage {
+        return Err(Error::BadRequest(
+            "This workspace isolates its object storage from its parent under a prefix derived \
+             from its id, so its id cannot be changed"
+                .to_string(),
+        ));
+    }
+
    let mut tx = db.begin().await?;

    // The settings copy below carries every data table entry to the new id, which fork cleanup of
@@ -1109,6 +1126,34 @@ pub(crate) async fn delete_workspace(
            tracing::warn!("deleting workspace {w_id}: preparing ducklake cleanup: {e:#}");
            vec![]
        });
+    // A child fork reads through to this fork's prefix, or to the one this fork shares, and a
+    // child sharing the storage writes into it: deleting this link would destroy the child's
+    // data, or leave its chain without the isolating fork, writing the root's own keys. Under the
+    // lock forking this workspace takes, so no child can commit between the check and the delete.
+    windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?;
+    let chain = windmill_common::workspaces::workspace_with_fork_ancestors(&db, &w_id).await?;
+    let isolating_with_children = sqlx::query_scalar!(
+        "SELECT EXISTS (SELECT 1 FROM fork_storage_namespace WHERE workspace_id = ANY($1))
+            AND EXISTS (SELECT 1 FROM workspace WHERE parent_workspace_id = $2)",
+        &chain,
+        &w_id
+    )
+    .fetch_one(&mut *tx)
+    .await?
+    .unwrap_or(false);
+    if isolating_with_children {
+        return Err(Error::BadRequest(format!(
+            "the forks of {w_id} reach its fork chain's isolated object storage through it: \
+             delete them first"
+        )));
+    }
+    // Nothing can reach a deleted fork's prefix once it is gone, so it is always purged.
+    let fork_storage_purges = prepare_fork_storage_purges(&db, &w_id, None)
+        .await
+        .unwrap_or_else(|e| {
+            tracing::warn!("deleting workspace {w_id}: preparing storage purge: {e:#}");
+            vec![]
+        });

    sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id)
        .execute(&mut *tx)
@@ -1379,6 +1424,14 @@ pub(crate) async fn delete_workspace(
            e.msg
        );
    }
+    match purge_fork_storages_unless_live(&db, &w_id, fork_storage_purges).await {
+        Ok(errors) => {
+            for e in errors {
+                tracing::warn!("deleted workspace {w_id}: storage purge: {e}");
+            }
+        }
+        Err(e) => tracing::warn!("deleted workspace {w_id}: storage purge: {e:#}"),
+    }

    if let Some(parent) = dev_lock_parent {
        windmill_common::workspaces::invalidate_protection_rules_cache(&parent);
@@ -2308,6 +2361,109 @@ async fn delete_fork_ducklake_data(
    ))
 }

+/// A storage a fork isolated, resolved while its `$res:` still exists.
+pub(crate) struct PreparedForkStoragePurge {
+    storage: String,
+    store: std::result::Result<(windmill_types::s3::LargeFileStorage, serde_json::Value), String>,
+}
+
+/// Read-only: one entry per `fork_storage_namespace` row of `w_id`. `res_fallback_w_id` as in
+/// [`prepare_fork_ducklake_cleanups`].
+pub(crate) async fn prepare_fork_storage_purges(
+    db: &DB,
+    w_id: &str,
+    res_fallback_w_id: Option<&str>,
+) -> Result<Vec<PreparedForkStoragePurge>> {
+    let rows = sqlx::query!(
+        "SELECT storage, storage_ref FROM fork_storage_namespace WHERE workspace_id = $1",
+        w_id
+    )
+    .fetch_all(db)
+    .await?;
+    let mut prepared = Vec::with_capacity(rows.len());
+    for row in rows {
+        let storage = Some(row.storage.as_str()).filter(|s| !s.is_empty());
+        let store = resolve_fork_storage(db, w_id, res_fallback_w_id, storage, &row.storage_ref)
+            .await
+            .map_err(|e| e.to_string());
+        prepared.push(PreparedForkStoragePurge { storage: row.storage, store });
+    }
+    Ok(prepared)
+}
+
+/// [`purge_fork_storages`] under the lock same-id fork creation takes, unless a workspace with
+/// that id exists again: the rows are then the new fork's, and its creation already purged the
+/// ones it found.
+pub(crate) async fn purge_fork_storages_unless_live(
+    db: &DB,
+    w_id: &str,
+    prepared: Vec<PreparedForkStoragePurge>,
+) -> Result<Vec<String>> {
+    let mut lock = db.begin().await?;
+    sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork:' || $1))")
+        .bind(w_id)
+        .execute(&mut *lock)
+        .await?;
+    let live = sqlx::query_scalar!(
+        "SELECT EXISTS (SELECT 1 FROM workspace WHERE id = $1)",
+        w_id
+    )
+    .fetch_one(&mut *lock)
+    .await?
+    .unwrap_or(false);
+    let errors = if live {
+        vec![]
+    } else {
+        purge_fork_storages(db, w_id, prepared).await
+    };
+    lock.commit().await?;
+    Ok(errors)
+}
+
+/// Delete everything under the fork's `__wm_forks/<segment>/` in each prepared storage, then its
+/// registry row. Returns one message per storage that could not be purged; its row survives.
+async fn purge_fork_storages(
+    db: &DB,
+    w_id: &str,
+    prepared: Vec<PreparedForkStoragePurge>,
+) -> Vec<String> {
+    let area = format!(
+        "{}/{}",
+        windmill_common::workspaces::FORK_DUCKLAKE_DATA_DIR,
+        windmill_common::workspaces::fork_data_dir_segment(w_id)
+    );
+    let mut errors = vec![];
+    for p in prepared {
+        let name = if p.storage.is_empty() {
+            "default storage"
+        } else {
+            p.storage.as_str()
+        };
+        let purged = match p.store {
+            Ok((lfs, value)) => delete_fork_ducklake_data(lfs, value, &area)
+                .await
+                .map_err(|e| e.to_string()),
+            Err(e) => Err(e),
+        };
+        if let Err(e) = purged {
+            errors.push(format!("{name}: {e}"));
+            continue;
+        }
+        if let Err(e) = sqlx::query!(
+            "DELETE FROM fork_storage_namespace WHERE workspace_id = $1 AND storage = $2",
+            w_id,
+            p.storage
+        )
+        .execute(db)
+        .await
+        {
+            errors.push(format!("{name}: removing registry row: {e}"));
+        }
+    }
+    windmill_common::fork_storage::invalidate_fork_storage_keys_cache(w_id);
+    errors
+}
+
 /// Destroying an ATTACHED dev workspace (or its data environments) must be a prod-admin
 /// action, not just the dev's own owner (dev ownership can diverge from prod's) — mirrors
 /// detach_dev_workspace. Shared by `delete_workspace` and `drop_forked_ducklake_namespaces`