From 09a80040ca268a4379d5302e3401435ba93247e0 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Sat, 20 Jun 2026 12:22:15 +0200 Subject: [PATCH 001/273] fix(frontend): clear branch step state when switching outer loop iterations (#9650) Co-authored-by: Claude Opus 4.8 (1M context) --- frontend/src/lib/components/FlowStatusViewerInner.svelte | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index d0bf66d61b..ed1c536ff5 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -413,7 +413,12 @@ function updateRecursiveRefresh(jobId: string) { if (jobId) { updateRecursiveRefreshFn?.(jobId, async (clear, root) => { - if (globalModuleStates.length > 0 || isSubflow) { + // During a clear pass we must descend into children even when this + // subtree is currently deselected (globalModuleStates empty): clearing + // the parent loop's selection deselects its iteration viewers before the + // recursion reaches them, and their stale branch-step states would leak + // into the newly selected iteration otherwise. + if (clear || globalModuleStates.length > 0 || isSubflow) { await refresh(clear, root) } }) From 3ebf24359d66048d6361ce65cd879cdc04b737ed Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Jun 2026 15:42:03 +0200 Subject: [PATCH 002/273] feat: ducklake materialization for data pipelines (#9689) --- ...5039aa631bb5b0993ad09bc9a1381f6715f19.json | 47 + ...df037178ba246092dbc1d2b3816e7a9269dc3.json | 111 +++ backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- ...170118_add_materialized_partition.down.sql | 3 + ...19170118_add_materialized_partition.up.sql | 29 + .../windmill-parser/src/asset_parser.rs | 134 ++- backend/parsers/windmill-parser/src/lib.rs | 1 + .../windmill-parser/src/sql_materialize.rs | 817 ++++++++++++++++++ .../tests/fixtures/pipeline_annotations.json | 67 ++ .../tests/pipeline_annotations_parity.rs | 36 + backend/windmill-api-assets/src/lib.rs | 57 ++ backend/windmill-api-scripts/Cargo.toml | 1 + backend/windmill-api-scripts/src/scripts.rs | 86 ++ backend/windmill-api/openapi.yaml | 7 + backend/windmill-common/src/lib.rs | 1 + .../windmill-common/src/materialization.rs | 135 +++ backend/windmill-worker/src/agent_workers.rs | 18 + .../windmill-worker/src/duckdb_executor.rs | 227 +++++ backend/windmill-worker/src/worker.rs | 45 +- cli/src/guidance/skills.gen.ts | 90 ++ docs/ducklake-materialization.md | 250 ++++++ docs/pipelines-vs-dbt.md | 5 + .../src/lib/components/DisplayResult.svelte | 71 +- .../src/lib/components/ScriptEditor.svelte | 52 ++ .../AssetGraph/AssetGraphDetailsPane.svelte | 9 + .../AssetGraph/BackfillRangeDialog.svelte | 79 ++ .../AssetGraph/DucklakeResultPreview.svelte | 136 +++ .../AssetGraph/PartitionStatusGrid.svelte | 134 +++ .../AssetGraph/PipelineInsertMenu.svelte | 4 +- .../parsePipelineAnnotations.parity.test.ts | 30 +- .../AssetGraph/parsePipelineAnnotations.ts | 61 ++ .../assets/AssetGraph/pipelineTemplates.ts | 83 +- frontend/src/lib/components/dbOps.ts | 15 +- python-client/wmill/wmill/client.py | 179 ++++ system_prompts/auto-generated/prompts.ts | 44 + system_prompts/auto-generated/script.md | 44 + system_prompts/auto-generated/sdks/python.md | 21 + .../auto-generated/sdks/typescript.md | 23 + .../skills/write-script-bun/SKILL.md | 23 + .../skills/write-script-bunnative/SKILL.md | 23 + .../skills/write-script-deno/SKILL.md | 23 + .../skills/write-script-python3/SKILL.md | 21 + typescript-client/build.jsr.sh | 2 +- typescript-client/build.sh | 6 +- typescript-client/client.d.ts | 4 + typescript-client/client.ts | 6 +- typescript-client/sqlUtils.d.ts | 15 + typescript-client/sqlUtils.ts | 182 +++- 49 files changed, 3407 insertions(+), 53 deletions(-) create mode 100644 backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json create mode 100644 backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json create mode 100644 backend/migrations/20260619170118_add_materialized_partition.down.sql create mode 100644 backend/migrations/20260619170118_add_materialized_partition.up.sql create mode 100644 backend/parsers/windmill-parser/src/sql_materialize.rs create mode 100644 backend/windmill-common/src/materialization.rs create mode 100644 docs/ducklake-materialization.md create mode 100644 frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte diff --git a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json b/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json new file mode 100644 index 0000000000..17d372249b --- /dev/null +++ b/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n snapshot_id = EXCLUDED.snapshot_id,\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + }, + "Varchar", + "Text", + { + "Custom": { + "name": "materialization_status", + "kind": { + "Enum": [ + "running", + "materialized", + "failed" + ] + } + } + }, + "Int8", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19" +} diff --git a/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json new file mode 100644 index 0000000000..7981dbc983 --- /dev/null +++ b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json @@ -0,0 +1,111 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT asset_kind AS \"asset_kind: AssetKind\", asset_path, partition,\n status AS \"status: MaterializationStatus\", snapshot_id,\n row_count, job_id, materialized_at, error\n FROM materialized_partition\n WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3\n ORDER BY partition DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "partition", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "status: MaterializationStatus", + "type_info": { + "Custom": { + "name": "materialization_status", + "kind": { + "Enum": [ + "running", + "materialized", + "failed" + ] + } + } + } + }, + { + "ordinal": 4, + "name": "snapshot_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "row_count", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "materialized_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + true + ] + }, + "hash": "c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 367c17e912..79c61dd687 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14354,6 +14354,7 @@ dependencies = [ "windmill-parser", "windmill-parser-py", "windmill-parser-py-asset", + "windmill-parser-sql", "windmill-parser-sql-asset", "windmill-parser-ts", "windmill-parser-ts-asset", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 151e85295b..8bf4c6e5dd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ba677ea142011462ad4dfe77e8375a6dd274cdef +23b5f55a943dd4d4f72a5406398b68f22782a8b8 \ No newline at end of file diff --git a/backend/migrations/20260619170118_add_materialized_partition.down.sql b/backend/migrations/20260619170118_add_materialized_partition.down.sql new file mode 100644 index 0000000000..4932338956 --- /dev/null +++ b/backend/migrations/20260619170118_add_materialized_partition.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_materialized_partition_asset_status; +DROP TABLE IF EXISTS materialized_partition; +DROP TYPE IF EXISTS MATERIALIZATION_STATUS; diff --git a/backend/migrations/20260619170118_add_materialized_partition.up.sql b/backend/migrations/20260619170118_add_materialized_partition.up.sql new file mode 100644 index 0000000000..da5f806e4f --- /dev/null +++ b/backend/migrations/20260619170118_add_materialized_partition.up.sql @@ -0,0 +1,29 @@ +-- Per-partition materialization state for managed `// materialize` assets. +-- One row per (asset, partition): the latest materialization of that slice. +-- Drives: the partition-status grid (CE observability), run-stale/gap +-- detection, and the EE backfill worklist (missing/failed partitions). The +-- `partition` column uses '' as the sentinel for an unpartitioned (whole-table) +-- materialization, since partition is part of the primary key and cannot be +-- NULL. +CREATE TYPE MATERIALIZATION_STATUS AS ENUM ('running', 'materialized', 'failed'); + +CREATE TABLE IF NOT EXISTS materialized_partition ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + asset_kind ASSET_KIND NOT NULL, + asset_path VARCHAR(255) NOT NULL, + partition TEXT NOT NULL DEFAULT '', + status MATERIALIZATION_STATUS NOT NULL, + -- DuckLake snapshot id produced by the write; NULL while running / on + -- failure. The pin that makes downstream reads reproducible. + snapshot_id BIGINT, + row_count BIGINT, + job_id UUID, + materialized_at TIMESTAMPTZ NOT NULL DEFAULT now(), + error TEXT, + PRIMARY KEY (workspace_id, asset_kind, asset_path, partition) +); + +-- Backfill enumeration / grid "show only gaps": filter an asset's partitions +-- by status without scanning the whole table. +CREATE INDEX IF NOT EXISTS idx_materialized_partition_asset_status + ON materialized_partition (workspace_id, asset_kind, asset_path, status); diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index ce0be81f00..61e30a343f 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -107,6 +107,11 @@ pub struct ParseAssetsOutput { // The delay is a raw duration string parsed at deploy (parser-light). #[serde(skip_serializing_if = "Option::is_none", default)] pub retry: Option, + // `// materialize [manual] [append] [key=]` — + // managed-materialization target + its strategy. At most one per script. + // Drives the worker's write-strategy + snapshot capture. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub materialize: Option, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -209,6 +214,27 @@ pub struct RetrySpec { pub delay: Option, } +// `// materialize [manual] [append] [key=]` — declares that this +// script produces a *managed* materialization of `` (a `ducklake://` +// table). By default the runtime generates the write DDL around the script's +// single trailing `SELECT` and owns idempotency, partition-state and snapshot +// capture. `manual` is the escape hatch: the script writes its own DDL and the +// runtime only records state (track-only). The reconciliation strategy options +// (`append`, `key=`) apply to managed mode: none → DELETE-by-partition + +// INSERT (replace); `key=` → MERGE (dedup within slice); `append` → +// INSERT-only. `append` wins if both are given (deploy-time warning). +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct MaterializeSpec { + pub target_kind: AssetKind, + pub target_path: String, + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub manual: bool, + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub append: bool, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub unique_key: Option, +} + // `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger // firing runs the script (current behaviour). `All` = AND: the script // runs only once every partition-bearing input has materialized at the @@ -239,6 +265,7 @@ pub struct PipelineAnnotations { pub debounce_default: Option, pub tag: Option, pub retry: Option, + pub materialize: Option, } impl ParseAssetsOutput { @@ -262,6 +289,7 @@ impl ParseAssetsOutput { debounce_default: pipeline.debounce_default, tag: pipeline.tag, retry: pipeline.retry, + materialize: pipeline.materialize, } } } @@ -571,6 +599,15 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { continue; } + if let Some(after_kw) = consume_keyword(rest, "materialize") { + if out.materialize.is_none() { + if let Some(spec) = parse_materialize_spec(after_kw.trim()) { + out.materialize = Some(spec); + } + } + continue; + } + if let Some(after_kw) = consume_keyword(rest, "on") { let spec_text = after_kw.trim(); if spec_text.is_empty() { @@ -618,6 +655,35 @@ fn parse_retry_spec(s: &str) -> Option { Some(RetrySpec { count, delay }) } +// Parse a `// materialize [manual] [append] [key=]` right-hand +// side. An optional leading `manual` token (whitespace-delimited) opts out of +// managed mode (track-only). The next whitespace token is the target asset URI +// (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the +// remainder are strategy options — bare `append` and `key=` (merge key), +// which apply to managed mode only. A missing/empty target yields `None` (the +// annotation is dropped, fail-safe). +fn parse_materialize_spec(s: &str) -> Option { + let (manual, rest) = match s.strip_prefix("manual") { + Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => { + (true, after.trim_start()) + } + _ => (false, s), + }; + let mut it = rest.trim().splitn(2, char::is_whitespace); + let asset_tok = it.next()?; + let opts_str = it.next().unwrap_or(""); + let (target_kind, path) = parse_asset_syntax(asset_tok.trim(), true)?; + if path.is_empty() { + return None; + } + let append = opts_str.split_whitespace().any(|t| t == "append"); + let unique_key = parse_kv_opts(opts_str) + .get("key") + .filter(|k| !k.is_empty()) + .cloned(); + Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key }) +} + // Parse a `// partitioned [opts]` right-hand side. Recognized kinds: // `daily`, `hourly`, `weekly`, `monthly` (with optional tz/format/start), // and `dynamic key=""` (plus optional format). @@ -1044,6 +1110,67 @@ mod pipeline_annotation_tests { assert!(out.retry.is_none()); } + #[test] + fn materialize_managed_default() { + let out = parse_pipeline_annotations("// materialize ducklake://analytics/orders_daily"); + let m = out.materialize.expect("materialize"); + assert_eq!(m.target_kind, AssetKind::Ducklake); + assert_eq!(m.target_path, "analytics/orders_daily"); + // managed by default; replace strategy (no append / key) + assert!(!m.manual); + assert!(!m.append); + assert_eq!(m.unique_key, None); + } + + #[test] + fn materialize_manual_escape_hatch() { + let out = + parse_pipeline_annotations("// materialize manual ducklake://analytics/orders_daily"); + let m = out.materialize.expect("materialize"); + assert!(m.manual); + assert_eq!(m.target_path, "analytics/orders_daily"); + } + + #[test] + fn materialize_merge_and_append_options() { + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders_daily key=order_id"); + let m = out.materialize.expect("materialize"); + assert_eq!(m.unique_key.as_deref(), Some("order_id")); + assert!(!m.append); + + let out = parse_pipeline_annotations("// materialize ducklake://a/events append"); + let m = out.materialize.expect("materialize"); + assert!(m.append); + assert_eq!(m.unique_key, None); + } + + #[test] + fn materialize_default_syntax_shorthand() { + let out = parse_pipeline_annotations("// materialize ducklake"); + let m = out.materialize.expect("materialize"); + assert_eq!(m.target_kind, AssetKind::Ducklake); + assert_eq!(m.target_path, "main"); + assert!(!m.manual); + } + + #[test] + fn materialize_manual_only_is_dropped() { + // `manual` with no target is not a valid materialization. + let out = parse_pipeline_annotations("// materialize manual"); + assert!(out.materialize.is_none()); + } + + #[test] + fn materialize_first_wins() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/x\n# materialize manual ducklake://b/y", + ); + let m = out.materialize.expect("materialize"); + assert_eq!(m.target_path, "a/x"); + assert!(!m.manual); + } + #[test] fn combined() { let code = concat!( @@ -1053,7 +1180,8 @@ mod pipeline_annotation_tests { "// partitioned daily tz=\"UTC\"\n", "// freshness 2h\n", "// tag heavy\n", - "// retry 3 5s\n" + "// retry 3 5s\n", + "// materialize ducklake://analytics/orders_daily key=order_id\n" ); let out = parse_pipeline_annotations(code); assert!(out.in_pipeline); @@ -1064,6 +1192,10 @@ mod pipeline_annotation_tests { let r = out.retry.expect("retry"); assert_eq!(r.count, 3); assert_eq!(r.delay.as_deref(), Some("5s")); + let m = out.materialize.expect("materialize"); + assert!(!m.manual); + assert_eq!(m.target_path, "analytics/orders_daily"); + assert_eq!(m.unique_key.as_deref(), Some("order_id")); } #[test] diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 5a7e90bf7e..19bc5602cd 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -13,6 +13,7 @@ use serde::Serialize; use serde_json::Value; pub mod asset_parser; +pub mod sql_materialize; /// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) #[derive(Clone, Copy, Debug)] diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs new file mode 100644 index 0000000000..c1b0cc97aa --- /dev/null +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -0,0 +1,817 @@ +//! Eligibility classifier + materialization SQL codegen for managed `// materialize`. +//! +//! Managed `// materialize` (the default) promises the script is "setup +//! statements, then one trailing SELECT" — Windmill generates the write DDL +//! around that SELECT (the `// materialize manual` escape hatch opts out and +//! writes its own DDL). This module is the single source of truth for *which +//! block is that SELECT* and *what DDL gets generated*, so save-time validation +//! (deploy path) and run-time codegen (DuckDB executor) can never disagree. +//! +//! Everything here is pure and string-level: no SQL is executed, no type +//! inference is done. The classifier is leading-keyword based and deliberately +//! conservative — anything it can't positively recognize as a read-only output +//! or a known-safe setup statement is rejected, so a script is only accepted +//! for managed mode when its shape is unambiguous. + +/// One top-level statement's role in a wrap-mode script. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlockClass { + /// Read-only relation the wrap writes from: `SELECT` / `WITH …SELECT` / + /// `FROM` (DuckDB from-first) / `VALUES` / `TABLE x` / `(UN)PIVOT`. + Output, + /// Known-safe preamble: `ATTACH` / `INSTALL` / `LOAD` / `SET` / `PRAGMA` / + /// `USE` / `CREATE TEMP …`. Runs verbatim before the generated write. + Setup, + /// Anything that writes or whose effect we can't vouch for: non-temp + /// `CREATE` / `INSERT` / `UPDATE` / `DELETE` / `MERGE` / `DROP` / `COPY` / + /// `ALTER` / `TRUNCATE`, or an unrecognized leading keyword. Disqualifies + /// managed mode (the user should use `// materialize manual`). + Disallowed, +} + +/// A script accepted for wrapping: zero+ setup blocks then one terminal SELECT. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WrapPlan { + /// Setup statements in source order, verbatim, **without** trailing `;`. + pub setup: Vec, + /// The single terminal output statement, verbatim, **without** trailing `;`. + pub output: String, +} + +/// Why a script is not eligible for managed `// materialize`. Carries enough to +/// render the targeted save-time messages. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WrapError { + /// No statements at all (empty / comments only). + Empty, + /// No terminal SELECT — nothing to wrap. + NoOutput, + /// More than one top-level SELECT. `count` is how many were found. + MultipleOutputs { count: usize }, + /// A SELECT exists but isn't the last statement (something runs after it). + OutputNotLast, + /// A write/unknown statement appears among the setup blocks. `snippet` is a + /// short prefix of the offending statement for the error message. + DisallowedBlock { snippet: String }, +} + +impl WrapError { + /// Human-facing, actionable message (matches the spec's rejection text). + pub fn message(&self) -> String { + let base = + "managed `// materialize` requires the script to be setup statements then a single trailing SELECT"; + let manual = "use `// materialize manual` to write the DDL yourself"; + match self { + WrapError::Empty => format!("{base}: the script is empty."), + WrapError::NoOutput => format!("{base}: found no SELECT — {manual}."), + WrapError::MultipleOutputs { count } => format!( + "{base}: found {count} SELECT statements; combine them with a CTE, or {manual}." + ), + WrapError::OutputNotLast => format!( + "{base}: found statements after the SELECT — move them above it, or {manual}." + ), + WrapError::DisallowedBlock { snippet } => { + format!("{base}: `{snippet}` writes or is unrecognized — {manual}.") + } + } + } +} + +/// Split SQL into top-level, `;`-separated statements, skipping line comments +/// (`-- …`), block comments (`/* … */`), single-quoted strings (`'…'` with +/// `''` escape) and double-quoted identifiers (`"…"`). Semicolons inside any of +/// those are not separators. Returns each statement trimmed, comments stripped, +/// empties dropped. Self-contained so the parser crate stays dependency-free; +/// it must stay behaviourally aligned with the executor's block splitter (both +/// route wrap through `classify_wrap`, so the split they see is this one). +pub fn split_statements(sql: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let bytes = sql.as_bytes(); + let mut i = 0; + let n = bytes.len(); + while i < n { + let c = bytes[i] as char; + // line comment — `--` (SQL) or `//`. The `//` form is not SQL, but it + // is how Windmill pipeline annotations (`// materialize`, `// pipeline`, + // …) are written, and they sit above the SQL in the same script; strip + // them so they don't pollute the first statement block's classification + // or the generated setup SQL. + if (c == '-' && i + 1 < n && bytes[i + 1] == b'-') + || (c == '/' && i + 1 < n && bytes[i + 1] == b'/') + { + while i < n && bytes[i] != b'\n' { + i += 1; + } + continue; + } + // block comment + if c == '/' && i + 1 < n && bytes[i + 1] == b'*' { + i += 2; + while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { + i += 1; + } + i += 2; + continue; + } + // single-quoted string + if c == '\'' { + cur.push(c); + i += 1; + while i < n { + cur.push(bytes[i] as char); + if bytes[i] == b'\'' { + // doubled '' is an escaped quote, stay in string + if i + 1 < n && bytes[i + 1] == b'\'' { + cur.push('\''); + i += 2; + continue; + } + i += 1; + break; + } + i += 1; + } + continue; + } + // double-quoted identifier + if c == '"' { + cur.push(c); + i += 1; + while i < n { + cur.push(bytes[i] as char); + if bytes[i] == b'"' { + i += 1; + break; + } + i += 1; + } + continue; + } + if c == ';' { + let t = cur.trim(); + if !t.is_empty() { + out.push(t.to_string()); + } + cur.clear(); + i += 1; + continue; + } + cur.push(c); + i += 1; + } + let t = cur.trim(); + if !t.is_empty() { + out.push(t.to_string()); + } + out +} + +/// Lowercased top-level keyword tokens of a single statement (parens collapsed +/// away: tokens *inside* balanced `(...)` are skipped, so a CTE body's verbs +/// don't leak up). Strings/identifiers are already gone from the split, but we +/// re-guard quotes defensively. Used to disambiguate `WITH …` and `CREATE …`. +fn top_level_keywords(stmt: &str) -> Vec { + let mut toks = Vec::new(); + let mut cur = String::new(); + let mut depth: i32 = 0; + let bytes = stmt.as_bytes(); + let mut i = 0; + let n = bytes.len(); + let flush = |cur: &mut String, toks: &mut Vec| { + if !cur.is_empty() { + toks.push(cur.to_lowercase()); + cur.clear(); + } + }; + while i < n { + let c = bytes[i] as char; + if c == '\'' || c == '"' { + let q = bytes[i]; + i += 1; + while i < n && bytes[i] != q { + i += 1; + } + i += 1; + continue; + } + if c == '(' { + flush(&mut cur, &mut toks); + depth += 1; + i += 1; + continue; + } + if c == ')' { + if depth > 0 { + depth -= 1; + } + i += 1; + continue; + } + if depth > 0 { + i += 1; + continue; + } + if c.is_alphanumeric() || c == '_' { + cur.push(c); + } else { + flush(&mut cur, &mut toks); + } + i += 1; + } + flush(&mut cur, &mut toks); + toks +} + +const OUTPUT_KW: &[&str] = &["select", "from", "values", "table", "pivot", "unpivot"]; +const SETUP_KW: &[&str] = &["attach", "install", "load", "set", "pragma", "use"]; +const WRITE_VERBS: &[&str] = &["insert", "update", "delete", "merge"]; + +/// Classify a single statement by its leading keyword (with `WITH`/`CREATE` +/// disambiguation). See [`BlockClass`]. +pub fn classify_block(stmt: &str) -> BlockClass { + let kws = top_level_keywords(stmt); + let Some(first) = kws.first().map(String::as_str) else { + return BlockClass::Disallowed; + }; + + // CREATE TEMP … is setup (staging); any other CREATE is a write. + if first == "create" { + let temp = kws + .iter() + .skip(1) + .take(3) + .any(|k| k == "temp" || k == "temporary"); + return if temp { + BlockClass::Setup + } else { + BlockClass::Disallowed + }; + } + + // WITH … : the main statement's verb decides. CTE bodies are parenthesized, + // so their verbs are not in `kws`; the first top-level write verb or SELECT + // after the CTE list is the real one. + if first == "with" { + for k in kws.iter().skip(1) { + if k == "select" { + return BlockClass::Output; + } + if WRITE_VERBS.contains(&k.as_str()) { + return BlockClass::Disallowed; + } + } + // `WITH x AS (...) SELECT` where SELECT got collapsed is impossible + // (SELECT here is top-level), so a WITH with no top-level verb is a + // malformed/unknown statement — reject conservatively. + return BlockClass::Disallowed; + } + + if OUTPUT_KW.contains(&first) { + return BlockClass::Output; + } + if SETUP_KW.contains(&first) { + return BlockClass::Setup; + } + BlockClass::Disallowed +} + +/// Validate a script for managed `// materialize` and, on success, return the +/// setup/output split. Enforces the four conditions from the spec: +/// 1. exactly one Output block, 2. it is last, 3. all preceding blocks are +/// Setup, 4. nothing after it. +pub fn classify_wrap(sql: &str) -> Result { + let stmts = split_statements(sql); + if stmts.is_empty() { + return Err(WrapError::Empty); + } + let classes: Vec = stmts.iter().map(|s| classify_block(s)).collect(); + + let output_idxs: Vec = classes + .iter() + .enumerate() + .filter(|(_, c)| **c == BlockClass::Output) + .map(|(i, _)| i) + .collect(); + + match output_idxs.len() { + 0 => return Err(WrapError::NoOutput), + 1 => {} + count => return Err(WrapError::MultipleOutputs { count }), + } + let out_idx = output_idxs[0]; + if out_idx != stmts.len() - 1 { + return Err(WrapError::OutputNotLast); + } + // Everything before the output must be Setup (no Disallowed preamble). + for (i, c) in classes.iter().enumerate().take(out_idx) { + if *c != BlockClass::Setup { + return Err(WrapError::DisallowedBlock { snippet: snippet(&stmts[i]) }); + } + } + Ok(WrapPlan { setup: stmts[..out_idx].to_vec(), output: stmts[out_idx].clone() }) +} + +fn snippet(stmt: &str) -> String { + let one_line: String = stmt.split_whitespace().collect::>().join(" "); + if one_line.chars().count() > 40 { + let truncated: String = one_line.chars().take(40).collect(); + format!("{truncated}…") + } else { + one_line + } +} + +// --------------------------------------------------------------------------- +// Codegen +// --------------------------------------------------------------------------- + +/// How a (partition of a) materialized table is reconciled on each run. +/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else +/// `unique_key` → `Merge`, else `Replace`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MaterializeStrategy { + /// DELETE the current partition, then INSERT — partition becomes exactly + /// what the SELECT returned. Full-refresh of the slice. + Replace, + /// Upsert within the slice on `unique_key` (delete-by-key + insert); rows + /// absent from the SELECT are left in place. + Merge { unique_key: String }, + /// INSERT only — immutable event-log semantics. + Append, +} + +/// Inputs to materialization codegen, all resolved at run time by the worker. +/// Pure: produces SQL text; executes nothing. +#[derive(Debug, Clone)] +pub struct MaterializeCodegen<'a> { + /// Fully-qualified target, e.g. `_wm_target.orders_daily`. Always qualified + /// so a user `USE …;` in setup can't redirect the write. + pub target_qualified: &'a str, + /// The user's output SELECT (verbatim, no trailing `;`) — embedded as a + /// subquery so its own shape is irrelevant to the generated wrapper. + pub select_sql: &'a str, + /// Physical partition column added to the managed table. + pub partition_col: &'a str, + /// SQL expression for the current partition value — a literal like + /// `'2026-06-19'` or a bind placeholder. The caller is responsible for + /// safe quoting/binding. + pub partition_value_sql: &'a str, + /// Whether `// partitioned` applies. When false the table is unpartitioned + /// and the partition column / `SET PARTITIONED BY` are omitted. + pub partitioned: bool, + pub strategy: MaterializeStrategy, +} + +impl<'a> MaterializeCodegen<'a> { + /// The ordered statements that perform the materialization, to be run after + /// the setup blocks and inside the caller's execution. The first-run + /// bootstrap is idempotent (`IF NOT EXISTS`), so this is safe to run every + /// time. The DELETE/INSERT body is wrapped in one transaction so a partial + /// failure leaves the prior snapshot intact. Every strategy reduces to + /// DELETE+INSERT (no `MERGE INTO`) — see the `Merge` arm for why. + pub fn statements(&self) -> Vec { + let t = self.target_qualified; + let sel = self.select_sql; + let pcol = self.partition_col; + let pval = self.partition_value_sql; + let mut out = Vec::new(); + + // Whole-table replace: rebuild the table to match the SELECT's *current* + // schema each run with one atomic `CREATE OR REPLACE` (which DuckLake + // still snapshots). This is the only path that survives a changed SELECT + // or a pre-existing table with a different schema — the persist-and- + // mutate paths below fix the schema at first create. + if !self.partitioned && matches!(self.strategy, MaterializeStrategy::Replace) { + out.push(format!( + "CREATE OR REPLACE TABLE {t} AS SELECT * FROM ({sel});" + )); + return out; + } + + // Persist-and-mutate (partitioned, or merge/append): bootstrap the table + // if absent, then write into it. The schema is fixed at first create — + // a later SELECT-schema change needs a manual rebuild (schema evolution + // is a follow-up). + if self.partitioned { + out.push(format!( + "CREATE TABLE IF NOT EXISTS {t} AS \ + SELECT *, CAST(NULL AS VARCHAR) AS {pcol} FROM ({sel}) WHERE false;" + )); + out.push(format!("ALTER TABLE {t} SET PARTITIONED BY ({pcol});")); + } else { + out.push(format!( + "CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;" + )); + } + + out.push("BEGIN TRANSACTION;".to_string()); + // The rows to write, with the partition column appended when partitioned. + let source = if self.partitioned { + format!("SELECT *, {pval} AS {pcol} FROM ({sel})") + } else { + format!("SELECT * FROM ({sel})") + }; + match &self.strategy { + MaterializeStrategy::Replace => { + // Only reached when partitioned (whole-table replace returned above). + out.push(format!("DELETE FROM {t} WHERE {pcol} = {pval};")); + out.push(format!("INSERT INTO {t} {source};")); + } + MaterializeStrategy::Append => { + out.push(format!("INSERT INTO {t} {source};")); + } + MaterializeStrategy::Merge { unique_key } => { + // Upsert within the slice via delete-by-key + insert (dbt's + // `delete+insert`): rows whose key is in the incoming SELECT are + // replaced, others are left in place. This deliberately avoids + // `MERGE INTO` — DuckLake's MERGE fails writing the first rows of + // a fresh partition (HTTP 404 on the new parquet), and a failed + // write leaves the table needing a DROP. DELETE+INSERT is the + // same write shape as `replace`, which is reliable. The DELETE is + // scoped to the current partition when partitioned so it stays + // slice-local (a key present in another partition is untouched). + let scope = if self.partitioned { + format!("{pcol} = {pval} AND ") + } else { + String::new() + }; + out.push(format!( + "DELETE FROM {t} WHERE {scope}{unique_key} IN (SELECT {unique_key} FROM ({sel}));" + )); + out.push(format!("INSERT INTO {t} {source};")); + } + } + out.push("COMMIT;".to_string()); + out + } +} + +/// The read that captures the DuckLake snapshot id produced by the write, for +/// the given attach alias (e.g. `_wm_target`). The worker runs this last and +/// records the result into `materialized_partition`. +pub fn snapshot_capture_sql(alias: &str) -> String { + format!("SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('{alias}');") +} + +/// Reserved attach alias for the materialization target, fully-qualified in all +/// generated SQL so a user `USE …;` in the setup blocks can't redirect the +/// write. The worker resolves the real `ATTACH 'ducklake:…' AS _wm_target (…)` +/// from the target ducklake's config and passes it in as `target_attach`. +pub const TARGET_ALIAS: &str = "_wm_target"; + +/// Assemble the full ordered statement list the DuckDB executor runs for a +/// managed `// materialize` script. This is the single entry point the worker +/// calls; it composes the already-tested pieces (classifier split → target +/// ATTACH → strategy codegen → snapshot capture) so their ordering lives in one +/// tested place rather than inline in the executor. +/// +/// `target_attach` is the real `ATTACH 'ducklake:…' AS _wm_target (…);` string +/// the worker built from config (it depends on resolved credentials, so it +/// can't be generated here). `target_table` is the table within that catalog +/// (e.g. `orders_daily`), referenced as `_wm_target.`. `asset_path` is +/// the full `/
` for the result summary. The trailing statement is +/// a one-row summary read (asset / rows / snapshot_id) that is both the job's +/// result (a useful preview) and what the worker records. +pub fn build_wrap_blocks( + plan: &WrapPlan, + target_attach: &str, + target_table: &str, + asset_path: &str, + partition_col: &str, + partition_value_sql: &str, + partitioned: bool, + strategy: MaterializeStrategy, +) -> Vec { + let target_qualified = format!("{TARGET_ALIAS}.{target_table}"); + let cg = MaterializeCodegen { + target_qualified: &target_qualified, + select_sql: &plan.output, + partition_col, + partition_value_sql, + partitioned, + strategy, + }; + let mut blocks: Vec = Vec::new(); + // Setup blocks come from the splitter with their `;` stripped — re-terminate + // each so that when the executor re-joins and re-splits the assembled query, + // adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH) + // don't merge into one malformed statement. + blocks.extend(plan.setup.iter().map(|s| terminate(s))); + blocks.push(target_attach.to_string()); + blocks.extend(cg.statements()); + blocks.push(materialize_result_sql( + &target_qualified, + asset_path, + partition_col, + partition_value_sql, + partitioned, + )); + blocks +} + +/// The trailing one-row summary the materialize run returns: the asset it +/// produced, the row count of the materialized slice (the partition when +/// partitioned, else the whole table), and the DuckLake snapshot it created. +/// This is both a useful preview result and the row the worker records. +pub fn materialize_result_sql( + target_qualified: &str, + asset_path: &str, + partition_col: &str, + partition_value_sql: &str, + partitioned: bool, +) -> String { + let (count_expr, partition_sel) = if partitioned { + // Row count is the slice this run wrote (the partition); `partition` + // lets the UI label the count and scope the preview to it. + ( + format!( + "(SELECT count(*) FROM {target_qualified} WHERE {partition_col} = {partition_value_sql})" + ), + format!("{partition_value_sql} AS partition, "), + ) + } else { + ( + format!("(SELECT count(*) FROM {target_qualified})"), + String::new(), + ) + }; + format!( + "SELECT 'ducklake://{asset_path}' AS materialized, \ + {partition_sel}{count_expr} AS rows, \ + (SELECT max(snapshot_id) FROM ducklake_snapshots('{TARGET_ALIAS}')) AS snapshot_id;" + ) +} + +// Ensure a statement ends with a single `;`. +fn terminate(stmt: &str) -> String { + let t = stmt.trim_end(); + if t.ends_with(';') { + t.to_string() + } else { + format!("{t};") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ok(sql: &str) -> WrapPlan { + classify_wrap(sql).expect("expected wrap-eligible") + } + fn err(sql: &str) -> WrapError { + classify_wrap(sql).expect_err("expected wrap-ineligible") + } + + #[test] + fn split_respects_strings_comments_idents() { + let sql = "SET x=1; -- a; comment\nSELECT ';' AS a, \"weird;col\" /* ; */ FROM t;"; + let s = split_statements(sql); + assert_eq!(s.len(), 2); + assert_eq!(s[0], "SET x=1"); + assert!(s[1].starts_with("SELECT")); + assert!(s[1].contains("\"weird;col\"")); + } + + #[test] + fn split_handles_escaped_quote() { + let s = split_statements("SELECT 'it''s; fine' AS a;"); + assert_eq!(s.len(), 1); + assert!(s[0].contains("it''s; fine")); + } + + #[test] + fn pipeline_annotations_are_stripped() { + // The real shape: `//` annotation lines above the SQL must not pollute + // the first block's classification (regression — they were being read + // as a leading `pipeline` keyword and rejected). + let p = ok("// pipeline\n// materialize ducklake://main/t\n// partitioned daily\nATTACH 'ducklake://main' AS dl;\nSELECT 1 AS id"); + assert_eq!(p.setup.len(), 1); + // The annotation lines are gone — the setup block starts at the real + // SQL (the `//` inside `ducklake://main` is legitimately retained). + assert!(p.setup[0].starts_with("ATTACH")); + assert!(p.output.starts_with("SELECT")); + } + + #[test] + fn bare_select_is_eligible() { + let p = ok("SELECT a, b FROM t WHERE c = '{partition}'"); + assert!(p.setup.is_empty()); + assert!(p.output.starts_with("SELECT")); + } + + #[test] + fn setup_then_select_is_eligible() { + let p = ok( + "ATTACH 'ducklake://main' AS dl;\n SET memory_limit='4GB';\n SELECT * FROM dl.orders", + ); + assert_eq!(p.setup.len(), 2); + assert!(p.output.starts_with("SELECT")); + } + + #[test] + fn create_temp_staging_is_setup() { + let p = ok("CREATE TEMP TABLE s AS SELECT 1; SELECT * FROM s"); + assert_eq!(p.setup.len(), 1); + assert_eq!( + classify_block("CREATE TEMP TABLE s AS SELECT 1"), + BlockClass::Setup + ); + assert_eq!( + classify_block("CREATE OR REPLACE TEMPORARY VIEW v AS SELECT 1"), + BlockClass::Setup + ); + } + + #[test] + fn with_cte_select_is_output_write_is_disallowed() { + assert_eq!( + classify_block("WITH x AS (SELECT 1) SELECT * FROM x"), + BlockClass::Output + ); + // CTE whose main statement inserts is a write, even though it starts WITH. + assert_eq!( + classify_block("WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x"), + BlockClass::Disallowed + ); + } + + #[test] + fn from_first_and_values_are_output() { + assert_eq!(classify_block("FROM t SELECT a"), BlockClass::Output); + assert_eq!(classify_block("VALUES (1),(2)"), BlockClass::Output); + assert_eq!(classify_block("TABLE t"), BlockClass::Output); + } + + #[test] + fn trailing_write_rejected() { + assert_eq!( + err("SELECT * FROM t; INSERT INTO u VALUES (1)"), + WrapError::OutputNotLast + ); + } + + #[test] + fn write_in_preamble_rejected() { + match err("INSERT INTO t VALUES (1); SELECT * FROM t") { + WrapError::DisallowedBlock { snippet } => assert!(snippet.starts_with("INSERT")), + e => panic!("wrong error: {e:?}"), + } + } + + #[test] + fn multiple_selects_rejected() { + assert_eq!( + err("SELECT 1; SELECT 2"), + WrapError::MultipleOutputs { count: 2 } + ); + } + + #[test] + fn no_select_and_empty_rejected() { + assert_eq!(err("CREATE TABLE t (a INT)"), WrapError::NoOutput); + assert_eq!(err(" -- just a comment\n"), WrapError::Empty); + } + + #[test] + fn use_cannot_redirect_is_classified_setup() { + // `USE` is allowed setup; generated SQL is fully qualified regardless. + assert_eq!(classify_block("USE dl"), BlockClass::Setup); + } + + #[test] + fn codegen_replace_partitioned() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT a FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: true, + strategy: MaterializeStrategy::Replace, + }; + let st = cg.statements(); + assert!(st[0].contains("CREATE TABLE IF NOT EXISTS _wm_target.orders_daily")); + assert!(st[0].contains("CAST(NULL AS VARCHAR) AS _wm_partition")); + assert!(st.iter().any( + |s| s == "ALTER TABLE _wm_target.orders_daily SET PARTITIONED BY (_wm_partition);" + )); + assert!(st.iter().any(|s| s.starts_with( + "DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'" + ))); + assert!(st.iter().any(|s| s.contains( + "INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19' AS _wm_partition" + ))); + assert_eq!(st.first().map(|_| &st[st.len() - 1]).unwrap(), "COMMIT;"); + } + + #[test] + fn codegen_merge_is_delete_by_key_plus_insert() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT order_id, amount FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: true, + strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() }, + }; + let st = cg.statements(); + // upsert = delete-by-key (partition-scoped) + insert — NO `MERGE INTO` + // (DuckLake's MERGE fails on fresh partitions). + assert!(!st.iter().any(|s| s.contains("MERGE INTO"))); + let del = st + .iter() + .find(|s| s.starts_with("DELETE FROM")) + .expect("delete stmt"); + assert!(del.contains( + "WHERE _wm_partition = '2026-06-19' AND order_id IN (SELECT order_id FROM (SELECT order_id, amount FROM dl.orders))" + )); + assert!(st + .iter() + .any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'"))); + } + + #[test] + fn codegen_append_inserts_only() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.events", + select_sql: "SELECT * FROM dl.raw", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: true, + strategy: MaterializeStrategy::Append, + }; + let st = cg.statements(); + assert!(st + .iter() + .any(|s| s.starts_with("INSERT INTO _wm_target.events"))); + assert!(!st.iter().any(|s| s.starts_with("DELETE"))); + assert!(!st.iter().any(|s| s.starts_with("MERGE"))); + } + + #[test] + fn codegen_whole_table_replace_is_create_or_replace() { + // Unpartitioned replace must use CREATE OR REPLACE so a changed SELECT + // schema (or a pre-existing table with a different schema) doesn't break + // — and nothing else (no bootstrap / DELETE / INSERT / txn). + let cg = MaterializeCodegen { + target_qualified: "_wm_target.customer_dim", + select_sql: "SELECT a, b, c FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Replace, + }; + let st = cg.statements(); + assert_eq!( + st, + vec![ + "CREATE OR REPLACE TABLE _wm_target.customer_dim AS SELECT * FROM (SELECT a, b, c FROM dl.src);" + .to_string() + ] + ); + } + + #[test] + fn snapshot_capture_targets_alias() { + assert_eq!( + snapshot_capture_sql("_wm_target"), + "SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('_wm_target');" + ); + } + + #[test] + fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() { + let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'"); + let blocks = build_wrap_blocks( + &plan, + "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');", + "orders_daily", + "main/orders_daily", + "_wm_partition", + "'2026-06-19'", + true, + MaterializeStrategy::Replace, + ); + // setup block first, then the target ATTACH, then codegen, then result. + assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl")); + // every setup block must be `;`-terminated so re-splitting can't merge it + // with the synthetic target ATTACH that follows. + assert!(blocks[0].ends_with(';')); + assert_eq!( + blocks[1], + "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');" + ); + assert!(blocks.iter().any(|b| b.contains("_wm_target.orders_daily"))); + assert!(blocks.iter().any(|b| b.starts_with( + "DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'" + ))); + // the trailing block is the one-row summary (asset / rows / snapshot_id), + // partition-scoped for the row count + let last = blocks.last().unwrap(); + assert!(last.contains("'ducklake://main/orders_daily' AS materialized")); + assert!(last.contains("'2026-06-19' AS partition")); + assert!(last.contains("WHERE _wm_partition = '2026-06-19') AS rows")); + assert!(last.contains("ducklake_snapshots('_wm_target')")); + } +} diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 158246022c..083ee7a8ba 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -234,5 +234,72 @@ "tag": null, "retry": null } + }, + { + "name": "materialize managed (default) with merge key", + "code": "// pipeline\n// materialize ducklake://analytics/orders_daily key=order_id\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders_daily", + "unique_key": "order_id" + } + } + }, + { + "name": "materialize manual escape hatch, first value wins", + "code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders_daily", + "manual": true + } + } + }, + { + "name": "materialize default-syntax shorthand with append", + "code": "// materialize ducklake append\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "main", + "append": true + } + } + }, + { + "name": "materialize manual with no target is dropped", + "code": "// materialize manual\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } } ] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 3efc9f0c9d..5e863fc06e 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -34,6 +34,22 @@ struct Expected { freshness: Option, tag: Option, retry: Option, + // Default-on-absent so the pre-existing fixtures (which omit it) keep + // deserializing; only fixtures exercising materialization set it. + #[serde(default)] + materialize: Option, +} + +#[derive(Deserialize)] +struct ExpectedMaterialize { + target_kind: String, + target_path: String, + #[serde(default)] + manual: bool, + #[serde(default)] + append: bool, + #[serde(default)] + unique_key: Option, } #[derive(Deserialize)] @@ -153,5 +169,25 @@ fn pipeline_annotation_fixtures_match() { want.is_some() ), } + + match (&got.materialize, &f.expected.materialize) { + (None, None) => {} + (Some(m), Some(e)) => { + assert_eq!( + kind_str(m.target_kind), + e.target_kind, + "{ctx}: materialize kind" + ); + assert_eq!(m.target_path, e.target_path, "{ctx}: materialize path"); + assert_eq!(m.manual, e.manual, "{ctx}: materialize manual"); + assert_eq!(m.append, e.append, "{ctx}: materialize append"); + assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key"); + } + (got, want) => panic!( + "{ctx}: materialize mismatch — got {:?}, want present={}", + got, + want.is_some() + ), + } } } diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index be685b2fe6..a982b02d35 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -22,6 +22,63 @@ pub fn workspaced_service() -> Router { .route("/list_favorites", get(list_favorites)) .route("/graph", get(asset_graph)) .route("/pipelines", get(list_pipeline_folders)) + .route("/partitions", get(list_partitions)) + .route("/record_materialization", post(record_materialization)) +} + +#[derive(Deserialize)] +struct PartitionsQuery { + // The materialized asset path (`/
`). + path: String, +} + +// Per-partition materialization status for a ducklake asset — drives the +// partition-status grid and the backfill worklist. Materialization targets are +// ducklake-only in v1, so the kind is fixed. +async fn list_partitions( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = windmill_common::materialization::list_materialized_partitions( + &mut *tx, + &w_id, + AssetKind::Ducklake, + &q.path, + ) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +// Record a materialization outcome from a polyglot (Python/TS) `wmill.ducklake` +// helper running as a pipeline step. The DuckDB `// materialize` engine records +// this itself; the SDK helpers post here instead so SDK-materialized slices show +// up in the grid identically. RLS-scoped to the caller's workspace. +async fn record_materialization( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Json(req): Json, +) -> JsonResult<()> { + let mut tx = user_db.begin(&authed).await?; + windmill_common::materialization::record_materialization( + &mut *tx, + &w_id, + req.asset_kind, + &req.asset_path, + &req.partition, + req.status, + req.snapshot_id, + req.row_count, + req.job_id, + req.error.as_deref(), + ) + .await?; + tx.commit().await?; + Ok(Json(())) } #[derive(Deserialize)] diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index 2dee051c93..ecf0ff0b61 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -26,6 +26,7 @@ windmill-parser-ts.workspace = true windmill-parser.workspace = true windmill-parser-ts-asset.workspace = true windmill-parser-sql-asset.workspace = true +windmill-parser-sql.workspace = true windmill-parser-yaml.workspace = true axum.workspace = true diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 8115e2ef78..ea54b83c31 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1251,6 +1251,72 @@ async fn create_script_internal<'c>( windmill_common::pipeline_advanced::freshness_enforcement_todo() ); } + // `// materialize` materializes a `ducklake:///
` target from a + // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake + // target would otherwise deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), + // and a non-DuckDB script never reaches the executor that records state. The + // managed-only checks (single trailing SELECT, no SQL args) come after — a + // `manual` script owns its DDL and skips them. + if let Some(m) = pipeline_annotations.materialize.as_ref() { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages.", + ns.language.as_str() + ))); + } + if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake target \ + (`ducklake:///
`); other asset kinds aren't materializable." + .to_string(), + )); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + if !m.manual { + if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { + return Err(Error::BadRequest(e.message())); + } + // Managed materialize strips line comments when it wraps the SELECT, + // so a `-- $name (TYPE)` declaration is lost while its `$name` + // reference survives in the embedded SELECT — it would run unbound. + // Managed materialize takes no SQL args (the partition is supplied by + // the engine, not bound). Reject declared args with a clear error. + if let Ok(sig) = windmill_parser_sql::parse_duckdb_sig(&ns.content) { + if !sig.args.is_empty() { + let names = sig + .args + .iter() + .map(|a| format!("${}", a.name)) + .collect::>() + .join(", "); + return Err(Error::BadRequest(format!( + "managed `// materialize` cannot take SQL arguments ({names}): wrapping your \ + SELECT drops the `-- $arg` declarations, so they would run unbound. The \ + partition is supplied by the engine — reference its value with the \ + `{{partition}}` token, or use `// materialize manual` to write the DDL (and \ + bind args) yourself." + ))); + } + } + } + // `key=` (merge) and `append` are mutually exclusive reconciliation + // strategies; append (INSERT-only) wins. Surface the conflict rather + // than silently dropping the dedup the author may have intended. + if m.unique_key.is_some() && m.append { + tracing::warn!( + "script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)", + ns.path + ); + } + } let in_pipeline = pipeline_annotations.in_pipeline; // `// trigger all` → AND join barrier (else OR, the default). let pipeline_join_all = !pipeline_annotations.join_mode.is_any(); @@ -1290,6 +1356,26 @@ async fn create_script_internal<'c>( &ns.content, ns.assets.take(), ); + // Register the `// materialize` target as a write asset so the deployed + // asset graph shows this script as the producer of the managed table — the + // body's `SELECT` doesn't express the write (the runtime generates it), so + // server-side inference wouldn't otherwise link it. + let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() { + let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind); + let mut a = effective_assets.unwrap_or_default(); + if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) { + a.push(windmill_common::assets::AssetWithAltAccessType { + path: m.target_path.clone(), + kind, + access_type: Some(windmill_common::assets::AssetUsageAccessType::W), + alt_access_type: None, + columns: None, + }); + } + Some(a) + } else { + effective_assets + }; let auto_kind = if in_pipeline { Some("pipeline".to_string()) } else if ci_test_refs.is_some() { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1e0e3cef92..8675c7724a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -22576,6 +22576,13 @@ components: type: string parent_hash: type: string + auto_parent: + type: boolean + description: >- + When true, the backend resolves the parent to the current deployed + head for this path within the transaction (ignoring parent_hash), + instead of failing with a "lineage must be linear" error when the + supplied parent_hash is stale. summary: type: string description: diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index c501bbed6e..7c2a475361 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -61,6 +61,7 @@ pub mod indexer; pub mod instance_config; pub mod job_metrics; pub mod log_context; +pub mod materialization; pub mod min_version; pub mod notify_events; pub mod runtime_assets; diff --git a/backend/windmill-common/src/materialization.rs b/backend/windmill-common/src/materialization.rs new file mode 100644 index 0000000000..ccc1f7782e --- /dev/null +++ b/backend/windmill-common/src/materialization.rs @@ -0,0 +1,135 @@ +//! CE materialization state — the per-partition status recorded by the managed +//! `// materialize` write (in windmill-worker), read by the partition-status +//! grid and by the EE backfill worklist. +//! +//! The write engine and this state are CE; only automatic partition +//! *resolution* (`partition_ee`) and *backfill* orchestration +//! (`pipeline_advanced_ee`) are enterprise. This module is the shared seam: +//! the EE backfill enumerates the partitions in a range, diffs them against +//! these rows to find the missing/failed set, and pushes one CE materialization +//! job per gap (with an explicit `partition` arg — which runs idempotently and +//! upserts the row here). Nothing about that orchestration lives in this file; +//! it only needs the rows to exist, which is why recording is CE. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::PgExecutor; +use uuid::Uuid; + +use crate::assets::AssetKind; +use crate::error::Result; + +/// Sentinel `partition` value for an unpartitioned (whole-table) +/// materialization — partition is part of the primary key and cannot be NULL. +pub const UNPARTITIONED: &str = ""; + +/// Mirrors the `MATERIALIZATION_STATUS` pg enum (see migration +/// `20260619170118_add_materialized_partition`). +#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[sqlx(type_name = "MATERIALIZATION_STATUS", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MaterializationStatus { + Running, + Materialized, + Failed, +} + +/// The materialization outcome an agent worker (`Connection::Http`, no direct +/// DB) sends to the API to be recorded. Mirrors the `record_materialization` +/// args; the API handler unpacks it and calls that function with its own DB. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecordMaterializationRequest { + pub asset_kind: AssetKind, + pub asset_path: String, + pub partition: String, + pub status: MaterializationStatus, + pub snapshot_id: Option, + pub row_count: Option, + pub job_id: Option, + pub error: Option, +} + +/// Upsert the latest materialization state for one (asset, partition) slice. +/// The worker records the terminal outcome once the write finishes: +/// `Materialized` (with the DuckLake `snapshot_id` + `row_count`) or `Failed` +/// (with `error`). `Running` mirrors the pg enum but has no writer in this flow. +/// Idempotent: re-running the same partition overwrites the row — exactly the +/// backfill / failure-recovery contract. +#[allow(clippy::too_many_arguments)] +pub async fn record_materialization<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_kind: AssetKind, + asset_path: &str, + partition: &str, + status: MaterializationStatus, + snapshot_id: Option, + row_count: Option, + job_id: Option, + error: Option<&str>, +) -> Result<()> { + sqlx::query!( + "INSERT INTO materialized_partition + (workspace_id, asset_kind, asset_path, partition, status, + snapshot_id, row_count, job_id, materialized_at, error) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9) + ON CONFLICT (workspace_id, asset_kind, asset_path, partition) + DO UPDATE SET status = EXCLUDED.status, + snapshot_id = EXCLUDED.snapshot_id, + row_count = EXCLUDED.row_count, + job_id = EXCLUDED.job_id, + materialized_at = now(), + error = EXCLUDED.error", + workspace_id, + asset_kind as AssetKind, + asset_path, + partition, + status as MaterializationStatus, + snapshot_id, + row_count, + job_id, + error, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// One materialized-partition row, for the status grid / backfill diff. +#[derive(sqlx::FromRow, Debug, Clone, Serialize)] +pub struct MaterializedPartition { + pub asset_kind: AssetKind, + pub asset_path: String, + pub partition: String, + pub status: MaterializationStatus, + pub snapshot_id: Option, + pub row_count: Option, + pub job_id: Option, + pub materialized_at: DateTime, + pub error: Option, +} + +/// All recorded partitions for one asset, newest first — the grid's data and +/// the backfill worklist's "what already exists" set. +pub async fn list_materialized_partitions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_kind: AssetKind, + asset_path: &str, +) -> Result> { + let rows = sqlx::query_as!( + MaterializedPartition, + r#"SELECT asset_kind AS "asset_kind: AssetKind", asset_path, partition, + status AS "status: MaterializationStatus", snapshot_id, + row_count, job_id, materialized_at, error + FROM materialized_partition + WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3 + ORDER BY partition DESC"#, + workspace_id, + asset_kind as AssetKind, + asset_path, + ) + .fetch_all(executor) + .await?; + Ok(rows) +} diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 9bf0ea1834..5320c8ef6c 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -78,4 +78,22 @@ pub async fn get_datatable_resource_from_agent_http( .await } +/// Record a materialization outcome from an agent worker (no direct DB) via the +/// API, so `materialized_partition` state lands the same as on a Sql worker. +// Only called from the duckdb executor, which is itself `#[cfg(feature = "duckdb")]`. +#[cfg(feature = "duckdb")] +pub async fn record_materialization_from_agent_http( + client: &HttpClient, + w_id: &str, + req: &windmill_common::materialization::RecordMaterializationRequest, +) -> anyhow::Result<()> { + client + .post( + &format!("/api/w/{}/agent_workers/record_materialization", w_id), + None, + req, + ) + .await +} + pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping"; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 1a474ee9a0..3b6a07f702 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -32,6 +32,178 @@ use crate::sql_utils::remove_comments; use windmill_common::client::AuthedClient; use windmill_object_store::DEFAULT_STORAGE; +// What a `// materialize` run records into `materialized_partition` once it +// finishes. `asset_path` is the full `/
` (the asset identity); +// `partition` is "" for an unpartitioned (whole-table) materialization. +struct MaterializeExec { + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + partition: String, +} + +// If `query` declares `// materialize `, return what to record plus, +// for the default managed mode, the rewritten managed-write SQL (in `manual` +// mode the script writes its own DDL, so the rewrite is `None`). The rewritten +// SQL contains a synthetic `ATTACH 'ducklake://' AS _wm_target` that the +// normal ATTACH-transform pass resolves to real credentials — the same path as +// the user's own ATTACH. Returns `None` when there is no materialize annotation +// or the target isn't a ducklake (only ducklake is materialized in v1). +fn build_materialized_query( + query: &str, + partition_value: Option<&str>, +) -> Result, MaterializeExec)>> { + use windmill_parser::asset_parser::{parse_pipeline_annotations, AssetKind as PAssetKind}; + use windmill_parser::sql_materialize::{ + build_wrap_blocks, classify_wrap, MaterializeStrategy, TARGET_ALIAS, + }; + + let ann = parse_pipeline_annotations(query); + let Some(m) = ann.materialize else { + return Ok(None); + }; + if m.target_kind != PAssetKind::Ducklake { + return Ok(None); + } + let partitioned = ann.partition.is_some(); + let partition = partition_value.unwrap_or("").to_string(); + // Partition *resolution* is enterprise; in its absence a partitioned + // materialize only runs with an explicit `partition` arg. Fail loudly rather + // than silently materialize the wrong (empty) slice. + if partitioned && partition.is_empty() { + return Err(Error::ExecutionErr( + "materialize: a `// partitioned` script ran with no resolved partition — pass an \ + explicit `partition` arg, or enable enterprise partition resolution" + .to_string(), + )); + } + // Convention: `ducklake:///
` — is the configured + // ducklake (resolved like a user ATTACH),
is the rest. + let (ducklake_name, table) = m + .target_path + .split_once('/') + .unwrap_or((m.target_path.as_str(), "")); + let meta = MaterializeExec { + asset_kind: windmill_common::assets::AssetKind::Ducklake, + asset_path: m.target_path.clone(), + partition: partition.clone(), + }; + + if m.manual { + // Escape hatch: the script owns its DDL; we only record state. + return Ok(Some((None, meta))); + } + if table.is_empty() { + return Err(Error::ExecutionErr(format!( + "materialize: target `ducklake://{}` has no table (use ducklake:///
)", + m.target_path + ))); + } + let mut plan = classify_wrap(query).map_err(|e| Error::ExecutionErr(e.message()))?; + // Resolve the `{partition}` token (same token `// on` asset URIs use) to the + // current partition value everywhere in the managed script, so a partitioned + // materialize can filter its source by the active slice, e.g. + // `WHERE day = {partition}`. The token is always replaced by a *complete* + // escaped SQL literal (`'…'` with `'` doubled) whether or not the author + // quoted it — so a run caller can't pass metacharacters that break out of + // the literal and alter statement boundaries. The pre-quoted form + // `'{partition}'` is matched first so it doesn't become `''…''`. Only + // meaningful when partitioned. + if partitioned { + let lit = format!("'{}'", partition.replace('\'', "''")); + let tok = windmill_common::assets::PARTITION_TOKEN; + let quoted_tok = format!("'{tok}'"); + plan.output = plan.output.replace("ed_tok, &lit).replace(tok, &lit); + for s in plan.setup.iter_mut() { + *s = s.replace("ed_tok, &lit).replace(tok, &lit); + } + } + let strategy = if m.append { + MaterializeStrategy::Append + } else if let Some(uk) = m.unique_key { + MaterializeStrategy::Merge { unique_key: uk } + } else { + MaterializeStrategy::Replace + }; + // Inline the partition as an escaped SQL literal (DuckLake has no bind for + // the partition column in our generated DDL). + let pval = format!("'{}'", partition.replace('\'', "''")); + let synthetic_attach = format!("ATTACH 'ducklake://{ducklake_name}' AS {TARGET_ALIAS};"); + let blocks = build_wrap_blocks( + &plan, + &synthetic_attach, + table, + &m.target_path, + "_wm_partition", + &pval, + partitioned, + strategy, + ); + Ok(Some((Some(blocks.join("\n")), meta))) +} + +// Pull a named i64 field (`snapshot_id` / `rows`) out of the trailing summary +// read — which in wrap mode is the job result. Shape-tolerant (object / array / +// nested), returns None if absent (literal mode, or capture failed). +fn extract_i64(result: &RawValue, field: &str) -> Option { + fn find(v: &Value, field: &str) -> Option { + match v { + Value::Number(n) => n.as_i64(), + Value::Object(m) => m.get(field).and_then(|x| find(x, field)), + Value::Array(a) => a.iter().find_map(|x| find(x, field)), + _ => None, + } + } + find(&serde_json::from_str::(result.get()).ok()?, field) +} + +// Best-effort record of a materialization outcome. On a Sql connection it writes +// the row directly; on an agent worker (Http, no direct DB) it posts to the API +// so state lands the same way. Never fails the job — a lost row degrades the +// grid, not the run. +async fn record_mat( + conn: &Connection, + w_id: &str, + job_id: Uuid, + meta: &MaterializeExec, + status: windmill_common::materialization::MaterializationStatus, + snapshot_id: Option, + row_count: Option, + error: Option<&str>, +) { + let req = windmill_common::materialization::RecordMaterializationRequest { + asset_kind: meta.asset_kind, + asset_path: meta.asset_path.clone(), + partition: meta.partition.clone(), + status, + snapshot_id, + row_count, + job_id: Some(job_id), + error: error.map(|e| e.to_string()), + }; + let res: anyhow::Result<()> = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + w_id, + req.asset_kind, + &req.asset_path, + &req.partition, + req.status, + req.snapshot_id, + req.row_count, + req.job_id, + req.error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http(client, w_id, &req).await + } + }; + if let Err(e) = res { + tracing::warn!("failed to record materialization state: {e:#}"); + } +} + pub async fn do_duckdb( job: &MiniPulledJob, client: &AuthedClient, @@ -68,6 +240,30 @@ pub async fn do_duckdb( let mut hidden_passwords = hidden_passwords.clone(); let mut bigquery_credentials = None; + // Materialization (`// materialize`): rewrite a wrap script into managed + // DDL (its synthetic target ATTACH is resolved by the transform pass + // below, like the user's own ATTACH); a literal script is left as-is. + // `materialize` also carries what to record once the run finishes. + let partition_value: Option = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|rv| serde_json::from_str::(rv.get()).ok()) + .filter(|s| !s.is_empty()); + let materialize = if query.contains("materialize") { + build_materialized_query(query, partition_value.as_deref())? + } else { + None + }; + let materialized_query; + let query: &str = match &materialize { + Some((Some(rewritten), _)) => { + materialized_query = rewritten.clone(); + &materialized_query + } + _ => query, + }; + let sig = parse_duckdb_sig(query)?.args; let mut job_args = build_args_values(job, client, conn).await?; @@ -199,6 +395,19 @@ pub async fn do_duckdb( let (result, column_order) = match result { Ok(r) => r, Err(e) => { + if let Some((_, meta)) = &materialize { + record_mat( + conn, + &job.workspace_id, + job.id, + meta, + windmill_common::materialization::MaterializationStatus::Failed, + None, + None, + Some(&e.to_string()), + ) + .await; + } if let Some(s3_proxy_err) = S3_PROXY_LAST_ERRORS_CACHE.get(&client.token) { return Err(Error::ExecutionErr(format!( "{}\n\nS3 Related Error: {}", @@ -210,6 +419,24 @@ pub async fn do_duckdb( } }; + if let Some((_, meta)) = &materialize { + // In wrap mode the job result is the summary read (snapshot_id + + // rows); in literal mode there is none, so both stay None. + let snapshot_id = extract_i64(&result, "snapshot_id"); + let row_count = extract_i64(&result, "rows"); + record_mat( + conn, + &job.workspace_id, + job.id, + meta, + windmill_common::materialization::MaterializationStatus::Materialized, + snapshot_id, + row_count, + None, + ) + .await; + } + drop(bigquery_credentials); *column_order_ref = column_order; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1ca77310ff..45ae8c3c25 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4324,20 +4324,25 @@ async fn resolve_partition_for_job( job: &MiniPulledJob, code: &str, conn: &Connection, -) -> error::Result> { +) -> error::Result<(Option, bool)> { use windmill_common::partition::{resolve_partition, PARTITION_ARG}; use windmill_parser::asset_parser::PartitionKind; - // Only deployed scripts participate in asset pipelines. Cheap - // substring guard so the overwhelming majority of script jobs (no - // `// partitioned` line) skip the full annotation scan on the hot - // path; a false positive only costs one extra parse, never wrong. - if !matches!(job.kind, JobKind::Script) || !code.contains("partitioned") { - return Ok(None); + // Only deployed scripts participate in asset pipelines. Cheap substring + // guard so the overwhelming majority of script jobs skip the annotation + // scan; when one might be present we parse *once* here and reuse the result + // for both `in_pipeline` (→ WM_PIPELINE env, read by the wmll.ducklake SDK to + // record state) and `partition` resolution — no second parse downstream. The + // bool is whether the script is a `// pipeline` member. + if !matches!(job.kind, JobKind::Script) + || !(code.contains("pipeline") || code.contains("partitioned")) + { + return Ok((None, false)); } - let Some(spec) = windmill_parser::asset_parser::parse_pipeline_annotations(code).partition - else { - return Ok(None); + let ann = windmill_parser::asset_parser::parse_pipeline_annotations(code); + let in_pipeline = ann.in_pipeline; + let Some(spec) = ann.partition else { + return Ok((None, in_pipeline)); }; // Already resolved upstream — explicit run arg, backfill, or @@ -4349,7 +4354,7 @@ async fn resolve_partition_for_job( .is_some_and(|s| !s.is_empty()) }); if already_set { - return Ok(None); + return Ok((None, in_pipeline)); } // `dynamic` extracts from the triggering payload (the `trigger` object @@ -4382,7 +4387,7 @@ async fn resolve_partition_for_job( job_id = %job.id, "partitioned script resolved to no partition (before start anchor); running without one" ); - return Ok(None); + return Ok((None, in_pipeline)); }; // Persist back so dispatch_asset_triggers (which reads the producer's @@ -4404,7 +4409,7 @@ async fn resolve_partition_for_job( windmill_common::worker::to_raw_value(&value), ); updated.args = Some(Json(map)); - Ok(Some(updated)) + Ok((Some(updated), in_pipeline)) } #[tracing::instrument(level = "trace", skip_all)] @@ -4566,7 +4571,8 @@ async fn handle_code_execution_job( // `// partitioned` (if any) and shadow `job` with a clone whose args // carry the resolved `partition` for the rest of execution. let _job_with_partition; - let job = match resolve_partition_for_job(job, code, conn).await? { + let (resolved_job, in_pipeline) = resolve_partition_for_job(job, code, conn).await?; + let job = match resolved_job { Some(j) => { _job_with_partition = j; &_job_with_partition @@ -4619,6 +4625,7 @@ async fn handle_code_execution_job( lock, &modules, false, + in_pipeline, ) .await } @@ -4685,6 +4692,9 @@ pub async fn run_language_executor( lock: &Option, modules: &Option>, run_inline: bool, + // Whether the script is a `// pipeline` member (parsed once upstream) — sets + // WM_PIPELINE so the wmll.ducklake SDK helpers record materialization state. + in_pipeline: bool, ) -> error::Result> { // Defense-in-depth (GHSA-wxjq-w5pj-jqhx): the entrypoint override is // interpolated verbatim into a code position of the generated language @@ -5047,6 +5057,11 @@ mount {{ #[allow(unused_mut)] let mut envs = build_envs(envs.as_ref())?; + // Signal pipeline context to the script so the wmll.ducklake SDK helpers + // record materialization state (the grid/backfill) and skip it otherwise. + if in_pipeline { + envs.insert("WM_PIPELINE".to_string(), "true".to_string()); + } let Some(language) = language else { return Err(Error::ExecutionErr( @@ -5832,6 +5847,7 @@ pub fn init_worker_internal_server_inline_utils( &None, &None, true, + false, ) .await }) @@ -5913,6 +5929,7 @@ pub fn init_worker_internal_server_inline_utils( &content_info.lockfile, &content_info.modules, true, + false, ) .await }) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 1e6202b443..422e2bc03a 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1102,6 +1102,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction * let sql = wmill.ducklake("my_lake:analytics") */ ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Idempotently materialize \`selectSql\` into a ducklake table for one + * partition (or the whole table when \`partition\` is omitted) — the client-side + * equivalent of the \`// materialize\` engine. + * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it + * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert). + * Safe to re-run for the same partition (backfill / failure-recovery). + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`. + */ +upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement + +/** + * INSERT-only materialization (no dedup/replace) for append-only tables. + * Re-running the same partition duplicates rows — use only for immutable + * event-log sources. + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`. + */ +appendPartition(opts: Omit,): SqlStatement `, "write-script-bunnative": `--- name: write-script-bunnative @@ -1833,6 +1856,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction * let sql = wmill.ducklake("my_lake:analytics") */ ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Idempotently materialize \`selectSql\` into a ducklake table for one + * partition (or the whole table when \`partition\` is omitted) — the client-side + * equivalent of the \`// materialize\` engine. + * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it + * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert). + * Safe to re-run for the same partition (backfill / failure-recovery). + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`. + */ +upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement + +/** + * INSERT-only materialization (no dedup/replace) for append-only tables. + * Re-running the same partition duplicates rows — use only for immutable + * event-log sources. + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`. + */ +appendPartition(opts: Omit,): SqlStatement `, "write-script-csharp": `--- name: write-script-csharp @@ -2656,6 +2702,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction * let sql = wmill.ducklake("my_lake:analytics") */ ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Idempotently materialize \`selectSql\` into a ducklake table for one + * partition (or the whole table when \`partition\` is omitted) — the client-side + * equivalent of the \`// materialize\` engine. + * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it + * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert). + * Safe to re-run for the same partition (backfill / failure-recovery). + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`. + */ +upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement + +/** + * INSERT-only materialization (no dedup/replace) for append-only tables. + * Re-running the same partition duplicates rows — use only for immutable + * event-log sources. + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`. + */ +appendPartition(opts: Omit,): SqlStatement `, "write-script-duckdb": `--- name: write-script-duckdb @@ -4306,6 +4375,27 @@ def stream_result(stream) -> None # SqlQuery instance for fetching results def query(sql: str, *args) -> SqlQuery +# Idempotently materialize the rows of \`select_sql\` into ducklake +# \`table\` for one \`partition\` (or the whole table when \`partition\` is +# None). Client-side equivalent of the \`// materialize\` engine: with +# \`unique_key\` it upserts within the slice (delete-by-key + insert); +# without it, it replaces (whole table → CREATE OR REPLACE; partition → +# delete the partition + insert). Re-running the same slice is safe — the +# backfill / failure-recovery contract. +# +# The partition value is bound as a DuckDB arg (never string-interpolated) +# so it cannot inject SQL. \`select_sql\` is trusted (your own query). +def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None) + +# INSERT-only materialization (no dedup / no replace) for an immutable +# event-log table — for one \`partition\`, or the whole table when +# \`partition\` is None. NOTE: unlike \`upsert_partition\`, re-running the same +# slice duplicates rows — use only for append-only sources. +def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None) + +# Read a materialized ducklake table, optionally a single partition. +def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None) + # Execute query and fetch results. # # Args: diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md new file mode 100644 index 0000000000..3b5c95c770 --- /dev/null +++ b/docs/ducklake-materialization.md @@ -0,0 +1,250 @@ +# DuckLake-native materialization + +Design sketch for "managed, versioned, incremental" assets built on the +DuckLake substrate. This is a companion to [`pipelines-vs-dbt.md`](./pipelines-vs-dbt.md) +and extends its **Path C (hybrid, partition-first)** recommendation. The new +contribution here is leveraging DuckLake's snapshot/time-travel layer, which +the earlier doc's incremental deep-dive did not use. The annotation grammar is +reconciled with that doc — `// partitioned` + `// unique_key` + `// append` +stay canonical; nothing here forks a competing vocabulary. + +## The core reframe + +dbt had to *build* a materialization engine (compile SQL → `CREATE TABLE AS` / +incremental `MERGE` / SCD2 snapshots) because the warehouse gives it nothing +but raw SQL over a mutable table. DuckLake hands us, at the storage layer, the +four things that engine exists to provide: + +| Capability | Source | +|---|---| +| ACID multi-statement transactions | DuckLake | +| A snapshot per commit + time-travel (`AT (VERSION => n)` / `AT (TIMESTAMP => ...)`) | DuckLake | +| Physical partitioning + pruning (`ALTER TABLE … SET PARTITIONED BY (…)`) | DuckLake | +| Schema evolution tracked in the catalog DB | DuckLake | + +Windmill already attaches DuckLake fully — see `transform_attach_ducklake` +(`backend/windmill-worker/src/duckdb_executor.rs:661`), which rewrites a user's +`ATTACH 'ducklake://name' AS dl` into the real +`ATTACH 'ducklake:postgres:…' AS dl (DATA_PATH 's3://…', OVERRIDE_DATA_PATH +TRUE, AUTOMATIC_MIGRATION TRUE)` at `duckdb_executor.rs:730`. But today every +write is a destructive overwrite and **all four capabilities above are thrown +away** — snapshots are never surfaced, partitioning is purely an orchestration +concept disconnected from the physical layout. + +So the materialization engine we need is a thin layer — a write-strategy +wrapper + snapshot capture — not a dbt rebuild. This is exactly the +"buy A's 80% without B's dialect-rewriting tax" tradeoff `pipelines-vs-dbt.md` +argued for; DuckLake is what makes the remaining 20% (versioning, reproducible +reads, materialization history) nearly free instead of a second project. + +## Annotation grammar (final) + +One self-documenting line; managed-by-default. Strategy options live *on* the +`materialize` line (they have no meaning without it), while `// partitioned` +stays separate because it is cross-cutting (cascade + scheduling + materialize). + +``` +// materialize ducklake://analytics/orders_daily → managed, replace (default) +// materialize ducklake://analytics/orders_daily key=order_id → managed, merge +// materialize ducklake://analytics/orders_daily append → managed, append +// materialize manual ducklake://analytics/orders_daily → track-only escape hatch +``` + +- **managed (default)** — the script is *setup + one trailing `SELECT`*; Windmill + generates the write DDL, captures the DuckLake snapshot, and records state. + DuckDB-only; validated at deploy (a non-SELECT script is rejected with a clear + error pointing to the `wmll.ducklake` helpers). +- **`manual`** — escape hatch: the script writes its own DDL; Windmill only + records state (no snapshot capture, no idempotency guarantee). Rare; explicit. +- **`key=`** → MERGE (dedup within slice); **`append`** → INSERT-only; + neither → DELETE-by-partition + INSERT (replace). `append` wins over `key` if + both are given (deploy warning). +- **`// partitioned `** — unit of work + state + backfill (separate; + cross-cutting). Polyglot / multi-statement writes use the `wmll.ducklake` + helpers instead of `// materialize`. + +There is no `wrap` keyword — `materialize` *is* "manage the write," so it was +redundant; the only reason for it was to carve out the weak track-only mode, +which is now the explicit `manual` opt-out. + +DuckLake snapshots are **orthogonal to all of the above** — they apply to every +strategy automatically because every write is a DuckLake commit. The user never +annotates for versioning; they get it. + +## Executor codegen + +### The seam + +`run_duckdb` already splits the script into statement blocks and rewrites +custom `ATTACH` blocks in a single pass before execution +(`duckdb_executor.rs:114-160`): + +```rust +let query_block_list = parse_sql_blocks(&query, true); +// each block: remove_comments → if ducklake/datatable ATTACH, expand; else passthrough +``` + +All blocks run in order on one DuckDB connection. Materialize has two modes: + +1. **Managed (default).** The user writes *setup + one trailing `SELECT`*. Windmill + replaces that SELECT with generated statements, wrapped in an *explicit + DuckLake transaction it controls* — never textual `BEGIN/COMMIT` injected + around the user's other statements (fragile across their own `ATTACH`s and + multi-statement SQL). For a partitioned `replace` the SELECT block expands to: + + ```sql + -- generated for: // partitioned daily ; target = dl.orders_daily ; partition = '2026-06-19' + CREATE TABLE IF NOT EXISTS dl.orders_daily AS + SELECT *, CAST(NULL AS VARCHAR) AS _wm_partition FROM () WHERE false; -- first-run bootstrap + ALTER TABLE dl.orders_daily SET PARTITIONED BY (_wm_partition); + BEGIN TRANSACTION; + DELETE FROM dl.orders_daily WHERE _wm_partition = '2026-06-19'; + INSERT INTO dl.orders_daily SELECT *, '2026-06-19' AS _wm_partition FROM (); + COMMIT; + ``` + + The strategy variants are all DELETE+INSERT-shaped — no `MERGE INTO`, which + DuckLake can't reliably run on a fresh partition (it 404s writing the first + rows): + - **whole-table replace** (no `// partitioned`) → a single `CREATE OR REPLACE + TABLE … AS ` (handles schema changes, still snapshots). + - **`key=`** → `DELETE FROM … WHERE [ AND] IN (SELECT + FROM ())` then `INSERT` (upsert within the slice). + - **`append`** → the `DELETE` is dropped (insert-only). + +2. **`manual`.** The user writes their own DDL inside their own `BEGIN … COMMIT`; + Windmill injects nothing into the body and only records state (no snapshot + capture, no idempotency guarantee). + +The `_wm_partition` column is the physical link the orchestration layer lacks: on +first materialize Windmill runs `ALTER TABLE … SET PARTITIONED BY (_wm_partition)` +so DuckLake prunes on read and DELETE-by-partition rewrites only that partition's +Parquet files. + +> Storage: DuckLake writes go to `s3://_default_/` through the windmill S3 proxy +> (`/api/w/{ws}/s3_proxy`, gated behind the `parquet` + `private` features). The +> proxy must sign the SigV4 canonical URI with **single** percent-encoding — the +> SigV4 default (`Double`) 401s Hive-partition keys like `_wm_partition=2026-06-19` +> (the `=` double-encodes to `%253D` vs the client's `%3D`). + +### Run summary capture + +After the generated blocks, Windmill appends one read block — it is both the job's +result (a useful preview rendered as the materialized table) and the row it records: + +```sql +SELECT 'ducklake:///
' AS materialized, + '' AS partition, -- only when partitioned + (SELECT count(*) FROM [WHERE _wm_partition = '']) AS rows, + (SELECT max(snapshot_id) FROM ducklake_snapshots('')) AS snapshot_id; +``` + +The `snapshot_id` and `rows` are persisted as `materialized_partition` metadata. +One extra round-trip per materialization, no new infra. + +## Metadata schema + +Extends the `materialized_partitions` table proposed in `pipelines-vs-dbt.md` +§"First implementation slice" with the DuckLake snapshot id: + +``` +materialized_partition ( + workspace_id TEXT, + asset_kind TEXT, -- 'ducklake' + asset_path TEXT, -- 'analytics/orders_daily' + partition TEXT, -- '2026-06-19' (NULL for unpartitioned) + snapshot_id BIGINT, -- DuckLake snapshot produced by this materialize + row_count BIGINT, + job_id UUID, + materialized_at TIMESTAMPTZ, + PRIMARY KEY (workspace_id, asset_kind, asset_path, partition) +) +``` + +This one table drives four things at once: + +- **Observability** — "last materialized: snapshot 42, 1.2M rows, 09:14" per + asset node (closes the Dagster-catalog gap from the v1-readiness review). +- **Run-stale / gap detection** — which partitions exist, which are missing. +- **Backfill** — the missing/failed set *is* the backfill worklist. +- **Snapshot pinning** — see below. + +## Reproducibility — the beyond-dbt part + +Because every materialization records the snapshot it produced, a downstream +consumer can read the *exact* upstream snapshot its run saw: + +```sql +FROM dl.orders_daily AT (VERSION => $WM_UPSTREAM_SNAPSHOT) +``` + +The cascade already threads a `trigger` blob (producer path, partition) to each +subscriber; add the producer's captured `snapshot_id` to it, and a consumer's +read is pinned to the upstream state at dispatch time. That makes the *whole +pipeline* reproducible and time-travelable — something dbt has no native answer +for (dbt models are always "whatever's in the warehouse now"). It also gives +rollback (re-point an asset to snapshot N) and "what did this table look like at +the failing run" debugging, for free off the same captured ids. + +This is the differentiator worth leaning on. It is not catch-up to dbt; it is a +capability dbt structurally cannot offer, and DuckLake gives it to us at the +cost of recording one integer per run. + +It also means **we do not build SCD2 snapshots** (gap #4 in `pipelines-vs-dbt.md`): +DuckLake time-travel is a strictly better answer for most of what dbt's +`{% snapshot %}` is used for. One fewer engine to write. + +## Scoping decision: DuckLake vs DataTable + +**Make DuckLake the materialization/versioning substrate; keep DataTable as the +live operational table with no versioning.** DataTable is plain Postgres +(`transform_attach_datatable`, `duckdb_executor.rs:742`) — no native snapshots +or time-travel — so giving *it* the versioned/incremental story means building +MVCC-on-top ourselves (history tables, SCD2), precisely the complexity this +DuckLake approach exists to avoid. Clean split: + +- `ducklake://` → analytics, versioned, reproducible, backfillable. +- `datatable://` → mutable app/operational state; partition idempotency via + DELETE+INSERT still works, but no snapshot/time-travel layer. + +Don't try to give both the full treatment for v1. + +## v1 slice (smallest viable) + +1. **Partition runtime context** — resolve `(value, start, end)` and surface as + `WM_PARTITION*` bind/env (Path C step 1; partly built per the + pipeline-partition-runtime work). +2. **Physical partition wiring** — `_wm_partition` column + `SET PARTITIONED BY` + on first materialize for `ducklake://` targets. +3. **Strategy templates** — DELETE+INSERT default (`CREATE OR REPLACE` for the + whole table); delete-by-key + insert when `key=`; INSERT-only when + `append`. Managed `// materialize` wraps a single-SELECT DuckDB script behind + these templates; `// materialize manual` opts out. +4. **Snapshot + metadata capture** — append `ducklake_snapshots` read, persist + `materialized_partition` rows. +5. **Surface it** — last-materialized/snapshot/row-count on the asset node; + missing-partition set feeds the backfill UI. +6. *v1.x* — snapshot pinning across the cascade (`$WM_UPSTREAM_SNAPSHOT`), + rollback, time-travel read helper. + +Steps 1–5 are a thin annotation+template layer plus one metadata table and one +extra read per run. They deliver managed/incremental/versioned assets, +idempotent partitioned materialization, the backfill substrate, and +materialization observability together — and stay recognizably Windmill-shaped. + +## Open decisions + +These ride on top of the six in `pipelines-vs-dbt.md` §"Decisions either path +forces"; DuckLake-specific: + +1. **Bootstrap of `SET PARTITIONED BY`.** First-materialize detection — table + absent vs. present-but-unpartitioned. Idempotent re-apply. +2. **Snapshot retention / compaction.** DuckLake snapshots accumulate; when do + we expire old ones, and does pinning hold a snapshot alive past retention? +3. **Pin scope.** Pin only direct producers, or the full transitive upstream + set per run? Storage and "stale pin" semantics differ. +4. **Managed multi-statement.** *Resolved:* managed `// materialize` accepts + setup statements (ATTACH/SET/…) followed by exactly one trailing SELECT, and + rejects anything else at deploy with a clear error pointing to + `// materialize manual`. The classifier (`sql_materialize.rs`) is the single + source of truth. diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index 9ec6dd11d3..4a957b2a42 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -280,6 +280,11 @@ introspection per substrate is its own project. Ships A's 80% case first without committing to B's dialect-rewriting tax. Wrapping becomes opt-in convenience for users who want dbt-style ergonomics. +> See [`ducklake-materialization.md`](./ducklake-materialization.md) for the +> DuckLake-native realization of this path: how snapshots make the assets +> versioned/reproducible for free, the executor codegen seam, and the +> materialization-metadata schema. + ### Decisions either path forces 1. **Partition window provenance.** Scheduler tick? Trigger event time diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 707cda66e1..4c4577d02f 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -7,7 +7,7 @@ import { base } from '$lib/base' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { appendViewToken } from '$lib/viewToken' - import { Button, Drawer, DrawerContent } from './common' + import { Badge, Button, Drawer, DrawerContent } from './common' import { ClipboardCopy, Download, @@ -16,8 +16,10 @@ Braces, Highlighter, ArrowDownFromLine, + Database, Loader2 } from 'lucide-svelte' + import DucklakeResultPreview from './assets/AssetGraph/DucklakeResultPreview.svelte' import Portal from '$lib/components/Portal.svelte' import DisplayResultControlBar from './DisplayResultControlBar.svelte' @@ -68,6 +70,7 @@ | 'filename' | 's3object' | 's3object-list' + | 'materialized' | 'plain' | 'markdown' | 'map' @@ -195,6 +198,28 @@ return keys.includes('s3') && typeof result.s3 === 'string' } + // The materialize-run summary — `[{ materialized: 'ducklake://…', rows, + // snapshot_id }]` or the bare object. The shape is narrow (a `ducklake://` + // value plus a `snapshot_id` key) so an ordinary user result isn't hijacked. + function parseMaterializedResult( + res: any + ): + | { materialized: string; partition?: string; rows?: number; snapshot_id?: number | null } + | undefined { + const obj = Array.isArray(res) && res.length === 1 ? res[0] : res + if ( + obj && + typeof obj === 'object' && + typeof obj.materialized === 'string' && + obj.materialized.startsWith('ducklake://') && + 'snapshot_id' in obj + ) { + return obj + } + return undefined + } + + let showMaterializedPreview = $state(true) let is_render_all = $state(false) let download_as_csv = $state(false) function inferResultKind(result: any) { @@ -221,6 +246,13 @@ } try { let keys = result && typeof result === 'object' ? Object.keys(result) : [] + + if (parseMaterializedResult(result)) { + largeObject = false + is_render_all = false + return 'materialized' + } + is_render_all = keys.length == 1 && keys.includes('render_all') && Array.isArray(result['render_all']) @@ -832,6 +864,43 @@ > + {:else if !forceJson && resultKind === 'materialized'} + {@const m = parseMaterializedResult(result)} + {#if m} +
+
+ + {m.materialized} + {#if m.partition} + partition {m.partition} + {/if} + {#if typeof m.rows === 'number'} + + {m.rows} + {m.rows === 1 ? 'row' : 'rows'}{m.partition ? ' in partition' : ''} + + {/if} + {#if m.snapshot_id != null} + snapshot {m.snapshot_id} + {/if} +
+ + {#if showMaterializedPreview} +
+ +
+ {/if} +
+ {/if} {:else if !forceJson && resultKind === 's3object'} {@const s3object = parseS3Object(result) as typeof result}
| undefined, + l: string | undefined, + c: string + ) { + try { + if (l !== 'duckdb' || !s?.properties) return + const part = parsePipelineAnnotations(c).partition + if (!part) return + // Date-based partition kinds render a date / datetime picker; a dynamic + // key is a free-form string. + const format = + part.kind === 'hourly' + ? 'date-time' + : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' + ? 'date' + : undefined + if (!s.properties['partition']) { + s.properties['partition'] = { + type: 'string', + ...(format ? { format } : {}), + // ISO output so partition keys sort lexicographically (the date + // picker defaults to dd-MM-yyyy otherwise). + ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), + description: + part.kind === 'dynamic' + ? 'Partition key value to materialize.' + : `Partition (${part.kind}) to materialize.` + } + if (Array.isArray(s.order) && !s.order.includes('partition')) { + s.order = ['partition', ...s.order] + } + } + // Pre-fill the *test* arg with the current slice for date kinds — a + // convenience default, kept on the args (not baked into the schema, + // where it would persist to the deployed script and go stale). + if (format && a && (a['partition'] == null || a['partition'] === '')) { + const now = new Date() + a['partition'] = + format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) + } + } catch (e) {} + } + async function inferModuleSchema() { if (activeModuleTab === null) return try { await inferArgs(effectiveLang, editorCode, testPanelSchema) + injectPartitionArg(testPanelSchema, testPanelArgs, effectiveLang, editorCode) moduleTestState[activeModuleTab] = { args: testPanelArgs, schema: testPanelSchema } } catch (e) { // Module code may be in-progress; silently ignore diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 39a6a2ae72..6073ee392c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -29,6 +29,7 @@ import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte' import S3FilePreview from '$lib/components/S3FilePreview.svelte' import DataTablePreview from './DataTablePreview.svelte' + import PartitionStatusGrid from './PartitionStatusGrid.svelte' import AssetRunsPanel from './AssetRunsPanel.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' import { fade } from 'svelte/transition' @@ -571,6 +572,12 @@ // deployed script, or a draft promoted from unsaved edits // to a deployed script) chains off it. parent_hash: script.hash ? String(script.hash) : undefined, + // Let the backend resolve the parent to the current head for + // this path (atomically, under an advisory lock) instead of + // rejecting a stale parent_hash with a "lineage must be + // linear" error — the pane is opened from a graph snapshot + // that can fall behind the deployed head between renders. + auto_parent: true, is_template: false, tag: script.tag, kind: script.kind as Script['kind'] | undefined, @@ -976,6 +983,8 @@ class="h-full" refreshKey={previewRefreshKey} /> + {:else if selection.asset_kind === 'ducklake'} + {:else}
No inline preview yet for {selection.asset_kind}. Use the producer/consumer arrows diff --git a/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte b/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte new file mode 100644 index 0000000000..1826ff6b41 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte @@ -0,0 +1,79 @@ + + + open ?? false, (v) => (open = v)} title={`Backfill ${assetPath}`}> +
+ {#if !$enterpriseLicense} +

+ Partition backfill is an enterprise feature. Materializing a single partition is available + in the open-source edition; reprocessing a historical range requires an enterprise license. +

+ {:else} +

+ Re-runs the materialization for each partition in the range. Re-running a partition is + idempotent, so this is safe to repeat. +

+
+
+ From + +
+
+ To + +
+
+ {#if error} +

{error}

+ {/if} + {/if} +
+ + {#snippet actions()} + + + {/snippet} + diff --git a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte new file mode 100644 index 0000000000..403a100aa2 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte @@ -0,0 +1,136 @@ + + +
+ {#if partition} +
+ (scope = e.detail)}> + {#snippet children({ item })} + + + {/snippet} + +
+ {/if} + {#if colDefs.loading && !colDefs.current} +
+ +
+ {:else if !tableColDefs} +
+ + Couldn't load a preview of this table. +
+ {:else if dbTableOps} + + {#key [refreshKey, scope]} +
+ +
+ {/key} + {/if} +
diff --git a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte new file mode 100644 index 0000000000..863130833d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte @@ -0,0 +1,134 @@ + + +
+
+ Materialized partitions +
+ +
+
+ +
+ {#if partitions.loading} +
+ Loading partitions… +
+ {:else if partitions.error} +

Failed to load: {partitions.error.message}

+ {:else if !partitions.current?.length} +

+ No partitions materialized yet. They appear here after a // materialize run. +

+ {:else} +
+ + + + + + + + + + + {#each partitions.current as p (p.partition)} + + + + + + + + {#if p.error} + + {/if} + {/each} + +
PartitionStatusSnapshotRowsMaterialized
{p.partition || '(whole table)'} + + {p.status} + + {p.snapshot_id ?? '—'}{p.row_count ?? '—'}{new Date(p.materialized_at).toLocaleString()}
{p.error}
+ {/if} + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte index 847f682049..e3c6c04378 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte @@ -231,7 +231,7 @@
pathEl, { timeout: 50 }) })} @@ -240,7 +240,7 @@ {#each visibleOutputKinds.length ? visibleOutputKinds : PIPELINE_OUTPUT_KINDS as k} {@const isSelected = selected.outputId === k.id}
+ +
+ {#if schemas.loading} +
+ Loading schema… +
+ {:else if schemas.error} +

Failed to load: {schemas.error.message}

+ {:else if !schemas.current?.length} +

+ No schema captured yet. The output schema is recorded automatically after a // materialize run. +

+ {:else if !canEvolve} + +
+
+ + This asset's schema is fixed — an append / merge / partitioned materialize INSERTs into + a fixed-schema table, so the columns can't change run-to-run. +
+ {#if selected} + {@render columnsTable(selected.columns)} + {/if} +
+ {:else} +
+ + +
+ {#each schemas.current as s, i (s.version)} + + {/each} +
+ + +
+ {#if selected} + {@render columnsTable(selected.columns)} + {/if} +
+ + +
+ {/if} +
+ diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 4a5d16d693..c942d28934 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -33,6 +33,11 @@ export interface AssetGraphRunnableNode { // asset. Surfaced as a count badge (with a per-test breakdown in the title) // so test coverage is visible on the node without opening the pane. data_tests?: DataTest[] + // Managed `// materialize` write strategy. Absent for non-materializing or + // `manual` scripts. Used (with `partition_kind`) to decide whether a + // produced asset's schema can evolve: only whole-table `replace` can, since + // `append`/`merge`/partitioned writes INSERT into a fixed-schema table. + materialize_strategy?: 'replace' | 'append' | 'merge' // Synthesized by the page from a local draft; the script doesn't exist // in the DB yet. Drives a dashed/lower-opacity rendering to mirror how // unsaved triggers are styled — visually distinct from persisted nodes. diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index e7f52ddbd4..3548ba6017 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -1958,6 +1958,27 @@ .map((e) => ({ kind: e.runnable_kind, path: e.runnable_path, unsaved: e.unsaved })) }) + // Whether the selected ducklake asset's captured schema can *evolve* (drives + // the asset panel's Schema tab: version history vs. a single fixed schema). + // Only a whole-table `replace` producer (CREATE OR REPLACE) can change + // columns run-to-run; `append`/`merge`/partitioned writes INSERT into a + // fixed-schema table, so their schema is pinned at first materialize. + // + // Fail open: show the fixed view only when we're *sure* — every producer is a + // known insert-style write. A producer with no `materialize_strategy` + // metadata (e.g. a draft-overlay runnable, which the graph synthesizes + // without it) is treated as unknown → evolvable, so captured history is never + // hidden behind a stale "fixed" verdict. + let schemaCanEvolve = $derived.by(() => { + const sel = selection + if (!sel || sel.kind !== 'asset' || sel.asset_kind !== 'ducklake') return true + const producerPaths = new Set(selectionProducers.map((p) => p.path)) + const producers = graphWithDraft.runnables.filter((r) => producerPaths.has(r.path)) + const knownFixed = (r: (typeof producers)[number]) => + !!r.materialize_strategy && !(r.materialize_strategy === 'replace' && !r.partition_kind) + return producers.length === 0 || !producers.every(knownFixed) + }) + // Downstream subscriber count for the currently-edited script. Drives // the Test button's cascade UX: when > 0, ScriptEditor renders a split // button exposing "just this step" (default, with `_wmill_skip_asset_dispatch`) @@ -2531,6 +2552,7 @@ onRunByPath={runByPathLegit} selection={activeDraft ? undefined : selection} selectionProducers={activeDraft ? [] : selectionProducers} + {schemaCanEvolve} {runsRefreshKey} {runsPendingJobId} {activeRunnable} From 577ceeee8679f054c6898d1a7889df30ab830f8f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 26 Jun 2026 21:37:33 +0200 Subject: [PATCH 114/273] perf(audit): re-anchor S3 audit export on enable + opt-in backfill (#9818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [ee] perf(audit): re-anchor S3 audit export on enable + opt-in backfill The S3/GCS audit-log export's steady-state query filters by `age(xmin)` (unindexable), so the only scan bound is the timestamp floor. On a fresh enable the floor was epoch, and on a re-enable the cursor resumed from its pre-disable position — either way the first run scanned the whole `audit_partitioned` table. Under a `statement_timeout` (e.g. Aiven) that scan never completes: the cursor never advances, nothing is exported, and the repeated full scans saturate the database. Re-anchor on enable (EE companion, windmill-ee-private#634): - New trigger migration records a recent timestamp floor instead of the epoch sentinel and `DO UPDATE`s the cursor to the current snapshot xmin on re-enable, so the export always resumes from ~now and never rescans history. Includes a one-time fixup for legacy epoch-sentinel checkpoints on upgrade. Opt-in historical backfill (new `audit_logs_s3_backfill` module + endpoints): - Exports a chosen `[from, to)` window on demand, scanning strictly by `timestamp` (the partition key) in bounded keyset pages — each query is an index scan capped at one page (verified via EXPLAIN: later partitions `never executed`, ~11ms/page), so it stays well under any statement timeout regardless of window size. Writes alongside the steady-state objects under logs/audit/, without touching the xmin cursor. - POST /settings/audit_logs_s3_backfill {from,to} (super-admin + Enterprise), GET /settings/audit_logs_s3_backfill_status. Also repurposes the status endpoint's `bootstrapping` flag to mean "draining a backlog" (the cursor is capped and catching up), and updates the setting description to point operators at the backfill. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(audit): heartbeat backfill lease per object; bump EE ref Address review (cubic): persist progress (refreshing the lease heartbeat) after every object PUT in the backfill page loop, not only once per page, so the gap between heartbeats stays well under STALE_HEARTBEAT_SECS even on slow uploads and another replica can't re-claim mid-page and run a concurrent backfill. Bumps ee-repo-ref.txt to pull in the EE test-race fix (folding the backlog-drain regression into the single audit e2e test). Co-Authored-By: Claude Opus 4.8 (1M context) * fix(audit): reject unstable backfill windows; bump EE ref Address review (P1): the backfill keyset-pages over rows visible at scan time and declares completion when the scan runs dry, but a row's `timestamp` is its inserting transaction's `xact_start`. A window whose upper bound is recent or in the future could silently omit a transaction that started inside `[from, to)` but commits after the scan passed that timestamp. `try_start` now rejects any `to` newer than the oldest in-flight `xact_start` (everything strictly older than the oldest running transaction is committed and stable), using the same trustworthy stats gating as the exporter's floor (restricted role / 2PC → a 7-day-old cutoff). Bumps ee-repo-ref.txt for the EE monotonic-checkpoint fix. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(audit): re-anchor legacy epoch checkpoints instead of synthetic floor Address review (P1): the legacy-checkpoint fixup stamped last_oldest_inflight_ts to now()-7d while leaving the old last_xmin in place. On an instance that enabled export on the old code >7 days ago and got stuck before the first successful batch, the next run would filter post-enable rows older than 7 days out via `timestamp >= ts_floor` while still advancing last_xmin over the interval — silently dropping them (the same floor-vs-cursor loss class fixed elsewhere in this PR), and contradicting the "nothing committed after enabling is skipped" guarantee. A stuck epoch-sentinel checkpoint cannot be safely resumed (its backlog can be arbitrarily old, so any recent floor prunes rows the cursor then skips, and an epoch floor reintroduces the full scan). Re-anchor it to the migration's current snapshot xmin instead — exactly like a fresh enable — so the export resumes cleanly from ~now and the never-exported pre-upgrade window is recovered via the opt-in backfill rather than silently dropped. Reword the setting description so it no longer implies the disabled/legacy window is covered by the cursor. Co-Authored-By: Claude Opus 4.8 (1M context) * test(audit): end-to-end integration tests for the object-store backfill The backfill previously had only SQL-level/EXPLAIN validation. Add real integration tests (in-memory object store, sqlx::test) exercising the public path: - backfill_exports_window_in_pages: with the page size forced to 2 rows, a settled 3-day window is exported across multiple keyset pages; asserts every in-window row lands exactly once, rows outside [from,to) are excluded, a day that straddles a page boundary yields more than one object, progress counts match, and a re-run is idempotent (deterministic keys overwritten, no dupes). - backfill_rejects_unstable_window: a future/live `to` is rejected as unstable, a window safely in the past is accepted. Adds a test-only PAGE_ROWS override so multi-page behaviour is exercised with a handful of rows. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(audit): note backfill scope is audit_partitioned only Make explicit that, like the steady-state export, the backfill reads only audit_partitioned; the pre-partitioning `audit` table is intentionally out of scope (not a missed case). Co-Authored-By: Claude Opus 4.8 (1M context) * fix(audit): reject backfill windows before the partitioned boundary Address review (Codex P1): the backfill reads only audit_partitioned, but pre-partitioning history lives in the legacy `audit` table (still read by audit list/get via UNION ALL, and retained for the configured period — 365 days by default on EE). Since the setting text points operators at this API for "pre-existing history", a window overlapping legacy rows would report completion while silently omitting them. Per the decision to not export the legacy table, reject instead of silently omit: try_start now rejects a `from` earlier than the oldest audit_partitioned timestamp (every legacy row predates the partition cutover, so a `from` at/after that boundary can never overlap them). Reworded the setting text to scope the backfill to the partitioned era. Added a regression test, plus an RAII guard (cubic P2) so the test-only globals are restored even if an assertion panics. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(audit): backfill object keys per-window; require trustworthy settled cutoff Address review (two P1s): - Object-key overwrite loss: keys were `dt=/audit_backfill_.ndjson`. A narrower, overlapping backfill can start a day's page at the same first row (same min_id) but hold fewer rows, and `put` would overwrite a broader run's object — silently dropping the rows only that object held. Include the requested window in the key so different ranges write disjoint objects (same window re-runs stay idempotent; consumers dedupe overlapping rows by id). New regression test (verified red→green). - Untrustworthy settled cutoff: when min(xact_start) isn't trustworthy (role lacks pg_read_all_stats/superuser, or a prepared 2PC txn exists), the old now()-7d fallback could still let an old transaction commit rows inside an accepted window after the scan, so a "complete" backfill silently missed them. Since a backfill asserts completeness, reject in those cases instead of falling back. (The continuous exporter keeps its 7-day fallback — it only claims bounded lag.) Also makes the tests robust under the parallel runner: run_backfill takes the store as a param, so tests pass a local in-memory store (no global OBJECT_STORE_SETTINGS race) and serialize on the PAGE_ROWS override. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(audit): reject backfill overlapping legacy table; regen deref openapi; trim migration comment Address review (1 P1 + 2 P2): - Empty-partition backfill (P1): the min(audit_partitioned) guard no-ops when audit_partitioned is empty, so an upgraded instance with legacy `audit` rows but no partitioned rows yet would accept a window and complete with zero rows, silently omitting the legacy rows. Check the legacy `audit` table directly: reject any window that overlaps a legacy row (subsumes the boundary check and covers the empty-partitioned case). Test updated accordingly. - openapi-deref (P2): regenerate openapi-deref.yaml/json (served via include_str!) so /openapi.{yaml,json} expose the new backfill endpoints. - Migration comment (P2): trim the PR-history narration to the durable constraints (why a recent floor and a monotonic cursor are required), per AGENTS.md. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to b821fecccbcba2efed544890576bf2b84321d70d This commit updates the EE repository reference after PR #634 was merged in windmill-ee-private. Previous ee-repo-ref: 6b191b77aabcf77658ad4f9031576e0d7b66bf89 New ee-repo-ref: b821fecccbcba2efed544890576bf2b84321d70d Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...100e926bb678171947e71d6d87fdd0c3f9299.json | 15 + ...b64b33c722afde7457912e8abd5743de8829.json} | 7 +- ...f172c89da5b216d3530c2743319ab1f4ca853.json | 20 + ...790770c32716ef953a2ea4af17d122d0a3b3c.json | 44 ++ backend/ee-repo-ref.txt | 2 +- ..._audit_logs_s3_reanchor_on_enable.down.sql | 19 + ...51_audit_logs_s3_reanchor_on_enable.up.sql | 70 ++ .../src/audit_logs_s3.rs | 9 +- .../src/audit_logs_s3_backfill.rs | 709 ++++++++++++++++++ backend/windmill-api-settings/src/lib.rs | 35 +- backend/windmill-api/openapi-deref.json | 211 +++++- backend/windmill-api/openapi-deref.yaml | 158 +++- backend/windmill-api/openapi.yaml | 86 +++ .../src/lib/components/instanceSettings.ts | 2 +- 14 files changed, 1375 insertions(+), 12 deletions(-) create mode 100644 backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json rename backend/.sqlx/{query-8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d.json => query-881d996af5aaa1ec01693e473519b64b33c722afde7457912e8abd5743de8829.json} (68%) create mode 100644 backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json create mode 100644 backend/.sqlx/query-fd023a9365388f1f74423416bd8790770c32716ef953a2ea4af17d122d0a3b3c.json create mode 100644 backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql create mode 100644 backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql create mode 100644 backend/windmill-api-settings/src/audit_logs_s3_backfill.rs diff --git a/backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json b/backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json new file mode 100644 index 0000000000..54db7ff310 --- /dev/null +++ b/backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', now(),\n 'last_oldest_inflight_ts', COALESCE(\n CASE WHEN (current_setting('is_superuser') = 'on'\n OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE'))\n AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts)\n THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL)\n ELSE NULL END,\n now() - interval '7 days'))\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299" +} diff --git a/backend/.sqlx/query-8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d.json b/backend/.sqlx/query-881d996af5aaa1ec01693e473519b64b33c722afde7457912e8abd5743de8829.json similarity index 68% rename from backend/.sqlx/query-8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d.json rename to backend/.sqlx/query-881d996af5aaa1ec01693e473519b64b33c722afde7457912e8abd5743de8829.json index 56b887e8b9..f8f8511709 100644 --- a/backend/.sqlx/query-8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d.json +++ b/backend/.sqlx/query-881d996af5aaa1ec01693e473519b64b33c722afde7457912e8abd5743de8829.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO background_task_state\n (name, value, running, owner, started_at, finished_at, updated_at)\n VALUES ($1, $2, false, $3, now(), now(), now())\n ON CONFLICT (name) DO UPDATE SET\n value = $2, running = false, owner = $3,\n finished_at = now(), updated_at = now()", + "query": "INSERT INTO background_task_state\n (name, value, running, owner, started_at, finished_at, updated_at)\n VALUES ($1, $2, false, $3, now(), now(), now())\n ON CONFLICT (name) DO UPDATE SET\n value = $2, running = false, owner = $3,\n finished_at = now(), updated_at = now()\n WHERE (background_task_state.value->>'last_xmin')::bigint <= $4", "describe": { "columns": [], "parameters": { "Left": [ "Text", "Jsonb", - "Text" + "Text", + "Int8" ] }, "nullable": [] }, - "hash": "8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d" + "hash": "881d996af5aaa1ec01693e473519b64b33c722afde7457912e8abd5743de8829" } diff --git a/backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json b/backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json new file mode 100644 index 0000000000..4c0d0ae14f --- /dev/null +++ b/backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT CASE WHEN (current_setting('is_superuser') = 'on'\n OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE'))\n AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts)\n THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL)\n ELSE NULL END AS \"cutoff?\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cutoff?", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853" +} diff --git a/backend/.sqlx/query-fd023a9365388f1f74423416bd8790770c32716ef953a2ea4af17d122d0a3b3c.json b/backend/.sqlx/query-fd023a9365388f1f74423416bd8790770c32716ef953a2ea4af17d122d0a3b3c.json new file mode 100644 index 0000000000..5ac11c6a5e --- /dev/null +++ b/backend/.sqlx/query-fd023a9365388f1f74423416bd8790770c32716ef953a2ea4af17d122d0a3b3c.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT to_char(timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS \"day!\",\n id AS \"id!\",\n timestamp AS \"ts!\",\n row_to_json(r)::text AS \"line!\"\n FROM (\n SELECT workspace_id, id, timestamp, username, operation,\n action_kind::text AS action_kind, resource, parameters, email, span\n FROM audit_partitioned\n WHERE timestamp >= $1 AND timestamp < $2\n AND (timestamp, id) > ($3, $4)\n ORDER BY timestamp, id\n LIMIT $5\n ) r\n ORDER BY timestamp, id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "day!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "id!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "ts!", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "line!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Timestamptz", + "Timestamptz", + "Timestamptz", + "Int8", + "Int8" + ] + }, + "nullable": [ + null, + false, + false, + null + ] + }, + "hash": "fd023a9365388f1f74423416bd8790770c32716ef953a2ea4af17d122d0a3b3c" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 85d87fe2ca..294707d466 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -7b92c8e0de4cfc6d986499d60a5f79cd1c6b9d0b +b821fecccbcba2efed544890576bf2b84321d70d diff --git a/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql new file mode 100644 index 0000000000..2c9e78659e --- /dev/null +++ b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql @@ -0,0 +1,19 @@ +-- Restore the previous anchor: epoch sentinel + preserve-cursor (DO NOTHING). +CREATE OR REPLACE FUNCTION audit_logs_s3_anchor_on_enable() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.value = to_jsonb(true) + AND (TG_OP = 'INSERT' OR OLD.value IS DISTINCT FROM NEW.value) THEN + INSERT INTO background_task_state (name, value) + VALUES ( + 'audit_logs_s3_export', + jsonb_build_object( + 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint, + 'last_ts', '1970-01-01T00:00:00+00:00' + ) + ) + ON CONFLICT (name) DO NOTHING; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql new file mode 100644 index 0000000000..3fd759da9e --- /dev/null +++ b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql @@ -0,0 +1,70 @@ +-- Anchors the audit→object-store export cursor when the setting is enabled. +-- +-- `last_ts`/`last_oldest_inflight_ts` must be a *recent* floor, not epoch: the +-- export's `timestamp >= floor` predicate is the only partition-pruning bound (the +-- `age(xmin)` cursor is unindexable), so an epoch floor would scan the whole +-- `audit_partitioned` table on the first run and never finish under a +-- `statement_timeout`. The floor must be at or below the timestamp of any row whose +-- xid >= this snapshot xmin; the oldest in-flight `xact_start` is that bound when +-- stats are visible (no restricted role / prepared 2PC txn), else a bounded 7-day +-- window. +-- +-- `ON CONFLICT DO UPDATE ... WHERE last_xmin <` keeps the cursor monotonic: a +-- re-enable re-anchors it forward (so the export resumes from ~now rather than +-- rescanning the disabled gap — that gap is the backfill's job), but it never moves +-- backwards, so it is HA-safe and can't be regressed by a slower concurrent writer. +-- +-- The task name literal must match +-- `windmill_common::global_settings::AUDIT_LOGS_S3_EXPORT_TASK`. + +CREATE OR REPLACE FUNCTION audit_logs_s3_anchor_on_enable() +RETURNS TRIGGER AS $$ +DECLARE + v_floor timestamptz; +BEGIN + IF NEW.value = to_jsonb(true) + AND (TG_OP = 'INSERT' OR OLD.value IS DISTINCT FROM NEW.value) THEN + v_floor := COALESCE( + CASE WHEN (current_setting('is_superuser') = 'on' + OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')) + AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) + THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL) + ELSE NULL END, + now() - interval '7 days'); + INSERT INTO background_task_state (name, value) + VALUES ( + 'audit_logs_s3_export', + jsonb_build_object( + 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint, + 'last_ts', now(), + 'last_oldest_inflight_ts', v_floor + ) + ) + ON CONFLICT (name) DO UPDATE + SET value = EXCLUDED.value + WHERE (background_task_state.value->>'last_xmin')::bigint + < (EXCLUDED.value->>'last_xmin')::bigint; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Recovery for a legacy epoch-sentinel checkpoint (`last_ts = epoch`, never +-- drained). It cannot be safely resumed: its un-drained backlog can be arbitrarily +-- old, so stamping a recent floor over the old xmin would prune the older rows while +-- the cursor advanced past them (silent loss), and keeping the epoch floor would +-- reintroduce the full scan. Re-anchor it to now like a fresh enable; the pre-anchor +-- window is recoverable via the opt-in backfill, not silently dropped. +UPDATE background_task_state +SET value = jsonb_build_object( + 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint, + 'last_ts', to_jsonb(now()), + 'last_oldest_inflight_ts', to_jsonb(COALESCE( + CASE WHEN (current_setting('is_superuser') = 'on' + OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')) + AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) + THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL) + ELSE NULL END, + now() - interval '7 days'))) +WHERE name = 'audit_logs_s3_export' + AND (value->>'last_ts')::timestamptz <= 'epoch'::timestamptz; diff --git a/backend/windmill-api-settings/src/audit_logs_s3.rs b/backend/windmill-api-settings/src/audit_logs_s3.rs index 9fa4574a1e..d7d749cb58 100644 --- a/backend/windmill-api-settings/src/audit_logs_s3.rs +++ b/backend/windmill-api-settings/src/audit_logs_s3.rs @@ -20,9 +20,11 @@ use windmill_common::DB; pub struct AuditLogsS3ExportStatus { /// xid cursor: rows of transactions below this have been exported. pub last_xmin: i64, - /// Partition-pruning floor (the epoch sentinel while still bootstrapping). + /// Partition-pruning floor: the latest audit-row timestamp the cursor has + /// reached (also the read side's 7-day-fallback anchor). pub last_ts: Option>, - /// True until the initial post-enable backlog has been fully drained. + /// True while the exporter is draining a backlog — the last run was capped + /// at `MAX_XID_INTERVAL` xids and has not yet caught up to the live snapshot. pub bootstrapping: bool, /// The latest audit-row timestamp actually written to object storage so /// far (monotonic) — the "how current is the mirror" figure. @@ -50,8 +52,7 @@ pub async fn get_status(db: &DB) -> error::Result::from_timestamp(0, 0).unwrap(); - let bootstrapping = last_ts.map(|t| t <= epoch).unwrap_or(true); + let bootstrapping = v.get("draining").and_then(|x| x.as_bool()).unwrap_or(false); Ok(Some(AuditLogsS3ExportStatus { last_xmin: v.get("last_xmin").and_then(|x| x.as_i64()).unwrap_or(0), last_ts, diff --git a/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs b/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs new file mode 100644 index 0000000000..e86ceb6448 --- /dev/null +++ b/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs @@ -0,0 +1,709 @@ +#![cfg(feature = "parquet")] +//! Opt-in historical backfill of audit logs to the instance object store. +//! +//! The steady-state exporter (the EE `export_audit_logs_to_object_store`) cursors +//! on transaction xmin and, by design, only exports rows committed *after* the +//! feature was enabled — it never rescans history (an `age(xmin)` predicate is +//! unindexable, so scanning the whole partitioned table can't survive a +//! `statement_timeout`). This module covers the complementary need: exporting a +//! chosen historical `[from, to)` window (e.g. the gap left while the export was +//! disabled) on demand. +//! +//! It is safe to run on a large table because it scans strictly by `timestamp` +//! (the partition key — pruned and indexed) in bounded keyset pages, so every +//! query touches at most one page worth of rows and survives a statement timeout. +//! It does not touch the xmin cursor / checkpoint at all. +//! +//! Objects are written next to the steady-state ones under `logs/audit/dt=/` +//! as `audit_backfill__.ndjson`, with the exact same row shape, so +//! a consumer reads them uniformly. The key includes the requested window so two +//! different backfill ranges never overwrite each other (a per-page `min_id` alone +//! is not unique across windows). Re-running the *same* window is deterministic +//! (audit history is append-only), so it overwrites the same objects rather than +//! duplicating. A window that overlaps already-exported steady-state rows simply +//! re-emits them under a different key; consumers dedupe by `id`. +//! +//! Scope: like the steady-state export, this reads only `audit_partitioned`. The +//! pre-partitioning `audit` table is intentionally not exported; a window that +//! overlaps any legacy `audit` row is rejected (see [`try_start`]) so a backfill +//! never silently reports success while omitting them. +//! +//! Progress is persisted in `background_task_state` (name [`TASK_NAME`]) so any +//! API replica can serve the status endpoint, mirroring `log_cleanup`. + +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; + +use crate::background_task; +use windmill_common::error::{self}; +use windmill_common::tracing_init::LOGS_AUDIT; +use windmill_common::{DB, INSTANCE_NAME}; + +use windmill_object_store::object_store_reexports::{ObjectStore, Path as ObjectPath}; + +pub const TASK_NAME: &str = "audit_logs_s3_backfill"; + +/// Rows fetched per keyset page. Bounds each query so it stays well under any +/// `statement_timeout` even on a busy partition, and bounds peak memory (one +/// page of ndjson is buffered before the day-grouped PUTs). +const PAGE_ROWS: i64 = 10_000; + +/// Test-only override for [`PAGE_ROWS`] (0 = use the default), so a test can force +/// multi-page / page-spanning-day keyset behaviour with only a handful of rows. +#[cfg(test)] +static PAGE_ROWS_OVERRIDE: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +fn page_rows() -> i64 { + #[cfg(test)] + { + match PAGE_ROWS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) { + 0 => PAGE_ROWS, + n => n, + } + } + #[cfg(not(test))] + { + PAGE_ROWS + } +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct AuditBackfillProgress { + pub running: bool, + pub started_at: DateTime, + pub finished_at: Option>, + /// Human-readable description of the current phase. + pub phase: String, + /// Inclusive lower / exclusive upper bound of the window being exported. + pub from: DateTime, + pub to: DateTime, + /// Audit rows written to object storage so far. + pub rows_written: u64, + /// Object PUTs issued so far (one per day per page). + pub objects_written: u64, + /// Keyset cursor: the timestamp of the last row exported (how far the + /// backfill has progressed through the window). + pub last_ts: Option>, + pub errors: u64, + pub last_error: Option, +} + +impl AuditBackfillProgress { + fn new_running(from: DateTime, to: DateTime) -> Self { + Self { + running: true, + started_at: Utc::now(), + finished_at: None, + phase: "starting".to_string(), + from, + to, + rows_written: 0, + objects_written: 0, + last_ts: None, + errors: 0, + last_error: None, + } + } +} + +struct Session { + db: DB, + owner: String, + progress: RwLock, +} + +impl Session { + async fn update(&self, f: F) { + let snapshot = { + let mut p = self.progress.write().await; + f(&mut p); + p.clone() + }; + if let Err(e) = + background_task::update_state(&self.db, TASK_NAME, &self.owner, &snapshot).await + { + tracing::warn!("audit backfill: failed to persist progress: {e:#}"); + } + } + + async fn record_error(&self, msg: String) { + tracing::error!("audit backfill: {msg}"); + self.update(|p| { + p.errors = p.errors.saturating_add(1); + p.last_error = Some(msg); + }) + .await; + } + + async fn release(&self) { + let snapshot = { + let mut p = self.progress.write().await; + p.running = false; + p.finished_at = Some(Utc::now()); + p.phase = "done".to_string(); + p.clone() + }; + tracing::info!( + "audit backfill finished: {} row(s) in {} object(s) for [{}, {}), {} error(s)", + snapshot.rows_written, + snapshot.objects_written, + snapshot.from, + snapshot.to, + snapshot.errors + ); + if let Err(e) = background_task::release(&self.db, TASK_NAME, &self.owner, &snapshot).await + { + tracing::warn!("audit backfill: failed to release lease: {e:#}"); + } + } +} + +#[derive(Deserialize)] +pub struct BackfillRequest { + pub from: DateTime, + pub to: DateTime, +} + +/// Atomically claim the backfill lease, or error if one is already running. +pub async fn try_start(db: &DB, from: DateTime, to: DateTime) -> error::Result<()> { + if from >= to { + return Err(error::Error::BadRequest( + "audit backfill: `from` must be strictly before `to`".to_string(), + )); + } + // The backfill keyset-pages by `(timestamp, id)` over rows visible at scan time and + // declares the window fully exported once the scan runs dry. But a row's `timestamp` + // is its inserting transaction's `xact_start`, so a transaction that started inside + // `[from, to)` yet commits after the scan has passed that timestamp — or any row + // committed when `to` is in the future — would be silently omitted. Require `to` to + // be at or before the oldest in-flight `xact_start`: everything strictly older than + // the oldest running transaction is already committed and stable. + // + // That bound is only sound when we can see every xmin-holding transaction. A role + // without pg_read_all_stats/superuser sees only its own sessions, and a prepared + // (2PC) transaction is invisible to pg_stat_activity — in either case an old + // transaction could still commit rows inside an accepted window after our scan ends. + // Since a backfill asserts completeness, we REJECT in those cases rather than fall + // back to a best-effort margin (NULL below). (The continuous exporter, which only + // claims bounded lag, keeps the 7-day fallback instead.) + let settled_cutoff: Option> = sqlx::query_scalar!( + r#"SELECT CASE WHEN (current_setting('is_superuser') = 'on' + OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')) + AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) + THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL) + ELSE NULL END AS "cutoff?""# + ) + .fetch_one(db) + .await?; + let Some(settled_cutoff) = settled_cutoff else { + return Err(error::Error::BadRequest( + "audit backfill: cannot determine a trustworthy settled-time boundary, so completeness \ + can't be guaranteed. The windmill DB role needs pg_read_all_stats (or superuser) and \ + there must be no prepared (2PC) transactions in progress — otherwise an old or \ + invisible transaction could later commit audit rows inside the requested window and \ + the backfill would miss them. Grant the privilege / resolve prepared transactions and \ + retry." + .to_string(), + )); + }; + if to > settled_cutoff { + return Err(error::Error::BadRequest(format!( + "audit backfill: `to` ({to}) must be at or before {settled_cutoff}, the newest point \ + guaranteed settled (the oldest in-flight transaction's start); choose an earlier \ + upper bound." + ))); + } + // The backfill (like the steady-state export) reads only `audit_partitioned`. Audit + // history from before partitioning was introduced lives in the legacy `audit` table + // and is intentionally not exported. If the requested window overlaps any legacy row, + // reject — otherwise a "completed" backfill would silently omit them. Checking the + // legacy table directly (rather than min(audit_partitioned)) also covers an upgraded + // instance whose `audit_partitioned` is still empty, where a min() guard would no-op. + // Non-macro query: no compile-time-checked entry needed. + let overlaps_legacy: bool = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM audit WHERE timestamp >= $1 AND timestamp < $2)", + ) + .bind(from) + .bind(to) + .fetch_one(db) + .await?; + if overlaps_legacy { + return Err(error::Error::BadRequest( + "audit backfill: the requested window overlaps rows in the legacy (pre-partitioning) \ + `audit` table, which is not exported to object storage. Restrict the window to the \ + partitioned era (after audit-log partitioning was introduced)." + .to_string(), + )); + } + let claimed = background_task::try_claim( + db, + TASK_NAME, + &*INSTANCE_NAME, + &AuditBackfillProgress::new_running(from, to), + ) + .await?; + if !claimed { + return Err(error::Error::BadRequest( + "An audit log backfill is already running".to_string(), + )); + } + Ok(()) +} + +/// Fetch the current backfill status. Any API server can call this. +pub async fn get_status(db: &DB) -> error::Result> { + let Some(r) = background_task::get(db, TASK_NAME).await? else { + return Ok(None); + }; + match serde_json::from_value::(r.value) { + Ok(mut p) => { + // get() collapses `running` to false when the heartbeat is stale. + p.running = r.running; + Ok(Some(p)) + } + Err(e) => Err(error::Error::internal_err(format!( + "deserialize audit backfill progress: {e:#}" + ))), + } +} + +pub fn spawn_backfill(db: DB, from: DateTime, to: DateTime) { + use futures::FutureExt; + use std::panic::AssertUnwindSafe; + + tokio::spawn(async move { + let session = Arc::new(Session { + db: db.clone(), + owner: INSTANCE_NAME.clone(), + progress: RwLock::new(AuditBackfillProgress::new_running(from, to)), + }); + + let s = session.clone(); + let task = async move { + let store = match windmill_object_store::get_object_store().await { + Some(st) => st, + None => { + s.record_error("Object storage is not configured".to_string()) + .await; + return; + } + }; + if let Err(e) = run_backfill(&s, &db, &store, from, to).await { + s.record_error(format!("backfill failed: {e:#}")).await; + } + }; + + // catch_unwind so a panic can't leave the lease held forever. + if let Err(panic) = AssertUnwindSafe(task).catch_unwind().await { + let msg = panic + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| panic.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()); + session + .record_error(format!("backfill task panicked: {msg}")) + .await; + } + + session.release().await; + }); +} + +/// Export `[from, to)` in keyset pages ordered by `(timestamp, id)`. Each page is +/// a bounded, partition-pruned scan; rows are grouped by UTC day and written one +/// object per day per page. +async fn run_backfill( + session: &Session, + db: &DB, + store: &Arc, + from: DateTime, + to: DateTime, +) -> error::Result<()> { + session.update(|p| p.phase = "exporting".to_string()).await; + + // Keyset cursor over (timestamp, id). `id` starts below any real value so the + // first page includes rows at exactly `from`. + let mut cursor_ts = from; + let mut cursor_id: i64 = -1; + let page_rows = page_rows(); + // Namespace object keys by the requested window. The per-page `min_id` alone is not + // unique across runs: a narrower, overlapping backfill can start a day's page at the + // same first row (same `min_id`) but contain fewer rows, and `put` would overwrite a + // broader run's object — silently dropping the rows only that object held. Including + // the window makes different ranges write disjoint keys (same window re-runs stay + // idempotent); consumers already dedupe overlapping rows by `id`. + let window_key = format!("{}_{}", from.timestamp_millis(), to.timestamp_millis()); + + loop { + let rows = sqlx::query!( + r#"SELECT to_char(timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS "day!", + id AS "id!", + timestamp AS "ts!", + row_to_json(r)::text AS "line!" + FROM ( + SELECT workspace_id, id, timestamp, username, operation, + action_kind::text AS action_kind, resource, parameters, email, span + FROM audit_partitioned + WHERE timestamp >= $1 AND timestamp < $2 + AND (timestamp, id) > ($3, $4) + ORDER BY timestamp, id + LIMIT $5 + ) r + ORDER BY timestamp, id"#, + from, + to, + cursor_ts, + cursor_id, + page_rows + ) + .fetch_all(db) + .await?; + + if rows.is_empty() { + break; + } + + // Group this page's ndjson lines by day, preserving (timestamp, id) order, + // and track the min id per day for a deterministic, collision-free key. + let mut by_day: Vec<(String, i64, String)> = Vec::new(); // (day, min_id, ndjson) + for row in &rows { + match by_day.last_mut() { + Some((day, _min_id, acc)) if *day == row.day => { + acc.push('\n'); + acc.push_str(&row.line); + } + _ => by_day.push((row.day.clone(), row.id, row.line.clone())), + } + } + + for (day, min_id, ndjson) in &by_day { + let object_path = ObjectPath::from(format!( + "{LOGS_AUDIT}dt={day}/audit_backfill_{window_key}_{min_id}.ndjson" + )); + store + .put(&object_path, ndjson.clone().into_bytes().into()) + .await + .map_err(|e| error::Error::internal_err(format!("upload {object_path}: {e:#}")))?; + let n = ndjson.lines().count() as u64; + // Persist progress (and refresh the lease heartbeat) after every object, + // not just once the page completes: a stale heartbeat lets another replica + // re-claim the lease and run a concurrent backfill, so the gap between + // heartbeats must stay well under STALE_HEARTBEAT_SECS even if a page's + // uploads are slow. + session + .update(|p| { + p.rows_written = p.rows_written.saturating_add(n); + p.objects_written = p.objects_written.saturating_add(1); + }) + .await; + } + + // Advance the keyset cursor past the last row of this page. + let last = rows.last().expect("page is non-empty"); + cursor_ts = last.ts; + cursor_id = last.id; + + let new_last_ts = last.ts; + session.update(|p| p.last_ts = Some(new_last_ts)).await; + + // A short page means the window is exhausted. + if (rows.len() as i64) < page_rows { + break; + } + } + + Ok(()) +} + +#[cfg(all(test, feature = "parquet"))] +mod tests { + use super::*; + use futures::stream::StreamExt; + use std::sync::atomic::Ordering; + use std::sync::Arc; + use windmill_object_store::object_store_reexports::{InMemory, ObjectStore, Path as OsPath}; + + /// A private, per-test object store. `run_backfill` takes the store as a parameter, + /// so tests use a local one and never touch the process-global `OBJECT_STORE_SETTINGS` + /// (which would otherwise race across the parallel test runner). + fn local_store() -> (Arc, Arc) { + let store = Arc::new(InMemory::new()); + let dynstore: Arc = store.clone(); + (store, dynstore) + } + + /// Serializes the tests that touch the `PAGE_ROWS_OVERRIDE` process global (read + /// inside `run_backfill`) so they can't observe each other's value under the parallel + /// runner. + static PAGE_OVERRIDE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + /// Resets `PAGE_ROWS_OVERRIDE` on drop so a failing assertion can't leak a non-default + /// page size into another test. + struct ResetPageOverride; + impl Drop for ResetPageOverride { + fn drop(&mut self) { + PAGE_ROWS_OVERRIDE.store(0, Ordering::Relaxed); + } + } + + /// Insert an audit row `days` days in the past (creating the daily partition if + /// needed). The row's `timestamp` defaults to that point, landing it in the + /// matching partition. + async fn insert_audit_days_ago(db: &DB, operation: &str, days: i64) -> i64 { + sqlx::query(&format!( + "DO $$ DECLARE d date := current_date - {days}; BEGIN \ + EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_partitioned \ + FOR VALUES FROM (%L) TO (%L)', 'audit_'||to_char(d,'YYYYMMDD'), d, d + 1); END $$;" + )) + .execute(db) + .await + .ok(); + sqlx::query_scalar::<_, i64>(&format!( + "INSERT INTO audit_partitioned + (workspace_id, username, operation, action_kind, parameters, timestamp) + VALUES ('test-ws','tester',$1,'create'::action_kind,'{{}}'::jsonb, + now() - interval '{days} days') + RETURNING id" + )) + .bind(operation) + .fetch_one(db) + .await + .expect("insert audit row") + } + + /// Insert an audit row at an exact timestamp (creating the daily partition if + /// needed), for tests that need distinct in-day timestamps. + async fn insert_audit_at(db: &DB, operation: &str, ts: DateTime) -> i64 { + sqlx::query(&format!( + "DO $$ DECLARE d date := '{}'; BEGIN \ + EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_partitioned \ + FOR VALUES FROM (%L) TO (%L)', 'audit_'||to_char(d,'YYYYMMDD'), d, d + 1); END $$;", + ts.format("%Y-%m-%d") + )) + .execute(db) + .await + .ok(); + sqlx::query_scalar::<_, i64>( + "INSERT INTO audit_partitioned + (workspace_id, username, operation, action_kind, parameters, timestamp) + VALUES ('test-ws','tester',$1,'create'::action_kind,'{}'::jsonb,$2) + RETURNING id", + ) + .bind(operation) + .bind(ts) + .fetch_one(db) + .await + .expect("insert audit row") + } + + /// All ids across every `audit_backfill_*.ndjson` object, and the set of object + /// paths (to assert pagination/day keying). + async fn backfilled(store: &InMemory) -> (Vec, Vec) { + let prefix = OsPath::from("logs/audit"); + let metas = store + .list(Some(&prefix)) + .collect::>() + .await + .into_iter() + .map(|m| m.expect("list object")) + .collect::>(); + let mut ids = Vec::new(); + let mut paths = Vec::new(); + for meta in metas { + paths.push(meta.location.to_string()); + let bytes = store + .get(&meta.location) + .await + .expect("get object") + .bytes() + .await + .expect("read bytes"); + for line in std::str::from_utf8(&bytes).unwrap().lines() { + if line.is_empty() { + continue; + } + let v: serde_json::Value = serde_json::from_str(line).expect("valid ndjson"); + ids.push(v.get("id").and_then(|x| x.as_i64()).expect("row has id")); + } + } + ids.sort(); + paths.sort(); + (ids, paths) + } + + fn session(db: &DB, from: DateTime, to: DateTime) -> Session { + Session { + db: db.clone(), + owner: INSTANCE_NAME.clone(), + progress: RwLock::new(AuditBackfillProgress::new_running(from, to)), + } + } + + // End-to-end backfill: a settled multi-day window is exported in bounded keyset + // pages (forced to 2 rows/page) — every in-window row lands exactly once, rows + // outside [from,to) are excluded, a day that spans a page boundary produces more + // than one object, and re-running is idempotent (same keys overwritten, no dupes). + #[sqlx::test(migrations = "../migrations")] + async fn backfill_exports_window_in_pages(db: DB) -> anyhow::Result<()> { + let _serial = PAGE_OVERRIDE_LOCK.lock().await; + let _reset = ResetPageOverride; // restores the page override even on panic + let (store, dyn_store) = local_store(); + // Force multi-page / page-spanning-day keyset behaviour with a handful of rows. + PAGE_ROWS_OVERRIDE.store(2, Ordering::Relaxed); + + // In window [now-6d, now-2d): days 5, 4, 3 ago. + let mut want = Vec::new(); + for i in 0..3 { + want.push(insert_audit_days_ago(&db, &format!("bf.d5.{i}"), 5).await); + } + for i in 0..2 { + want.push(insert_audit_days_ago(&db, &format!("bf.d4.{i}"), 4).await); + } + for i in 0..2 { + want.push(insert_audit_days_ago(&db, &format!("bf.d3.{i}"), 3).await); + } + want.sort(); + // Out of window: before `from` and at/after `to`. + let before = insert_audit_days_ago(&db, "bf.before", 7).await; + let after = insert_audit_days_ago(&db, "bf.after", 1).await; + + let from = Utc::now() - chrono::Duration::days(6); + let to = Utc::now() - chrono::Duration::days(2); + + let s = session(&db, from, to); + run_backfill(&s, &db, &dyn_store, from, to).await?; + + let (ids, paths) = backfilled(&store).await; + assert_eq!(ids, want, "exactly the in-window rows, each once: {ids:?}"); + assert!( + !ids.contains(&before) && !ids.contains(&after), + "rows outside [from,to) must not be exported" + ); + // 3 rows on the day-5 partition at a 2-row page size => that day spans pages, + // so it yields >1 object — proving keyset paging across a day boundary. + let day5_objects = paths + .iter() + .filter(|p| p.contains("audit_backfill_")) + .count(); + assert!( + day5_objects >= 4, + "expected multiple paged objects (incl. a split day), got {paths:?}" + ); + { + let p = s.progress.read().await; + assert_eq!(p.rows_written, want.len() as u64, "progress row count"); + } + + // Idempotent re-run: deterministic keys are overwritten, never duplicated. + let s2 = session(&db, from, to); + run_backfill(&s2, &db, &dyn_store, from, to).await?; + let (ids2, _) = backfilled(&store).await; + assert_eq!(ids2, want, "re-run stays exactly once per row: {ids2:?}"); + + Ok(()) + } + + // A narrower backfill overlapping a broader one must not overwrite (and drop rows + // from) the broader run's object: the object key includes the window. The two + // windows share a day and the same first row (so the same `min_id`), but the + // narrower one holds fewer rows. + #[sqlx::test(migrations = "../migrations")] + async fn backfill_window_in_key_prevents_overwrite(db: DB) -> anyhow::Result<()> { + // Hold the lock so no concurrent test's PAGE_ROWS_OVERRIDE is observed; this test + // wants the default (large) page size so each day is one object per window. + let _serial = PAGE_OVERRIDE_LOCK.lock().await; + let (store, dyn_store) = local_store(); + + // Four rows on the same day at distinct times. + let base = Utc::now() - chrono::Duration::days(5); + let r0 = insert_audit_at(&db, "ov.0", base).await; + let r1 = insert_audit_at(&db, "ov.1", base + chrono::Duration::seconds(10)).await; + let r2 = insert_audit_at(&db, "ov.2", base + chrono::Duration::seconds(20)).await; + let r3 = insert_audit_at(&db, "ov.3", base + chrono::Duration::seconds(30)).await; + + // Broad run covers all four (one object for the day, keyed by r0). + let a_from = base - chrono::Duration::seconds(1); + let a_to = base + chrono::Duration::seconds(31); + run_backfill(&session(&db, a_from, a_to), &db, &dyn_store, a_from, a_to).await?; + + // Narrow run starts at the same first row (same min_id) but holds only r0, r1. + let b_from = base - chrono::Duration::seconds(1); + let b_to = base + chrono::Duration::seconds(15); + run_backfill(&session(&db, b_from, b_to), &db, &dyn_store, b_from, b_to).await?; + + let (ids, _) = backfilled(&store).await; + for id in [r0, r1, r2, r3] { + assert!( + ids.contains(&id), + "row {id} lost — a narrower overlapping window overwrote the broader run's \ + object: {ids:?}" + ); + } + Ok(()) + } + + // The endpoint rejects a window whose upper bound is not yet settled (a row's + // timestamp is its txn's xact_start, so a future/live `to` could miss late + // commits), but accepts a window safely in the past. + #[sqlx::test(migrations = "../migrations")] + async fn backfill_rejects_unstable_window(db: DB) -> anyhow::Result<()> { + let future = Utc::now() + chrono::Duration::days(1); + let past_from = Utc::now() - chrono::Duration::days(2); + let err = try_start(&db, past_from, future).await.unwrap_err(); + assert!( + matches!(err, error::Error::BadRequest(_)), + "a future `to` must be rejected as unstable, got {err:?}" + ); + + // A window fully in the settled past is accepted. + let from = Utc::now() - chrono::Duration::days(3); + let to = Utc::now() - chrono::Duration::days(2); + try_start(&db, from, to) + .await + .expect("a settled past window is accepted"); + Ok(()) + } + + /// Insert a row into the legacy (non-partitioned) `audit` table at an exact time. + async fn insert_legacy_audit_at(db: &DB, operation: &str, ts: DateTime) { + sqlx::query( + "INSERT INTO audit (workspace_id, username, operation, action_kind, parameters, timestamp) + VALUES ('test-ws','tester',$1,'create'::action_kind,'{}'::jsonb,$2)", + ) + .bind(operation) + .bind(ts) + .execute(db) + .await + .expect("insert legacy audit row"); + } + + // A window overlapping rows in the legacy (non-partitioned) `audit` table is rejected: + // those rows are not exported, so the backfill must not report success while silently + // omitting them. Covers the empty-`audit_partitioned` case (a min(partitioned) guard + // would no-op there). + #[sqlx::test(migrations = "../migrations")] + async fn backfill_rejects_window_overlapping_legacy(db: DB) -> anyhow::Result<()> { + // A legacy row ~5 days ago, and no partitioned rows at all. + insert_legacy_audit_at(&db, "legacy.row", Utc::now() - chrono::Duration::days(5)).await; + + // A window covering it is rejected. + let from = Utc::now() - chrono::Duration::days(6); + let to = Utc::now() - chrono::Duration::days(2); + let err = try_start(&db, from, to).await.unwrap_err(); + assert!( + matches!(err, error::Error::BadRequest(_)), + "a window overlapping legacy audit rows must be rejected, got {err:?}" + ); + + // A window clear of any legacy row is accepted. + let from_ok = Utc::now() - chrono::Duration::days(2); + let to_ok = Utc::now() - chrono::Duration::days(1); + try_start(&db, from_ok, to_ok) + .await + .expect("a window with no legacy overlap is accepted"); + Ok(()) + } +} diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 3c179168f5..a7e1322ad9 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -14,6 +14,8 @@ use std::{ #[cfg(feature = "parquet")] mod audit_logs_s3; #[cfg(feature = "parquet")] +mod audit_logs_s3_backfill; +#[cfg(feature = "parquet")] mod background_task; #[cfg(feature = "private")] mod ee; @@ -204,7 +206,12 @@ pub fn global_service() -> Router { ) .route("/run_log_cleanup", post(run_log_cleanup)) .route("/log_cleanup_status", get(log_cleanup_status)) - .route("/audit_logs_s3_status", get(audit_logs_s3_status)); + .route("/audit_logs_s3_status", get(audit_logs_s3_status)) + .route("/audit_logs_s3_backfill", post(run_audit_logs_s3_backfill)) + .route( + "/audit_logs_s3_backfill_status", + get(audit_logs_s3_backfill_status), + ); } #[cfg(not(feature = "parquet"))] @@ -601,6 +608,32 @@ async fn audit_logs_s3_status( Ok(Json(audit_logs_s3::get_status(&db).await?)) } +#[cfg(feature = "parquet")] +async fn run_audit_logs_s3_backfill( + Extension(db): Extension, + authed: ApiAuthed, + Json(req): Json, +) -> error::Result { + require_super_admin(&db, &authed.email).await?; + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + return Err(error::Error::BadRequest( + "Audit log export to object storage is an Enterprise feature".to_string(), + )); + } + audit_logs_s3_backfill::try_start(&db, req.from, req.to).await?; + audit_logs_s3_backfill::spawn_backfill(db.clone(), req.from, req.to); + Ok(axum::http::StatusCode::ACCEPTED) +} + +#[cfg(feature = "parquet")] +async fn audit_logs_s3_backfill_status( + Extension(db): Extension, + authed: ApiAuthed, +) -> error::JsonResult> { + require_super_admin(&db, &authed.email).await?; + Ok(Json(audit_logs_s3_backfill::get_status(&db).await?)) +} + #[derive(Deserialize)] pub struct TestKey { pub license_key: String, diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 540d6bea52..e511578821 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -1,7 +1,7 @@ { "openapi": "3.0.3", "info": { - "version": "1.739.0", + "version": "1.740.0", "title": "Windmill API", "contact": { "name": "Windmill Team", @@ -2719,6 +2719,124 @@ } } }, + "/settings/audit_logs_s3_backfill": { + "post": { + "summary": "start an opt-in historical backfill of audit logs to object storage", + "operationId": "runAuditLogsS3Backfill", + "tags": [ + "setting" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "from": { + "type": "string", + "format": "date-time", + "description": "inclusive lower bound of the window to export" + }, + "to": { + "type": "string", + "format": "date-time", + "description": "exclusive upper bound of the window to export" + } + }, + "required": [ + "from", + "to" + ] + } + } + } + }, + "responses": { + "202": { + "description": "backfill started" + } + } + } + }, + "/settings/audit_logs_s3_backfill_status": { + "get": { + "summary": "get status of the audit-log object-store historical backfill", + "operationId": "getAuditLogsS3BackfillStatus", + "tags": [ + "setting" + ], + "responses": { + "200": { + "description": "current backfill status (null if never run)", + "content": { + "application/json": { + "schema": { + "nullable": true, + "type": "object", + "properties": { + "running": { + "type": "boolean" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "finished_at": { + "type": "string", + "format": "date-time", + "nullable": true + }, + "phase": { + "type": "string" + }, + "from": { + "type": "string", + "format": "date-time" + }, + "to": { + "type": "string", + "format": "date-time" + }, + "rows_written": { + "type": "integer", + "format": "int64" + }, + "objects_written": { + "type": "integer", + "format": "int64" + }, + "last_ts": { + "type": "string", + "format": "date-time", + "nullable": true + }, + "errors": { + "type": "integer", + "format": "int64" + }, + "last_error": { + "type": "string", + "nullable": true + } + }, + "required": [ + "running", + "started_at", + "phase", + "from", + "to", + "rows_written", + "objects_written", + "errors" + ] + } + } + } + } + } + } + }, "/settings/send_stats": { "post": { "summary": "send stats", @@ -18440,6 +18558,14 @@ "type": "boolean" } }, + { + "name": "timeout", + "description": "custom timeout in seconds for this preview run", + "in": "query", + "schema": { + "type": "integer" + } + }, { "$ref": "#/components/parameters/NewJobId" } @@ -20465,6 +20591,89 @@ } } }, + "/w/{workspace}/jobs_u/get_flow_all_logs_structured/{id}": { + "get": { + "summary": "get all logs for a flow job in a structured format", + "operationId": "getFlowAllLogsStructured", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/JobId" + } + ], + "responses": { + "200": { + "description": "structured logs of all flow steps, one entry per job", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "job_id": { + "type": "string" + }, + "label": { + "type": "string", + "description": "human-readable label describing the job's position in the flow tree" + }, + "kind": { + "type": "string", + "description": "job kind (script, flow, forloopflow, ...)" + }, + "flow_step_id": { + "type": "string", + "nullable": true + }, + "step_path": { + "type": "string", + "nullable": true, + "description": "materialized step path (e.g. \"a/b\")" + }, + "depth": { + "type": "integer", + "description": "depth in the flow tree (0 for the root flow job)" + }, + "parent_module_type": { + "type": "string", + "nullable": true, + "description": "parent module type (forloopflow, branchall, ...)" + }, + "sibling_index": { + "type": "integer", + "description": "1-based index of this job among siblings sharing the same step" + }, + "sibling_count": { + "type": "integer", + "description": "total number of siblings sharing the same step" + }, + "logs": { + "type": "string" + } + }, + "required": [ + "job_id", + "label", + "kind", + "depth", + "sibling_index", + "sibling_count", + "logs" + ] + } + } + } + } + } + } + } + }, "/w/{workspace}/jobs_u/get_completed_logs_tail/{id}": { "get": { "summary": "get completed job logs tail", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 127fda6bed..e17c38efaf 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.739.0 + version: 1.740.0 title: Windmill API contact: name: Windmill Team @@ -2727,6 +2727,90 @@ paths: - bootstrapping - last_run_exported - updated_at + /settings/audit_logs_s3_backfill: + post: + summary: start an opt-in historical backfill of audit logs to object storage + operationId: runAuditLogsS3Backfill + tags: + - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + from: + type: string + format: date-time + description: inclusive lower bound of the window to export + to: + type: string + format: date-time + description: exclusive upper bound of the window to export + required: + - from + - to + responses: + '202': + description: backfill started + /settings/audit_logs_s3_backfill_status: + get: + summary: get status of the audit-log object-store historical backfill + operationId: getAuditLogsS3BackfillStatus + tags: + - setting + responses: + '200': + description: current backfill status (null if never run) + content: + application/json: + schema: + nullable: true + type: object + properties: + running: + type: boolean + started_at: + type: string + format: date-time + finished_at: + type: string + format: date-time + nullable: true + phase: + type: string + from: + type: string + format: date-time + to: + type: string + format: date-time + rows_written: + type: integer + format: int64 + objects_written: + type: integer + format: int64 + last_ts: + type: string + format: date-time + nullable: true + errors: + type: integer + format: int64 + last_error: + type: string + nullable: true + required: + - running + - started_at + - phase + - from + - to + - rows_written + - objects_written + - errors /settings/send_stats: post: summary: send stats @@ -18747,6 +18831,11 @@ paths: in: query schema: type: boolean + - name: timeout + description: custom timeout in seconds for this preview run + in: query + schema: + type: integer - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -21710,6 +21799,73 @@ paths: text/plain: schema: type: string + /w/{workspace}/jobs_u/get_flow_all_logs_structured/{id}: + get: + summary: get all logs for a flow job in a structured format + operationId: getFlowAllLogsStructured + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_4 + - name: id + in: path + required: true + schema: *ref_178 + responses: + '200': + description: structured logs of all flow steps, one entry per job + content: + application/json: + schema: + type: array + items: + type: object + properties: + job_id: + type: string + label: + type: string + description: >- + human-readable label describing the job's position in + the flow tree + kind: + type: string + description: job kind (script, flow, forloopflow, ...) + flow_step_id: + type: string + nullable: true + step_path: + type: string + nullable: true + description: materialized step path (e.g. "a/b") + depth: + type: integer + description: depth in the flow tree (0 for the root flow job) + parent_module_type: + type: string + nullable: true + description: parent module type (forloopflow, branchall, ...) + sibling_index: + type: integer + description: >- + 1-based index of this job among siblings sharing the + same step + sibling_count: + type: integer + description: total number of siblings sharing the same step + logs: + type: string + required: + - job_id + - label + - kind + - depth + - sibling_index + - sibling_count + - logs /w/{workspace}/jobs_u/get_completed_logs_tail/{id}: get: summary: get completed job logs tail diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0bbce245d3..7b7f4c4388 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1812,6 +1812,92 @@ paths: - last_run_exported - updated_at + /settings/audit_logs_s3_backfill: + post: + summary: start an opt-in historical backfill of audit logs to object storage + operationId: runAuditLogsS3Backfill + tags: + - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + from: + type: string + format: date-time + description: inclusive lower bound of the window to export + to: + type: string + format: date-time + description: exclusive upper bound of the window to export + required: + - from + - to + responses: + "202": + description: backfill started + + /settings/audit_logs_s3_backfill_status: + get: + summary: get status of the audit-log object-store historical backfill + operationId: getAuditLogsS3BackfillStatus + tags: + - setting + responses: + "200": + description: current backfill status (null if never run) + content: + application/json: + schema: + nullable: true + type: object + properties: + running: + type: boolean + started_at: + type: string + format: date-time + finished_at: + type: string + format: date-time + nullable: true + phase: + type: string + from: + type: string + format: date-time + to: + type: string + format: date-time + rows_written: + type: integer + format: int64 + objects_written: + type: integer + format: int64 + last_ts: + type: string + format: date-time + nullable: true + errors: + type: integer + format: int64 + last_error: + type: string + nullable: true + required: + - running + - started_at + - phase + - from + - to + - rows_written + - objects_written + - errors + /settings/send_stats: post: summary: send stats diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 72ccbeca2b..81a7b841b7 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -443,7 +443,7 @@ export const settings: Record = { { label: 'Store audit logs in object storage', description: - 'When enabled and instance object storage is configured, audit logs are also exported as newline-delimited JSON to the dedicated logs/audit/ folder (partitioned by day). Export is incremental and runs off the hot path. Pre-existing history is not backfilled: export starts from when the setting is enabled (transactions in flight at that moment may include a bounded set of just-prior rows). No audit log committed after enabling is ever skipped.', + 'When enabled and instance object storage is configured, audit logs are also exported as newline-delimited JSON to the dedicated logs/audit/ folder (partitioned by day). Export is incremental and runs off the hot path. Enabling (or re-enabling) anchors the export at ~now: while it stays enabled, every audit log committed from that point on is exported (transactions in flight at the moment of enabling may include a bounded set of just-prior rows). Pre-existing history, and any window during which export was disabled, are NOT exported by this cursor — use the opt-in backfill API to export a chosen historical range, back to when audit-log partitioning was introduced (older rows in the legacy audit table are not exported, and a window overlapping them is rejected): POST /settings/audit_logs_s3_backfill {from, to} (status at GET /settings/audit_logs_s3_backfill_status).', key: 'store_audit_logs_s3', fieldType: 'boolean', storage: 'setting', From 9172a0945bef7c82432006cc580ac3e635e0c72e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 26 Jun 2026 21:43:02 +0200 Subject: [PATCH 115/273] chore(main): release 1.741.0 (#9804) * chore(main): release 1.741.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 27 +++ backend/Cargo.lock | 160 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 +++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 147 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3b1808a0f..5f07f85012 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [1.741.0](https://github.com/windmill-labs/windmill/compare/v1.740.0...v1.741.0) (2026-06-26) + + +### Features + +* **ai-chat:** add create_folder tool to global chat ([#9819](https://github.com/windmill-labs/windmill/issues/9819)) ([44c25de](https://github.com/windmill-labs/windmill/commit/44c25de418612ab98341adb15d5671222b54367e)) +* **ai-chat:** hint /compact in context usage tooltip ([#9777](https://github.com/windmill-labs/windmill/issues/9777)) ([aadfb62](https://github.com/windmill-labs/windmill/commit/aadfb620c0b7dcd7e94367b761875e14ef9abe69)) +* **ai-chat:** let global chat edit the user's personal instructions ([#9771](https://github.com/windmill-labs/windmill/issues/9771)) ([3be2752](https://github.com/windmill-labs/windmill/commit/3be27521b05de33e48582e80c6651071f889f048)) +* **ai-chat:** surface raw apps in the @-mention context picker ([#9800](https://github.com/windmill-labs/windmill/issues/9800)) ([1602244](https://github.com/windmill-labs/windmill/commit/16022447c7b445be753b9545b10b4c67da0893d5)) +* capture managed-materialize output schema as asset metadata ([#2](https://github.com/windmill-labs/windmill/issues/2)a) ([#9812](https://github.com/windmill-labs/windmill/issues/9812)) ([ade74b2](https://github.com/windmill-labs/windmill/commit/ade74b297f6a03441e700a20ffc2d7291c8a85fd)) +* **sdk:** allow overriding worker tag when running jobs (WIN-2105) ([#9807](https://github.com/windmill-labs/windmill/issues/9807)) ([52fc7bf](https://github.com/windmill-labs/windmill/commit/52fc7bf94cf3f87f68d9dba9884944d87e7d5d57)) + + +### Bug Fixes + +* apply step timeout to 'Test this step' preview ([#9810](https://github.com/windmill-labs/windmill/issues/9810)) ([d04062b](https://github.com/windmill-labs/windmill/commit/d04062bff58c9c4c79ce542a4321e71bcbcf0e98)) +* **flows:** reject corrupt step paths at deploy + atomic cache writes ([#9751](https://github.com/windmill-labs/windmill/issues/9751)) ([#9813](https://github.com/windmill-labs/windmill/issues/9813)) ([3cda447](https://github.com/windmill-labs/windmill/commit/3cda44762148bcd2ee5c0ea821db884950376ead)) +* **frontend:** clarify instance data table unavailable on cloud ([#9806](https://github.com/windmill-labs/windmill/issues/9806)) ([c3e8c78](https://github.com/windmill-labs/windmill/commit/c3e8c789ac05c9c28991d9ab6f2358f61fa87971)) +* hide GCS service account key behind a reveal in object storage settings ([#9815](https://github.com/windmill-labs/windmill/issues/9815)) ([0ec5061](https://github.com/windmill-labs/windmill/commit/0ec5061270749ed078e01f5a4bc7397a1755ca32)) +* ping job during volume setup to prevent false zombie restarts ([#9803](https://github.com/windmill-labs/windmill/issues/9803)) ([43bb676](https://github.com/windmill-labs/windmill/commit/43bb676dc5652cb06fe1414b8d3aacf295bae36b)) +* skipped suspend step no longer parks the flow forever ([#9821](https://github.com/windmill-labs/windmill/issues/9821)) ([40110bc](https://github.com/windmill-labs/windmill/commit/40110bc7158bc42c3d84bd4637a12b82fcd72a9a)) + + +### Performance Improvements + +* **audit:** re-anchor S3 audit export on enable + opt-in backfill ([#9818](https://github.com/windmill-labs/windmill/issues/9818)) ([577ceee](https://github.com/windmill-labs/windmill/commit/577ceeee8679f054c6898d1a7889df30ab830f8f)) + ## [1.740.0](https://github.com/windmill-labs/windmill/compare/v1.739.0...v1.740.0) (2026-06-25) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9830bed379..8f11a0cb98 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -3412,9 +3412,9 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" [[package]] name = "debug-helper" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e" +checksum = "80a4af69c60438a1a82af89d362f4729fd38db7b73f305a237636fad31ceb2bf" [[package]] name = "debugid" @@ -13734,7 +13734,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-nats", @@ -13816,7 +13816,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.740.0" +version = "1.741.0" dependencies = [ "async-stream", "async-trait", @@ -13849,7 +13849,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13862,7 +13862,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "argon2", @@ -14000,7 +14000,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14023,7 +14023,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14037,7 +14037,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14063,7 +14063,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.740.0" +version = "1.741.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14073,7 +14073,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14090,7 +14090,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14112,7 +14112,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14135,7 +14135,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14151,7 +14151,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14172,7 +14172,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14193,7 +14193,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14207,7 +14207,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-nats", @@ -14242,7 +14242,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14267,7 +14267,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14285,7 +14285,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14307,7 +14307,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14327,7 +14327,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14364,7 +14364,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14392,7 +14392,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.740.0" +version = "1.741.0" dependencies = [ "lazy_static", "serde", @@ -14404,7 +14404,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.740.0" +version = "1.741.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14429,7 +14429,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14443,7 +14443,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.740.0" +version = "1.741.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14476,7 +14476,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.740.0" +version = "1.741.0" dependencies = [ "chrono", "lazy_static", @@ -14490,7 +14490,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14509,7 +14509,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.740.0" +version = "1.741.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14611,7 +14611,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.740.0" +version = "1.741.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14630,7 +14630,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.740.0" +version = "1.741.0" dependencies = [ "regex", "serde", @@ -14645,7 +14645,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14669,7 +14669,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "futures", @@ -14686,7 +14686,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.740.0" +version = "1.741.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14702,7 +14702,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -14723,7 +14723,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -14754,7 +14754,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "arc-swap", @@ -14779,7 +14779,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-stream", @@ -14813,7 +14813,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "futures", @@ -14831,7 +14831,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.740.0" +version = "1.741.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14840,7 +14840,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -14852,7 +14852,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde_json", @@ -14864,7 +14864,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "gosyn", @@ -14876,7 +14876,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -14888,7 +14888,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde_json", @@ -14900,7 +14900,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "nu-parser", @@ -14911,7 +14911,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14922,7 +14922,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14934,7 +14934,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14945,7 +14945,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-recursion", @@ -14967,7 +14967,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde_json", @@ -14979,7 +14979,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -14993,7 +14993,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15010,7 +15010,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -15023,7 +15023,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde", @@ -15035,7 +15035,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -15053,7 +15053,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15069,7 +15069,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15085,7 +15085,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde", @@ -15096,7 +15096,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-recursion", @@ -15135,7 +15135,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "const_format", @@ -15174,7 +15174,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.740.0" +version = "1.741.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15185,7 +15185,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-recursion", @@ -15219,7 +15219,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15276,7 +15276,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15309,7 +15309,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15329,7 +15329,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15363,7 +15363,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15399,7 +15399,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15422,7 +15422,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15446,7 +15446,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-nats", @@ -15470,7 +15470,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15505,7 +15505,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15533,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-trait", @@ -15558,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15577,7 +15577,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-once-cell", @@ -15687,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.740.0" +version = "1.741.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f6453298cd..d721dcc272 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.740.0" +version = "1.741.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.740.0" +version = "1.741.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 7616f1b324..78db9dd2b0 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.740.0" +version = "1.741.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.740.0" +version = "1.741.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.740.0" +version = "1.741.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.740.0" +version = "1.741.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 4538d1be39..6faadc1cec 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.740.0" +version = "1.741.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7b7f4c4388..e9c57c3fb0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.740.0 + version: 1.741.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 596737fb93..2b3b23746a 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.740.0"; +export const VERSION = "v1.741.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index b84b5a5939..1e40e04ac7 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.740.0"; +export const VERSION = "1.741.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index beb9b9fe12..ed9dd1d7ea 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.740.0", + "version": "1.741.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.740.0", + "version": "1.741.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index db92ba42f2..154497ef76 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.740.0", + "version": "1.741.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 5b1de97770..cda87ce8d7 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.740.0" +wmill = ">=1.741.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index a14b6e3d1d..9ccfa3ceab 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.740.0 + version: 1.741.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 8048bc9d50..ebdfaa8215 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.740.0' + ModuleVersion = '1.741.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 4244d43338..d0942b025c 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.740.0" +version = "1.741.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 2ba27adde2..13d6388e30 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.740.0", + "version": "1.741.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index f710adb116..42ebc2096d 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.740.0", + "version": "1.741.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index d964f54b8d..b27f19abcc 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.740.0 +1.741.0 From 003a262a4e9d6c2a63ada01aa8429aea1fbb6031 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 27 Jun 2026 21:08:00 +0200 Subject: [PATCH 116/273] feat: column-level lineage for DuckLake pipelines (SQL-AST inferred + traceable) (#9814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: column-level lineage for ducklake pipelines via // column annotation Co-Authored-By: Claude Opus 4.8 * feat: auto-derive column lineage from DuckDB SQL AST (annotation as override) Co-Authored-By: Claude Opus 4.8 * docs: clarify column-lineage inference is server-side; drafts use annotations Co-Authored-By: Claude Opus 4.8 * feat(frontend): surface inferred column lineage in live pipeline drafts Threads the DuckDB SQL-AST column lineage (from the WASM asset parser) through ScriptEditor -> details pane -> page -> resolveGraph, merged with // column annotations (annotation wins) so the live preview matches the deployed graph. Takes effect once windmill-parser-wasm-asset is republished with the inference. Co-Authored-By: Claude Opus 4.8 * chore(frontend): bump windmill-parser-wasm-asset to 1.740.0 for SQL column-lineage inference Co-Authored-By: Claude Opus 4.8 * docs: column-lineage inference now runs live (WASM) too, merged with annotations Co-Authored-By: Claude Opus 4.8 * feat(frontend): transitive column-lineage trace (impact analysis) Stitches every producer's column_lineage into a pipeline-wide column graph (columnLineageGraph.ts) and replaces the single-hop diagram with an interactive ColumnLineageTrace: select an asset to see its columns' full upstream/downstream lineage across scripts; click any column to highlight its complete transitive impact set (forward + backward) and dim the rest. Co-Authored-By: Claude Opus 4.8 * fix: address CI review on column lineage (parse-fallback, node-id, perf, leak) - backend: DuckDB SQL parse failure now falls back to `// column` annotation lineage instead of dropping it (Codex P1) - columnLineageGraph: collision-proof JSON node ids; deterministic first-write output anchoring when a producer has multiple ducklake writes (cubic P2 ×2) - pipeline page: gate buildColumnGraph to a ducklake-asset selection so it doesn't rebuild on every editor keystroke (cubic P2) - ScriptEditor: clear inferredColumnLineage on parse error so it can't leak across a script switch (cubic P2) - AssetGraphEdge: widen badge stacking offset 12px->18px to fully clear (cubic P3) Co-Authored-By: Claude Opus 4.8 * fix: resolve JOIN inputs + anchor column lineage to // materialize target Addresses the second Codex review pass (two P1s): - SQL inference now walks JOINed tables: build_from_maps maps every FROM entry AND its joins into the alias map, and single-table attribution requires no joins. `SELECT o.x, c.y FROM a o JOIN b c` now resolves c.y (was dropped). - The column graph anchors a producer's lineage to its declared // materialize target (surfaced on the runnable node) instead of guessing a ducklake write-edge, which is unordered for deployed graphs and ambiguous for multi-output scripts. Falls back to a write-edge when no materialize target. Co-Authored-By: Claude Opus 4.8 * fix: gate column-lineage badge to the // materialize target write-edge The canvas badge keyed on `e.asset_kind === 'ducklake'`, so a multi-output producer showed the same column mapping on every ducklake write-edge. Use the same materialize-target anchor as buildColumnGraph: the badge lands only on the declared output's edge, falling back to the ducklake write-edge when there's no materialize annotation. (Codex P1) Co-Authored-By: Claude Opus 4.8 * fix: build column trace from displayGraph so View hides draft lineage The transitive column trace was built from graphWithDraft regardless of mode, so in View with drafts hidden it could surface draft `// column` lineage the deployed canvas doesn't show. Build it from `displayGraph` (the graph the canvas actually renders) so the trace matches: draft overlays in edit / show-drafts, deployed-only in plain View. (Codex P2) Co-Authored-By: Claude Opus 4.8 * fix: don't infer column lineage for local/temp staging CTAS A CTAS into a local/temp staging table isn't the materialized output, but its projection was inferred and (flat) column_lineage anchored to the script's // materialize target — so staging columns showed up as the final asset's. Gate inference to the actual output: a top-level managed-materialize SELECT, or a CTAS/CREATE VIEW whose target resolves to a real asset. (Codex P1) Co-Authored-By: Claude Opus 4.8 * fix: scope inferred column lineage to one output asset Inference accumulated columns from every output-producing query into one flat list, all anchored (frontend) to the script's // materialize target — so an auxiliary CTAS into a different asset showed its columns on the materialized one. Tag each inferred entry with its output asset and, in parse_assets, scope the list to the // materialize target (keeping untagged top-level-SELECT entries); with no declared target, drop inference when entries span multiple output assets rather than attribute them to an arbitrary one. Parser-internal — no wire change. (Codex P1) Co-Authored-By: Claude Opus 4.8 * fix: treat CREATE TEMP TABLE/VIEW as local even under an active USE A one-part temp name under `USE dl` resolved to an asset (ducklake://…/tmp) before being registered local, so a final SELECT reading it invented `final.total <- warehouse/tmp.amt` (a phantom DuckLake column) and recorded a phantom asset. track_table_definition now registers any temporary table/view as local up front, bypassing active-asset resolution; CreateTable/CreateView pass their `temporary` flag. (Codex P1) Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- ...ecdac6806e1a306dca880943d97bb6d6a889d.json | 67 ++ backend/Cargo.lock | 1 + .../src/asset_parser.rs | 669 ++++++++++++++++-- .../windmill-parser/src/asset_parser.rs | 233 ++++++ .../tests/fixtures/pipeline_annotations.json | 91 +++ .../tests/pipeline_annotations_parity.rs | 13 + backend/windmill-api-assets/Cargo.toml | 1 + backend/windmill-api-assets/src/lib.rs | 63 +- backend/windmill-common/src/assets.rs | 4 +- docs/pipelines-vs-dbt.md | 38 +- frontend/package-lock.json | 58 +- frontend/package.json | 2 +- .../src/lib/components/ScriptEditor.svelte | 20 +- .../assets/AssetGraph/AssetGraphCanvas.svelte | 42 +- .../AssetGraph/AssetGraphDetailsPane.svelte | 45 +- .../assets/AssetGraph/AssetGraphEdge.svelte | 50 +- .../AssetGraph/ColumnLineageTrace.svelte | 173 +++++ .../AssetGraph/columnLineageGraph.test.ts | 162 +++++ .../assets/AssetGraph/columnLineageGraph.ts | 169 +++++ .../parsePipelineAnnotations.parity.test.ts | 8 +- .../parsePipelineAnnotations.test.ts | 32 +- .../AssetGraph/parsePipelineAnnotations.ts | 83 ++- .../assets/AssetGraph/resolveGraph.ts | 36 +- .../lib/components/assets/AssetGraph/types.ts | 10 +- frontend/src/lib/infer.ts | 5 + .../(logged)/pipeline/[folder]/+page.svelte | 47 +- 26 files changed, 1973 insertions(+), 149 deletions(-) create mode 100644 backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json create mode 100644 frontend/src/lib/components/assets/AssetGraph/ColumnLineageTrace.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts diff --git a/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json b/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json new file mode 100644 index 0000000000..ffce4491e3 --- /dev/null +++ b/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json @@ -0,0 +1,67 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\",\n language AS \"language!: windmill_common::scripts::ScriptLang\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ORDER BY path, created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "content!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "language!: windmill_common::scripts::ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 8f11a0cb98..c970648000 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14033,6 +14033,7 @@ dependencies = [ "tracing", "windmill-api-auth", "windmill-common", + "windmill-parser-sql-asset", ] [[package]] diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index 812bfa785d..f7c578a274 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -9,8 +9,9 @@ use sqlparser::{ parser::Parser, }; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, - AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, + asset_was_used, merge_assets, merge_column_lineage, parse_asset_syntax, + parse_pipeline_annotations, AssetKind, AssetUsageAccessType, ColumnLineage, ColumnRef, + ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -33,7 +34,55 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - let pipeline = parse_pipeline_annotations(input); + let mut pipeline = parse_pipeline_annotations(input); + // Scope inferred lineage to a single output asset so columns from an + // auxiliary CTAS aren't attributed to the materialized one (the flat list + // has no per-entry output on the wire). The `// materialize` target, when + // declared, IS the output: keep entries tagged with it plus untagged + // top-level-SELECT entries (which describe that target). Without a declared + // target, keep inference only when every tagged entry shares one output + // asset — otherwise it's ambiguous which asset the flat list describes, so + // drop it rather than show false dependencies. + let target = pipeline + .materialize + .as_ref() + .map(|m| (m.target_kind, m.target_path.clone())); + let inferred: Vec = match &target { + Some(t) => collector + .column_lineage + .into_iter() + .filter(|(out, _)| out.as_ref().map_or(true, |o| o == t)) + .map(|(_, cl)| cl) + .collect(), + None => { + let mut first: Option<&(AssetKind, String)> = None; + let mut ambiguous = false; + for (out, _) in &collector.column_lineage { + if let Some(o) = out { + match first { + None => first = Some(o), + Some(f) if f != o => { + ambiguous = true; + break; + } + _ => {} + } + } + } + if ambiguous { + Vec::new() + } else { + collector + .column_lineage + .into_iter() + .map(|(_, cl)| cl) + .collect() + } + } + }; + // Body-inferred column lineage, with `// column` annotations taking + // precedence per output column (explicit declaration overrides inference). + pipeline.column_lineage = merge_column_lineage(inferred, pipeline.column_lineage); Ok(ParseAssetsOutput::new( merge_assets(collector.assets), Vec::new(), @@ -54,6 +103,16 @@ struct AssetCollector { cte_name_stack: Vec>, // Locally created tables (not attached to an asset) local_table_names: HashSet, + // Inferred column-level lineage: one entry per output column of an + // output-producing query, mapping it to the upstream source columns its + // expression reads. Each is tagged with the *output asset* it belongs to — + // `Some((kind, path))` for a CTAS / CREATE VIEW into a real asset, `None` + // for a top-level managed-materialize SELECT (its output is the `// + // materialize` target, known only in `parse_assets`). `parse_assets` uses + // the tag to scope the flat list to a single output asset so columns from an + // auxiliary output don't get attributed to the materialized one. Best-effort: + // dynamic/raw SQL, INSERT…SELECT, and wildcards are left to annotations. + column_lineage: Vec<(Option<(AssetKind, String)>, ColumnLineage)>, } impl AssetCollector { @@ -65,15 +124,24 @@ impl AssetCollector { currently_used_asset: None, cte_name_stack: Vec::new(), local_table_names: HashSet::new(), + column_lineage: Vec::new(), } } - /// If the name resolves to an attached asset, record it. Otherwise, register it as a local - /// table/view so that subsequent references are not mistakenly attributed to the active asset. - fn track_table_definition(&mut self, name: &ObjectName) { - if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(W)) { - self.assets.push(asset); - } else if let Some(simple_name) = get_trivial_obj_name(name) { + /// Record a `CREATE TABLE`/`VIEW` target. A *temporary* table/view is always + /// local — even a one-part name under an active `USE dl`, which would + /// otherwise resolve to an asset (`ducklake://…/tmp`) and then leak as a + /// column source for later references. A non-temp name that resolves to an + /// attached asset is recorded as that asset; anything else is registered + /// local so subsequent references aren't attributed to the active asset. + fn track_table_definition(&mut self, name: &ObjectName, is_temporary: bool) { + if !is_temporary { + if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(W)) { + self.assets.push(asset); + return; + } + } + if let Some(simple_name) = get_trivial_obj_name(name) { self.local_table_names.insert(simple_name.to_lowercase()); } } @@ -280,6 +348,22 @@ impl AssetCollector { } } + // Infer the output-column lineage of a query that produces an asset, tagging + // each entry with its `output` asset. Called only for an output-producing + // query — a top-level managed-materialize SELECT (`output: None`, resolved + // to the `// materialize` target later) or a CTAS / CREATE VIEW into a real + // asset (`output: Some`). A CTAS into a local/temp staging table is never + // an output, so it's simply not passed here. + fn infer_query_output( + &mut self, + query: &sqlparser::ast::Query, + output: Option<(AssetKind, String)>, + ) { + if let Some(select) = query.body.as_select() { + self.infer_column_lineage(&select.projection, &select.from, output); + } + } + fn handle_table_with_joins( &mut self, table_with_joins: &sqlparser::ast::TableWithJoins, @@ -302,17 +386,57 @@ impl AssetCollector { } } - // Extract columns from SELECT items and create individual asset results for each column - // Only processes columns that reference known assets to avoid false positives - fn extract_column_assets( - &mut self, - projection: &[SelectItem], + // The alias-map entry (key → asset) for one FROM/JOIN table factor, or + // `None` if it isn't an asset-backed table. The key is its alias, else the + // bare table name; S3 table-functions and string-literal tables are only + // keyed when aliased (an unaliased one is ambiguous). Returns the asset with + // a single matched relation so the caller can attribute qualified columns. + fn table_alias_entry(&self, relation: &TableFactor) -> Option<(String, ParseAssetsResult)> { + let TableFactor::Table { name, alias, args, .. } = relation else { + return None; + }; + let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty()); + if has_args { + let alias = alias.as_ref()?; + let asset = self.get_s3_asset_from_table_function(relation)?; + return Some((alias.name.value.clone(), asset)); + } + let asset = self + .get_associated_asset_from_obj_name(name, Some(R)) + .or_else(|| self.get_s3_asset_from_str_literal_table(relation))?; + if get_str_lit_from_obj_name(name).is_some() { + // String-literal S3 table: only unambiguous when aliased. + let alias = alias.as_ref()?; + return Some((alias.name.value.clone(), asset)); + } + let key = match alias { + Some(a) => a.name.value.clone(), + None => name + .0 + .last() + .and_then(|id| id.as_ident()) + .map(|id| id.value.clone()) + .unwrap_or_default(), + }; + Some((key, asset)) + } + + // Resolve a query's FROM clause into (single-table asset, alias→asset map). + // `single_table` is `Some` only for an unambiguous one-table FROM with no + // joins (so bare column refs can be attributed); `table_to_asset` keys by + // alias/table name for qualified refs and includes every JOINed table. + // Shared by `extract_column_assets` (read columns) and `infer_column_lineage` + // (output→input edges) so both resolve identically. + fn build_from_maps( + &self, from_tables: &[sqlparser::ast::TableWithJoins], + ) -> ( + Option, + BTreeMap, ) { - // Check if this is a single-table SELECT (to avoid ambiguity). - // For S3 table functions (read_parquet/read_csv/read_json), detect the asset even - // though args are present, since we know the file path from the string literal arg. - let single_table = if from_tables.len() == 1 { + // Single unambiguous table only when there's exactly one FROM entry AND + // it has no joins — otherwise a bare column could belong to any side. + let single_table = if from_tables.len() == 1 && from_tables[0].joins.is_empty() { let relation = &from_tables[0].relation; if let TableFactor::Table { name, args, .. } = relation { let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty()); @@ -329,51 +453,30 @@ impl AssetCollector { None }; - // Build a map of table aliases/names to assets for multi-table queries. - // For S3 table functions, only aliased references are unambiguous - // (e.g. SELECT t.col1 FROM read_parquet('s3://...') AS t). + // Alias → asset for qualified column refs, across every FROM entry AND + // its JOINed tables (so `c.col` in `FROM a JOIN c` resolves). let mut table_to_asset: BTreeMap = BTreeMap::new(); for table_with_joins in from_tables { - if let TableFactor::Table { name, alias, args, .. } = &table_with_joins.relation { - let has_args = args.as_ref().map_or(false, |a| !a.args.is_empty()); - if has_args { - // For table functions, only add to the alias map when an alias is present - if let Some(alias) = alias { - if let Some(asset) = - self.get_s3_asset_from_table_function(&table_with_joins.relation) - { - table_to_asset.insert(alias.name.value.clone(), asset); - } - } - } else if let Some(asset) = self - .get_associated_asset_from_obj_name(name, Some(R)) - .or_else(|| { - self.get_s3_asset_from_str_literal_table(&table_with_joins.relation) - }) - { - // For string literal S3 tables (e.g. FROM 's3:///file.parquet'), only add to - // the alias map when an alias is present (to avoid false positives). - // For regular named tables, use alias or table name as key. - let is_str_literal = get_str_lit_from_obj_name(name).is_some(); - if is_str_literal { - if let Some(alias) = alias { - table_to_asset.insert(alias.name.value.clone(), asset); - } - } else { - let table_key = if let Some(alias) = alias { - alias.name.value.clone() - } else { - name.0 - .last() - .and_then(|id| id.as_ident()) - .map(|id| id.value.clone()) - .unwrap_or_default() - }; - table_to_asset.insert(table_key, asset); - } + if let Some((k, a)) = self.table_alias_entry(&table_with_joins.relation) { + table_to_asset.insert(k, a); + } + for join in &table_with_joins.joins { + if let Some((k, a)) = self.table_alias_entry(&join.relation) { + table_to_asset.insert(k, a); } } } + (single_table, table_to_asset) + } + + // Extract columns from SELECT items and create individual asset results for each column + // Only processes columns that reference known assets to avoid false positives + fn extract_column_assets( + &mut self, + projection: &[SelectItem], + from_tables: &[sqlparser::ast::TableWithJoins], + ) { + let (single_table, table_to_asset) = self.build_from_maps(from_tables); // Process each SELECT item for item in projection { @@ -446,6 +549,136 @@ impl AssetCollector { } } } + + // Infer column-level lineage for an output-producing query's projection: + // each *named* output column → the upstream source columns its expression + // reads. Covers passthroughs (`amount`) and computed columns + // (`amount + tax AS total`) alike. Skipped: wildcards and unaliased + // expressions (no stable output name), and inputs that don't resolve to a + // known asset. A column with no resolved inputs is dropped. + // + // Best-effort and intentionally flat: results from every output query in the + // script accumulate into one list (the graph hangs them off the materialize + // write-edge), with no per-output-table association. This is exact for the + // common single-output member (a managed-materialize SELECT, or one CTAS), + // but a multi-statement script that stages through a TEMP table reports the + // *intermediate* column names (the final SELECT reads the temp table, whose + // columns don't resolve to an asset, so they drop out). A `// column` + // annotation overrides any output column where inference is wrong or coarse. + fn infer_column_lineage( + &mut self, + projection: &[SelectItem], + from_tables: &[sqlparser::ast::TableWithJoins], + output: Option<(AssetKind, String)>, + ) { + let (single_table, table_to_asset) = self.build_from_maps(from_tables); + for item in projection { + let (out_col, expr) = match item { + SelectItem::ExprWithAlias { expr, alias } => (alias.value.clone(), expr), + SelectItem::UnnamedExpr(expr @ Expr::Identifier(id)) => (id.value.clone(), expr), + SelectItem::UnnamedExpr(expr @ Expr::CompoundIdentifier(parts)) => { + match parts.last() { + Some(last) => (last.value.clone(), expr), + None => continue, + } + } + _ => continue, + }; + let mut collector = ColumnIdentCollector { refs: Vec::new(), query_depth: 0 }; + let _ = expr.visit(&mut collector); + let mut inputs: Vec = Vec::new(); + for parts in &collector.refs { + if let Some(cr) = self.resolve_column_ref(parts, &single_table, &table_to_asset) { + if !inputs.contains(&cr) { + inputs.push(cr); + } + } + } + if !inputs.is_empty() { + self.column_lineage + .push((output.clone(), ColumnLineage { column: out_col, inputs })); + } + } + } + + // Resolve identifier `parts` (e.g. `["t","amount"]` or `["amount"]`) to the + // source asset column it reads, mirroring `extract_column_assets`' + // resolution: a bare ident needs an unambiguous single-table FROM; a + // qualified ident resolves its prefix via the alias map, or (≥3 parts) as a + // db/schema-qualified object name. + fn resolve_column_ref( + &self, + parts: &[String], + single_table: &Option, + table_to_asset: &BTreeMap, + ) -> Option { + let asset_to_ref = |asset: &ParseAssetsResult, col: &str| ColumnRef { + from_kind: asset.kind, + from_path: asset.path.clone(), + from_column: col.to_string(), + }; + match parts { + [col] => single_table.as_ref().map(|a| asset_to_ref(a, col)), + [.., col] => { + let prefix = parts.first()?; + if let Some(asset) = table_to_asset.get(prefix) { + Some(asset_to_ref(asset, col)) + } else if parts.len() >= 3 { + let obj_parts: Vec = parts[..parts.len() - 1] + .iter() + .map(|p| ObjectNamePart::Identifier(sqlparser::ast::Ident::new(p.clone()))) + .collect(); + let asset = + self.get_associated_asset_from_obj_name(&ObjectName(obj_parts), Some(R))?; + Some(asset_to_ref(&asset, col)) + } else { + None + } + } + [] => None, + } + } +} + +// Collects the identifier paths an expression reads, for column-lineage +// inference: `Expr::Identifier(a)` → `["a"]`, `Expr::CompoundIdentifier(t.a)` → +// `["t","a"]`. The derived `Visit` walk recurses through operators, functions, +// casts and CASE, so every leaf identifier of the outer expression is captured. +struct ColumnIdentCollector { + refs: Vec>, + // Depth of nested (sub)queries inside the expression. Identifiers are only + // captured at depth 0: a scalar/correlated subquery's columns belong to ITS + // own FROM, not the outer projection's, so descending would misattribute + // (e.g. `(SELECT x FROM other) AS c FROM orders` must NOT bind `c` to + // `orders.x`). Subquery-derived columns are simply left to annotations. + query_depth: usize, +} + +impl Visitor for ColumnIdentCollector { + type Break = (); + + fn pre_visit_query(&mut self, _query: &sqlparser::ast::Query) -> std::ops::ControlFlow<()> { + self.query_depth += 1; + std::ops::ControlFlow::Continue(()) + } + + fn post_visit_query(&mut self, _query: &sqlparser::ast::Query) -> std::ops::ControlFlow<()> { + self.query_depth = self.query_depth.saturating_sub(1); + std::ops::ControlFlow::Continue(()) + } + + fn pre_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow { + if self.query_depth == 0 { + match expr { + Expr::Identifier(id) => self.refs.push(vec![id.value.clone()]), + Expr::CompoundIdentifier(parts) => self + .refs + .push(parts.iter().map(|id| id.value.clone()).collect()), + _ => {} + } + } + std::ops::ControlFlow::Continue(()) + } } impl Visitor for AssetCollector { @@ -505,7 +738,11 @@ impl Visitor for AssetCollector { ) -> std::ops::ControlFlow { match statement { sqlparser::ast::Statement::Query(q) => { + // A top-level SELECT is the managed-materialize output, so its + // columns ARE the materialized asset's columns (output resolved + // to the `// materialize` target in `parse_assets`). self.handle_query_reads(q); + self.infer_query_output(q, None); } sqlparser::ast::Statement::Insert(insert) => { @@ -658,18 +895,32 @@ impl Visitor for AssetCollector { } sqlparser::ast::Statement::CreateTable(create_table) => { - self.track_table_definition(&create_table.name); + self.track_table_definition(&create_table.name, create_table.temporary); // `CREATE TABLE x AS SELECT … FROM y` reads y. The AS-query // isn't a `Statement::Query`, so its FROM tables are only - // caught here. + // caught here. Only infer output lineage when `x` is a real + // asset — a CTAS into a local/temp staging table is not the + // materialized output (its columns aren't the asset's). if let Some(query) = &create_table.query { self.handle_query_reads(query); + // Infer only when `x` is a real asset (its output columns + // ARE that asset's), tagged with it so `parse_assets` can + // scope lineage per output. A local/temp staging table is + // not an asset → not inferred. + if let Some(asset) = + self.get_associated_asset_from_obj_name(&create_table.name, Some(W)) + { + self.infer_query_output(query, Some((asset.kind, asset.path))); + } } } - sqlparser::ast::Statement::CreateView { name, query, .. } => { - self.track_table_definition(name); + sqlparser::ast::Statement::CreateView { name, query, temporary, .. } => { + self.track_table_definition(name, *temporary); self.handle_query_reads(query); + if let Some(asset) = self.get_associated_asset_from_obj_name(name, Some(W)) { + self.infer_query_output(query, Some((asset.kind, asset.path))); + } } // DROP TABLE/VIEW is a write to the dropped object — the @@ -687,7 +938,9 @@ impl Visitor for AssetCollector { | sqlparser::ast::ObjectType::MaterializedView ) { for name in names { - self.track_table_definition(name); + // DROP is a write to the named object; resolve it as an + // asset if it is one (not a temp-creation context). + self.track_table_definition(name, false); } } } @@ -1920,8 +2173,9 @@ mod ctas_read_tests { assets ); assert!( - assets.iter().any(|a| a.path == "main/exciting_809" - && a.access_type == Some(W)), + assets + .iter() + .any(|a| a.path == "main/exciting_809" && a.access_type == Some(W)), "expected write of main/exciting_809, got {:?}", assets ); @@ -1935,9 +2189,292 @@ mod ctas_read_tests { "#; let assets = parse_assets(input).unwrap().assets; assert!( - assets.iter().any(|a| a.path == "main/fx_rates" && a.access_type == Some(R)), + assets + .iter() + .any(|a| a.path == "main/fx_rates" && a.access_type == Some(R)), "expected read of main/fx_rates, got {:?}", assets ); } + + fn lineage(input: &str) -> Vec { + parse_assets(input).unwrap().column_lineage + } + + #[test] + fn test_infer_lineage_computed_and_passthrough() { + // CTAS with a computed column (amount + tax) and a passthrough (id). + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.orders_daily AS + SELECT dl.orders.id, dl.orders.amount + dl.orders.tax AS order_total + FROM dl.orders; + "#; + let got = lineage(input); + assert_eq!( + got, + vec![ + ColumnLineage { + column: "id".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "id".to_string(), + }], + }, + ColumnLineage { + column: "order_total".to_string(), + inputs: vec![ + ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "amount".to_string(), + }, + ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "tax".to_string(), + }, + ], + }, + ] + ); + } + + #[test] + fn test_infer_lineage_bare_column_single_table() { + // Managed-materialize form: a plain top-level SELECT, bare columns + // attributed to the single FROM table. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + USE dl; + SELECT amount AS revenue FROM orders; + "#; + let got = lineage(input); + assert_eq!( + got, + vec![ColumnLineage { + column: "revenue".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "amount".to_string(), + }], + }] + ); + } + + #[test] + fn test_infer_lineage_annotation_overrides() { + // The `// column` annotation for `order_total` wins; `id` stays inferred. + let input = r#" + -- column order_total <- datatable://prod/manual.grand_total + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.orders_daily AS + SELECT dl.orders.id, dl.orders.amount + dl.orders.tax AS order_total + FROM dl.orders; + "#; + let got = lineage(input); + // Annotation entry is authoritative and listed first. + assert_eq!(got[0].column, "order_total"); + assert_eq!(got[0].inputs[0].from_path, "prod/manual"); + assert_eq!(got[0].inputs[0].from_column, "grand_total"); + // Inferred `id` survives; inferred `order_total` dropped (no dup). + assert!(got.iter().any(|c| c.column == "id")); + assert_eq!(got.iter().filter(|c| c.column == "order_total").count(), 1); + } + + #[test] + fn test_infer_lineage_skips_local_staging_ctas() { + // A CTAS into a TEMP/local table is NOT the materialized output, so its + // columns must not be reported (they'd be anchored to the script's + // `// materialize` target as if they were the final asset's columns). + // The final SELECT reads the local staging table → unresolved → empty. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TEMP TABLE tmp AS SELECT dl.orders.amount AS amt FROM dl.orders; + SELECT amt AS total FROM tmp; + "#; + assert!( + lineage(input).is_empty(), + "staging columns must not be reported as final output; got {:?}", + lineage(input) + ); + } + + #[test] + fn test_infer_lineage_ctas_into_asset_still_inferred() { + // A CTAS whose target IS an asset is the output, so it's still inferred. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.orders_daily AS SELECT dl.orders.amount AS amt FROM dl.orders; + "#; + assert_eq!( + lineage(input), + vec![ColumnLineage { + column: "amt".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "amount".to_string(), + }], + }] + ); + } + + #[test] + fn test_infer_lineage_temp_table_under_use_is_local() { + // A one-part TEMP table name under an active `USE dl` must NOT resolve to + // an asset (`warehouse/tmp`); it's local, so the final SELECT reading it + // can't invent `warehouse/tmp.amt` as a column source for the output. + let input = r#" + -- materialize ducklake://warehouse/final + ATTACH 'ducklake://warehouse' AS dl; + USE dl; + CREATE TEMP TABLE tmp AS SELECT amount AS amt FROM orders; + SELECT amt AS total FROM tmp; + "#; + let got = lineage(input); + assert!( + got.is_empty(), + "temp staging under USE must not leak warehouse/tmp as a source; got {:?}", + got + ); + // And no phantom `warehouse/tmp` asset is recorded. + let assets = parse_assets(input).unwrap().assets; + assert!( + !assets.iter().any(|a| a.path == "warehouse/tmp"), + "temp table must not be recorded as an asset; got {:?}", + assets + ); + } + + #[test] + fn test_infer_lineage_scopes_to_materialize_target() { + // A script with a `// materialize` target plus an AUXILIARY CTAS into a + // different asset: only the materialized target's columns are reported; + // the auxiliary output's columns must not be attributed to it. + let input = r#" + -- materialize ducklake://warehouse/final + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.audit AS SELECT dl.orders.id AS aid FROM dl.orders; + SELECT dl.orders.amount AS total FROM dl.orders; + "#; + assert_eq!( + lineage(input), + vec![ColumnLineage { + column: "total".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "amount".to_string(), + }], + }], + "auxiliary `audit` columns must not appear on the materialized target" + ); + } + + #[test] + fn test_infer_lineage_drops_ambiguous_multi_output() { + // No `// materialize` target and two real CTAS outputs: which asset the + // flat lineage describes is ambiguous, so inference is dropped rather + // than attributed to an arbitrary one. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.a AS SELECT dl.orders.id AS x FROM dl.orders; + CREATE TABLE dl.b AS SELECT dl.orders.amount AS y FROM dl.orders; + "#; + assert!( + lineage(input).is_empty(), + "ambiguous multi-output must drop inference" + ); + } + + #[test] + fn test_infer_lineage_wildcard_yields_nothing() { + // `SELECT *` has no enumerable output columns → no inferred lineage. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.orders_daily AS SELECT * FROM dl.orders; + "#; + assert!(lineage(input).is_empty()); + } + + #[test] + fn test_infer_lineage_resolves_joined_inputs() { + // Columns from BOTH sides of an explicit JOIN must resolve, incl. a + // computed column mixing the two. A bare column is dropped (ambiguous + // across the join) rather than misattributed to the first table. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.orders_daily AS + SELECT o.id, c.region AS cust_region, o.amount + c.discount AS net + FROM dl.orders o + JOIN dl.customers c ON c.id = o.customer_id; + "#; + let got = lineage(input); + assert_eq!( + got, + vec![ + ColumnLineage { + column: "id".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "id".to_string(), + }], + }, + ColumnLineage { + column: "cust_region".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/customers".to_string(), + from_column: "region".to_string(), + }], + }, + ColumnLineage { + column: "net".to_string(), + inputs: vec![ + ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "amount".to_string(), + }, + ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/customers".to_string(), + from_column: "discount".to_string(), + }, + ], + }, + ] + ); + } + + #[test] + fn test_infer_lineage_does_not_descend_into_subqueries() { + // A scalar subquery's bare column (`amount`) belongs to the subquery's + // own FROM, NOT the outer `dl.orders` — it must not be attributed to the + // outer table. The subquery column is left to annotations; the + // passthrough `id` still resolves. + let input = r#" + ATTACH 'ducklake://warehouse' AS dl; + CREATE TABLE dl.orders_daily AS + SELECT dl.orders.id, (SELECT amount FROM dl.other LIMIT 1) AS c + FROM dl.orders; + "#; + let got = lineage(input); + assert_eq!( + got, + vec![ColumnLineage { + column: "id".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "id".to_string(), + }], + }], + "subquery column `c` must be dropped, not misattributed to orders" + ); + } } diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 2103e9937b..2b7f6ecdbf 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -117,6 +117,11 @@ pub struct ParseAssetsOutput { // lines allowed). Drives the worker's post-materialize verifier probes. #[serde(skip_serializing_if = "Vec::is_empty", default)] pub data_tests: Vec, + // `// column <- .[, …]` — declared column-level lineage, + // one entry per output column. Accumulating. Pure metadata: drives the + // column-lineage graph view, executes nothing. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub column_lineage: Vec, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -275,6 +280,39 @@ pub enum DataTest { Custom { path: String }, } +// `// column <- .[, …]` — declared column-level +// lineage: one output column of this script's produced asset and the upstream +// source columns it derives from. A sibling of `DataTest` in the extensible +// annotation family (`docs/pipelines-vs-dbt.md` §3): same parse shape — a head +// token (the output column) then a per-variant tail — but accumulating, one +// line per output column. Unlike `data_test` these are pure metadata: they +// drive the column-lineage graph view, never a runtime probe. +// +// dbt derives column lineage from SQL-AST parsing; Windmill is polyglot +// (Python/TS/Bash/SQL in one DAG), so a uniform AST is not available. The +// annotation is the explicit, language-agnostic declaration — the same +// "annotations are real comments parsed strictly" stance as the rest of the +// pipeline grammar. Body-inferred per-asset column *sets* (`columns` on +// `ParseAssetsResult`) complement it but cannot express column→column edges. +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct ColumnLineage { + // The produced asset's output column this line describes. + pub column: String, + // Upstream source columns it derives from (≥1; malformed refs dropped). + pub inputs: Vec, +} + +// One `.` upstream reference inside a `// column` line. The +// asset URI accepts the default-syntax shorthands (like `// materialize` / +// `// data_test relationships`); the column is the segment after the final +// `.` (so a schema-qualified `warehouse/main.orders.amount` keeps `amount`). +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct ColumnRef { + pub from_kind: AssetKind, + pub from_path: String, + pub from_column: String, +} + // `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger // firing runs the script (current behaviour). `All` = AND: the script // runs only once every partition-bearing input has materialized at the @@ -307,6 +345,7 @@ pub struct PipelineAnnotations { pub retry: Option, pub materialize: Option, pub data_tests: Vec, + pub column_lineage: Vec, } impl ParseAssetsOutput { @@ -332,10 +371,33 @@ impl ParseAssetsOutput { retry: pipeline.retry, materialize: pipeline.materialize, data_tests: pipeline.data_tests, + column_lineage: pipeline.column_lineage, } } } +// Combine column lineage inferred from the body (SQL AST) with lineage declared +// via `// column` annotations. The annotation is the *override*: where both +// describe the same output column, the explicit declaration wins and the +// inferred entry is dropped. Inferred entries are also deduped by output column +// among themselves (first wins). Used by the language asset-parsers so a +// `// column` line can correct a mis-inferred edge without disabling inference +// for the rest of the columns. +pub fn merge_column_lineage( + inferred: Vec, + annotated: Vec, +) -> Vec { + let mut seen: std::collections::HashSet = + annotated.iter().map(|c| c.column.clone()).collect(); + let mut out = annotated; + for c in inferred { + if seen.insert(c.column.clone()) { + out.push(c); + } + } + out +} + #[derive(Debug, Clone, Serialize)] pub struct DelegateToGitRepoDetails { pub resource: String, @@ -680,6 +742,17 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { continue; } + // `// column <- .[, …]` — accumulating column lineage. + // A complete word, so it never swallows a body comment that happens to + // start with `column` followed by non-lineage prose (that has no `<-` + // and is dropped fail-safe). Checked before `on`/asset shorthands. + if let Some(after_kw) = consume_keyword(rest, "column") { + if let Some(spec) = parse_column_lineage_spec(after_kw.trim()) { + out.column_lineage.push(spec); + } + continue; + } + if let Some(after_kw) = consume_keyword(rest, "on") { let spec_text = after_kw.trim(); if spec_text.is_empty() { @@ -841,6 +914,38 @@ fn parse_relationships(s: &str) -> Option { Some(DataTest::Relationships { column, to_kind, to_path: to_path.to_string(), to_column }) } +// Parse a `// column <- [, …]` right-hand side. The head +// (before `<-`) is the output column; the tail is a comma-separated list of +// `.` upstream references. Mirrors `parse_accepted_values`' +// "drop empties, require ≥1" stance: individually malformed refs are dropped +// and the line is kept iff at least one ref parses; a missing `<-`, a non-ident +// output column, or zero valid refs drops the whole line (fail-safe). +fn parse_column_lineage_spec(s: &str) -> Option { + let (out_col, refs) = s.split_once("<-")?; + let column = single_ident(out_col)?; + let inputs: Vec = refs + .split(',') + .filter_map(|r| parse_column_ref(r.trim())) + .collect(); + if inputs.is_empty() { + return None; + } + Some(ColumnLineage { column, inputs }) +} + +// `.` — the referenced column is the segment after the final +// `.`; everything before it is the asset URI (default-syntax shorthands +// enabled, like `// materialize`). Same shape as `parse_relationships`' target. +fn parse_column_ref(s: &str) -> Option { + let (asset_uri, ref_col) = s.rsplit_once('.')?; + let from_column = single_ident(ref_col)?; + let (from_kind, from_path) = parse_asset_syntax(asset_uri.trim(), true)?; + if from_path.is_empty() { + return None; + } + Some(ColumnRef { from_kind, from_path: from_path.to_string(), from_column }) +} + // Parse a `// partitioned [opts]` right-hand side. Recognized kinds: // `daily`, `hourly`, `weekly`, `monthly` (with optional tz/format/start), // and `dynamic key=""` (plus optional format). @@ -1545,4 +1650,132 @@ mod pipeline_annotation_tests { vec![DataTest::Unique { column: "id".to_string() }] ); } + + #[test] + fn column_lineage_basic() { + let code = concat!( + "// column order_total <- ducklake://warehouse/orders.amount, ducklake://warehouse/orders.tax\n", + "// column user_name <- datatable://prod/users.name\n", + ); + let out = parse_pipeline_annotations(code); + assert_eq!( + out.column_lineage, + vec![ + ColumnLineage { + column: "order_total".to_string(), + inputs: vec![ + ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "amount".to_string(), + }, + ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/orders".to_string(), + from_column: "tax".to_string(), + }, + ], + }, + ColumnLineage { + column: "user_name".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::DataTable, + from_path: "prod/users".to_string(), + from_column: "name".to_string(), + }], + }, + ] + ); + } + + #[test] + fn column_lineage_schema_qualified_keeps_last_dot_as_column() { + // The column is the segment after the FINAL dot, so a schema-qualified + // ducklake table (`main.dim_products`) survives intact. + let out = parse_pipeline_annotations( + "// column sku <- ducklake://warehouse/main.dim_products.sku", + ); + assert_eq!( + out.column_lineage, + vec![ColumnLineage { + column: "sku".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "warehouse/main.dim_products".to_string(), + from_column: "sku".to_string(), + }], + }] + ); + } + + #[test] + fn column_lineage_drops_malformed_refs_keeps_valid() { + // `bad_no_dot` has no `.col` and is dropped; the line survives on its + // one valid ref. Mirrors accepted_values' drop-empties-keep-≥1 stance. + let out = parse_pipeline_annotations( + "// column total <- bad_no_dot, datatable://prod/orders.amount", + ); + assert_eq!( + out.column_lineage, + vec![ColumnLineage { + column: "total".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::DataTable, + from_path: "prod/orders".to_string(), + from_column: "amount".to_string(), + }], + }] + ); + } + + #[test] + fn merge_column_lineage_annotation_overrides_inferred() { + let inferred = vec![ + ColumnLineage { + column: "total".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "w/o".to_string(), + from_column: "amount".to_string(), + }], + }, + ColumnLineage { + column: "qty".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::Ducklake, + from_path: "w/o".to_string(), + from_column: "qty".to_string(), + }], + }, + ]; + // Annotation redefines `total` (wins) and leaves `qty` to inference. + let annotated = vec![ColumnLineage { + column: "total".to_string(), + inputs: vec![ColumnRef { + from_kind: AssetKind::DataTable, + from_path: "prod/x".to_string(), + from_column: "grand_total".to_string(), + }], + }]; + let merged = merge_column_lineage(inferred, annotated); + assert_eq!(merged.len(), 2); + // Annotation entry kept first and authoritative. + assert_eq!(merged[0].column, "total"); + assert_eq!(merged[0].inputs[0].from_column, "grand_total"); + // Inferred `qty` survives (no annotation for it); inferred `total` dropped. + assert_eq!(merged[1].column, "qty"); + } + + #[test] + fn column_lineage_malformed_lines_dropped_fail_safe() { + // No arrow, a multi-token output column, and a line whose every ref is + // malformed are all dropped entirely. + let out = parse_pipeline_annotations(concat!( + "// column no_arrow datatable://prod/x.y\n", // missing `<-` + "// column a b <- datatable://prod/x.y\n", // output not a single ident + "// column total <- bad_no_dot\n", // no valid ref + "// column\n", // bare keyword + )); + assert!(out.column_lineage.is_empty()); + } } diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 304a8c3842..4ecbc992bc 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -415,5 +415,96 @@ "retry": null, "data_tests": [{ "type": "unique", "column": "id" }] } + }, + { + "name": "column lineage maps output columns to upstream sources", + "code": "// column order_total <- ducklake://warehouse/orders.amount, ducklake://warehouse/orders.tax\n// column user_name <- datatable://prod/users.name\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "column_lineage": [ + { + "column": "order_total", + "inputs": [ + { + "from_kind": "ducklake", + "from_path": "warehouse/orders", + "from_column": "amount" + }, + { + "from_kind": "ducklake", + "from_path": "warehouse/orders", + "from_column": "tax" + } + ] + }, + { + "column": "user_name", + "inputs": [ + { "from_kind": "datatable", "from_path": "prod/users", "from_column": "name" } + ] + } + ] + } + }, + { + "name": "column lineage keeps duplicate input refs (dedup is a view concern)", + "code": "// column total <- ducklake://warehouse/orders.amount, ducklake://warehouse/orders.amount\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "column_lineage": [ + { + "column": "total", + "inputs": [ + { + "from_kind": "ducklake", + "from_path": "warehouse/orders", + "from_column": "amount" + }, + { + "from_kind": "ducklake", + "from_path": "warehouse/orders", + "from_column": "amount" + } + ] + } + ] + } + }, + { + "name": "column lineage keeps schema-qualified table, drops malformed refs", + "code": "// column sku <- ducklake://warehouse/main.dim_products.sku, bad_no_dot\n// column no_arrow datatable://prod/x.y\n// column total <- bad_no_dot\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "column_lineage": [ + { + "column": "sku", + "inputs": [ + { + "from_kind": "ducklake", + "from_path": "warehouse/main.dim_products", + "from_column": "sku" + } + ] + } + ] + } } ] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 094419300b..883ddebcbc 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -42,6 +42,11 @@ struct Expected { // compared against `serde_json::to_value(got.data_tests)`. Absent === []. #[serde(default)] data_tests: Vec, + // Snake_case `ColumnLineage` serde shape (e.g. {"column":"x","inputs": + // [{"from_kind":"datatable","from_path":"p","from_column":"c"}]}), compared + // against `to_value(got.column_lineage)`. Absent === []. + #[serde(default)] + column_lineage: Vec, } #[derive(Deserialize)] @@ -200,5 +205,13 @@ fn pipeline_annotation_fixtures_match() { serde_json::Value::Array(f.expected.data_tests.clone()), "{ctx}: data tests" ); + + let got_lineage = + serde_json::to_value(&got.column_lineage).expect("column_lineage serialize"); + assert_eq!( + got_lineage, + serde_json::Value::Array(f.expected.column_lineage.clone()), + "{ctx}: column lineage" + ); } } diff --git a/backend/windmill-api-assets/Cargo.toml b/backend/windmill-api-assets/Cargo.toml index c6ba973f80..c6e7676b6b 100644 --- a/backend/windmill-api-assets/Cargo.toml +++ b/backend/windmill-api-assets/Cargo.toml @@ -11,6 +11,7 @@ path = "src/lib.rs" [dependencies] windmill-api-auth.workspace = true windmill-common = { workspace = true, default-features = false } +windmill-parser-sql-asset.workspace = true axum.workspace = true chrono.workspace = true serde.workspace = true diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index d25bb4db5f..29122bd793 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -519,6 +519,17 @@ struct GraphRunnableNode { retry: Option, #[serde(skip_serializing_if = "Vec::is_empty", default)] data_tests: Vec, + // `// column <- .` declared column-level lineage, surfaced + // so the canvas can draw the column-lineage view on deployed nodes (not + // only live drafts). Lockstep with TS `AssetGraphRunnableNode.column_lineage`. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + column_lineage: Vec, + // `// materialize ` target — the asset this script's `column_lineage` + // describes. Lets the column-graph anchor lineage to the exact output asset + // instead of guessing a ducklake write-edge (a multi-output script writes + // several). Absent for scripts with no `// materialize` annotation. + #[serde(skip_serializing_if = "Option::is_none", default)] + materialize_target: Option, // Managed `// materialize` write strategy (`replace` | `append` | `merge`), // absent for non-materializing or `manual` scripts. Surfaced so the asset // panel can tell whether the captured schema can evolve: only whole-table @@ -528,6 +539,14 @@ struct GraphRunnableNode { materialize_strategy: Option, } +// The output asset a producer's column lineage belongs to (the `// materialize` +// target). Kept minimal — the column graph only needs (kind, path) to anchor. +#[derive(Serialize, Debug)] +struct MaterializeTargetNode { + kind: windmill_common::assets::AssetKind, + path: String, +} + // The partition's kind word for the node badge (the full PartitionSpec carries // tz/format/start, which the badge doesn't need). fn partition_kind_word(kind: &windmill_common::assets::PartitionKind) -> &'static str { @@ -738,7 +757,8 @@ async fn asset_graph( // defensive against transient overlaps). let pipeline_member_paths = sqlx::query!( r#" - SELECT DISTINCT ON (path) path AS "path!", content AS "content!" + SELECT DISTINCT ON (path) path AS "path!", content AS "content!", + language AS "language!: windmill_common::scripts::ScriptLang" FROM script WHERE workspace_id = $1 AND auto_kind = 'pipeline' @@ -790,6 +810,34 @@ async fn asset_graph( ) }) .collect(); + // Column-level lineage per member. The annotation-only lineage (already + // parsed above) is the baseline. For DuckDB scripts we additionally run the + // full SQL asset parser to infer output→input column edges from the AST; it + // merges them with the `// column` annotations (annotation wins). If the SQL + // can't be parsed (DuckDB accepts grammar `sqlparser` rejects), we fall back + // to the annotation-only baseline rather than dropping explicit annotations. + let column_lineage_by_path: std::collections::HashMap< + String, + Vec, + > = pipeline_member_paths + .iter() + .map(|r| { + let annotated = || { + annotations_by_path + .get(&r.path) + .map(|a| a.column_lineage.clone()) + .unwrap_or_default() + }; + let lineage = if r.language == windmill_common::scripts::ScriptLang::DuckDb { + windmill_parser_sql_asset::parse_assets(&r.content) + .map(|o| o.column_lineage) + .unwrap_or_else(|_| annotated()) + } else { + annotated() + }; + (r.path.clone(), lineage) + }) + .collect(); let pipeline_member_script_paths: std::collections::HashSet = pipeline_member_paths.into_iter().map(|r| r.path).collect(); let existing_script_paths: std::collections::HashSet = @@ -930,6 +978,19 @@ async fn asset_graph( tag: ann.and_then(|a| a.tag.clone()), retry: ann.and_then(|a| a.retry.clone()), data_tests: ann.map(|a| a.data_tests.clone()).unwrap_or_default(), + // Inferred (DuckDB AST) + annotation column lineage, gated to + // scripts like the badges above. + column_lineage: (usage_kind == AssetUsageKind::Script) + .then(|| column_lineage_by_path.get(&path)) + .flatten() + .cloned() + .unwrap_or_default(), + materialize_target: ann.and_then(|a| a.materialize.as_ref()).map(|m| { + MaterializeTargetNode { + kind: windmill_common::assets::asset_kind_from_parser(m.target_kind), + path: m.target_path.clone(), + } + }), materialize_strategy: ann.and_then(|a| a.materialize.as_ref()).and_then(|m| { if m.manual { None diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index e24b6806a2..c186f59141 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -5,8 +5,8 @@ use sqlx::{PgExecutor, Postgres, Transaction}; use crate::{error, scripts::ScriptHash}; pub use windmill_parser::asset_parser::{ - parse_pipeline_annotations, DataTest, PartitionKind, PipelineAnnotations, RetrySpec, - TriggerSpec, PARTITION_TOKEN, + merge_column_lineage, parse_pipeline_annotations, ColumnLineage, ColumnRef, DataTest, + PartitionKind, PipelineAnnotations, RetrySpec, TriggerSpec, PARTITION_TOKEN, }; pub use windmill_types::assets::*; diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index 5dd93b5a37..4344ef2296 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -48,7 +48,7 @@ Asset-centric, polyglot, annotation-driven, event-aware: |---|---|---| | Data tests | No | **Shipped** (`// data_test`) | | Incremental materializations | No, but pick a philosophy | TODO with design decision | -| Column lineage + docs site | No | Pure TODO | +| Column lineage | No | **Shipped** (`// column`); docs site still TODO | | Snapshots / SCD2 | No | New output kind | | Selective execution grammar | No | UI/CLI surface | | Schema contracts | No, but design metadata model | TODO with design work | @@ -84,10 +84,38 @@ See [Incremental deep-dive](#incremental-deep-dive) below. dbt: SQL-AST parsing for column-level deps; `dbt docs serve` produces a static lineage site with descriptions. -Today: graph is asset-level. `SqlQueryDetails` in the parser -(`backend/parsers/windmill-parser/src/asset_parser.rs:44`) already has a -column map — the scaffolding exists. No `// column` annotation, no docs -surface. Pure TODO; no abstraction stands in the way. +**Shipped**, inferred-first. For DuckDB scripts the column lineage is **derived +automatically from the SQL AST** — `windmill-parser-sql-asset` walks each +output-producing query's projection and maps every output column to the source +columns its expression reads (passthroughs *and* computed columns like +`amount + tax AS total`), resolving each input to its asset via the same +`ATTACH`/alias machinery the asset parser already uses. This is the dbt-style +AST lineage, and it needs no annotation. + +The `// column <- .[, …]` annotation is the +**override / escape hatch**, for the cases inference can't reach: polyglot +transforms (Python/TS/Bash — no SQL AST), dynamic SQL (`${sql.raw(...)}`, flagged +by `SqlQueryDetails.has_raw_interpolation`), or correcting a mis-inferred edge. +Inferred and annotated lineage are merged per output column with the annotation +winning (`merge_column_lineage`). The annotation is the second *extensible* +annotation family after `// data_test` — same head-then-tail parse shape (see +`ColumnLineage`/`ColumnRef` in `asset_parser.rs`) — and is pure metadata: it +drives the graph surface, never a runtime probe. + +Surfaced two ways in the asset graph: a count badge on the +producer→materialized-asset write-edge, and a **transitive column-lineage +trace** (`ColumnLineageTrace.svelte`, over the cross-script graph built by +`columnLineageGraph.ts`) in the asset details pane — select an asset to see its +columns and their full upstream/downstream lineage, click any column to +highlight its complete impact set across the pipeline (forward + backward). + +SQL-AST inference runs both **server-side** (the graph endpoint + deploy via +`parse_assets`, for *deployed* members) and **in the live editor** — the same +parser compiled to WASM (`windmill-parser-wasm-asset`) runs on the open draft's +buffer, and `resolveGraph` merges its `column_lineage` with the buffer's +`// column` annotations under the same annotation-wins precedence, so the draft +preview matches what deploys. The `dbt docs serve`-style static lineage *site* +is still TODO. ### 4. Snapshots / SCD2 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ed9dd1d7ea..bd2619a54e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -79,7 +79,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.728.1", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -878,7 +878,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -890,7 +889,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -901,7 +899,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1417,7 +1414,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1566,7 +1562,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1583,7 +1578,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1600,7 +1594,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1617,7 +1610,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1634,7 +1626,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1651,7 +1642,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1668,7 +1658,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1685,7 +1674,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1702,7 +1690,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1719,7 +1706,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1736,7 +1722,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1753,7 +1738,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1770,7 +1754,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1789,7 +1772,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1806,7 +1788,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2112,7 +2093,6 @@ "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7348,7 +7328,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7883,7 +7863,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7904,7 +7883,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7925,7 +7903,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7946,7 +7923,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7967,7 +7943,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7988,7 +7963,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8009,7 +7983,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8030,7 +8003,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8051,7 +8023,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8072,7 +8043,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8093,7 +8063,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12781,21 +12750,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13535,7 +13489,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -14321,9 +14275,9 @@ } }, "node_modules/windmill-parser-wasm-asset": { - "version": "1.728.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.728.1.tgz", - "integrity": "sha512-73cyU6XM3gYEjFBx3qOKnv+VV1t70eAr6OiT+x0QobjFVNmqZFEdA7ayMYYVCnnix8OZxcsTNEF1hT61w1XiKw==" + "version": "1.740.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.740.0.tgz", + "integrity": "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw==" }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", diff --git a/frontend/package.json b/frontend/package.json index 154497ef76..8bdf440a00 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -154,7 +154,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.728.1", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 54be27f787..f1944c73cc 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -104,6 +104,7 @@ import AssetsDropdownButton from './assets/AssetsDropdownButton.svelte' import { canHavePreprocessor } from '$lib/script_helpers' import { assetEq, type AssetWithAltAccessType } from './assets/lib' + import type { ColumnLineage } from './assets/AssetGraph/parsePipelineAnnotations' import { editor as meditor } from 'monaco-editor' import type { ReviewChangesOpts } from './copilot/chat/monaco-adapter' import GitRepoViewer from './GitRepoViewer.svelte' @@ -171,6 +172,11 @@ lastDeployedCode?: string | undefined disableAi?: boolean assets?: AssetWithAltAccessType[] + // Body-inferred column lineage (DuckDB SQL AST), surfaced alongside + // `assets` so the pipeline editor can render inferred column lineage on + // the live graph. Empty/undefined for non-DuckDB or when the parser + // build predates the inference. + inferredColumnLineage?: ColumnLineage[] modules?: { [key: string]: ScriptModule } | null editorBarRight?: import('svelte').Snippet enablePreprocessorSnippet?: boolean @@ -229,6 +235,7 @@ lastDeployedCode = undefined, disableAi = false, assets = $bindable(), + inferredColumnLineage = $bindable(), modules = $bindable(undefined), editorBarRight, enablePreprocessorSnippet = false, @@ -585,7 +592,13 @@ watch( () => inferAssetsRes.current, () => { - if (!inferAssetsRes.current || inferAssetsRes.current?.status === 'error') return + if (!inferAssetsRes.current || inferAssetsRes.current?.status === 'error') { + // Clear stale lineage on parse error / unset, so a script switch + // whose new body fails to parse can't leave the previous script's + // inferred column lineage bound to the new path. + if (inferredColumnLineage !== undefined) inferredColumnLineage = undefined + return + } let newAssets = inferAssetsRes.current.assets as AssetWithAltAccessType[] for (const asset of newAssets) { const old = assets?.find((a) => assetEq(a, asset)) @@ -593,6 +606,11 @@ } const normalizedAssets = newAssets.length > 0 ? newAssets : undefined if (!deepEqual(assets, normalizedAssets)) assets = normalizedAssets + + const newLineage = inferAssetsRes.current.column_lineage + const normalizedLineage = newLineage && newLineage.length > 0 ? newLineage : undefined + if (!deepEqual(inferredColumnLineage, normalizedLineage)) + inferredColumnLineage = normalizedLineage } ) diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 81d5c58cd5..c13e9abc10 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -219,6 +219,9 @@ // Producer's `// data_test` checks, on the write-edge to the // materialized asset — rendered as a flask badge on the link. data_tests?: NonNullable + // Producer's `// column` declared lineage, on the same write-edge — + // rendered as a columns badge on the link. + column_lineage?: NonNullable } // Graph-id of the script the user just launched (zero-latency hint), @@ -340,6 +343,28 @@ producerTests.set(`${r.usage_kind}:${r.path}`, r.data_tests) } } + // Producer → its `// column` declared lineage, keyed by runnable id, so + // the write-edge to the materialized asset can carry the columns badge — + // the edge *is* the transformation, and the lineage describes its output. + const producerColumnLineage = new Map< + string, + NonNullable + >() + // The `// materialize` target the lineage describes, so the badge lands + // only on that write-edge (a multi-output script writes several ducklake + // tables) — mirrors the anchor logic in `buildColumnGraph`. + const producerMaterializeTarget = new Map< + string, + NonNullable + >() + for (const r of g.runnables) { + if (r.column_lineage && r.column_lineage.length > 0) { + producerColumnLineage.set(`${r.usage_kind}:${r.path}`, r.column_lineage) + } + if (r.materialize_target) { + producerMaterializeTarget.set(`${r.usage_kind}:${r.path}`, r.materialize_target) + } + } for (const r of g.runnables) { const rid = `${r.usage_kind}:${r.path}` // Optimistic badge: the moment a run is launched from this view @@ -413,13 +438,23 @@ // write-edge carries the badge / custom-test nodes — a producer's // other (e.g. S3/datatable) outputs must not show them. const edgeTests = e.asset_kind === 'ducklake' ? producerTests.get(runnableId) : undefined + // Column lineage describes one materialized output, so the badge + // lands only on that asset's write-edge: the declared `// materialize` + // target when known (a multi-output script writes several ducklake + // tables), else the ducklake write-edge as the unambiguous fallback. + const matTarget = producerMaterializeTarget.get(runnableId) + const isOutputEdge = matTarget + ? e.asset_kind === matTarget.kind && e.asset_path === matTarget.path + : e.asset_kind === 'ducklake' + const edgeColumnLineage = isOutputEdge ? producerColumnLineage.get(runnableId) : undefined edges.push({ id: `prod:${runnableId}->${assetId}`, source: runnableId, target: assetId, kind: 'lineage-write', unsaved: e.unsaved, - data_tests: edgeTests + data_tests: edgeTests, + column_lineage: edgeColumnLineage }) // Each custom (`// data_test `) test → its own node // below the asset it validates, with a dashed "tests" edge. @@ -898,7 +933,10 @@ // producer's last-run status (the script fails if any test // fails) tints it green/red; neutral until it has run. data_tests: e.data_tests, - testsRunStatus: e.data_tests?.length ? runStates?.get(e.source)?.status : undefined + testsRunStatus: e.data_tests?.length ? runStates?.get(e.source)?.status : undefined, + // Column-lineage badge on the same write-edge (the link is the + // transformation whose output columns the lineage describes). + column_lineage: e.column_lineage }, animated, label, diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index b69db2e04f..46ebaaf624 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -25,7 +25,13 @@ import type { Schema } from '$lib/common' import type { AssetGraphSelection, PipelineMode } from './types' import PipelineScriptView from './PipelineScriptView.svelte' - import { parsePipelineAnnotations, type PipelineAnnotations } from './parsePipelineAnnotations' + import { + parsePipelineAnnotations, + type ColumnLineage, + type PipelineAnnotations + } from './parsePipelineAnnotations' + import ColumnLineageTrace from './ColumnLineageTrace.svelte' + import { assetColumnNodes, type ColumnLineageGraph } from './columnLineageGraph' import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte' import S3FilePreview from '$lib/components/S3FilePreview.svelte' import DataTablePreview from './DataTablePreview.svelte' @@ -72,7 +78,11 @@ // edges + synthesize asset nodes for drafts whose body has been // edited past the seeded template. Fires on every keystroke that // changes the inferred set. - onAssetsChange?: (scriptPath: string | undefined, assets: AssetWithAltAccessType[]) => void + onAssetsChange?: ( + scriptPath: string | undefined, + assets: AssetWithAltAccessType[], + columnLineage?: ColumnLineage[] + ) => void // Emits the live editor buffer on every keystroke so the parent can // autosave the in-flight content WITHOUT waiting for the pane teardown // (`onDraftPersist`). `onDraftPersist` stays the authoritative commit on @@ -114,6 +124,10 @@ path: string unsaved?: boolean }> + // Pipeline-wide column-lineage graph (built by the parent page from the + // resolved graph). Drives the transitive column-lineage trace shown for a + // selected materialized asset. + selectionColumnGraph?: ColumnLineageGraph // Whether the selected ducklake asset's schema can evolve (whole-table // `replace` producer). Forwarded to the Schema tab: version history when // true, a single fixed-schema view when false. Defaults to true (unknown). @@ -218,6 +232,7 @@ onScriptRenamed, onScriptRemoved, selectionProducers = [], + selectionColumnGraph, schemaCanEvolve = true, runsRefreshKey, runsPendingJobId, @@ -360,6 +375,10 @@ // edges as the user edits the body (e.g. renaming a CREATE TABLE // target updates the output asset node in real time). let liveBodyAssets = $state(undefined) + // Body-inferred column lineage (DuckDB SQL AST), bound out of ScriptEditor + // alongside `liveBodyAssets` and forwarded so the live graph can show + // inferred column lineage on the edited script before it deploys. + let liveColumnLineage = $state(undefined) // Bumped when the runs panel reports a watched job has reached a // terminal state. Drives S3FilePreview's refreshKey so the preview @@ -541,7 +560,8 @@ inPipeline: false, triggerAssets: [], nativeTriggers: [], - dataTests: [] + dataTests: [], + columnLineage: [] } ) $effect(() => { @@ -554,7 +574,7 @@ }) $effect(() => { if (readOnly) return - onAssetsChange?.(script?.path, liveBodyAssets ?? []) + onAssetsChange?.(script?.path, liveBodyAssets ?? [], liveColumnLineage) }) $effect(() => { if (readOnly) return @@ -999,7 +1019,21 @@ {#key selection.path} - +
+ {#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0} +
+ +
+ {/if} +
+ +
+
{/key} {:else}
@@ -1109,6 +1143,7 @@ bind:code={script.content} bind:schema={script.schema} bind:assets={liveBodyAssets} + bind:inferredColumnLineage={liveColumnLineage} {onTestStateChange} {args} /> diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte index 65607cc5a6..c4bef2dd1f 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte @@ -1,8 +1,8 @@ + +
+
+ + Column lineage + {#if targetLabel} + + {targetLabel} + {/if} + + {#if selected} + + {:else if component.size > 1} + click a column to trace + {/if} +
+ + {#if component.size === 0} + No column lineage for this asset. + {:else} +
+ + {#each edges as e (`${e.from}->${e.to}`)} + {@const hot = traced !== undefined && traced.has(e.from) && traced.has(e.to)} + {@const cold = traced !== undefined && !hot} + + {/each} + + + {#each [...component] as id (id)} + {@const n = graph.nodes.get(id)} + {@const p = pos.get(id)} + {#if n && p} + {@const isSeed = seedSet.has(id)} + + {/if} + {/each} +
+ {/if} +
diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts new file mode 100644 index 0000000000..6842dcfd86 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'vitest' +import type { AssetGraphResponse } from './types' +import { + buildColumnGraph, + colNodeId, + traceColumn, + connectedComponent, + assetColumnNodes, + computeDepths +} from './columnLineageGraph' + +// Two scripts chained through an intermediate ducklake table: +// s1: orders.amount -> staging.amt +// s2: staging.amt -> daily.total +// s2: customers.name -> daily.cust (a second source into the sink) +function chainGraph(): AssetGraphResponse { + return { + assets: [], + triggers: [], + runnables: [ + { + path: 's1', + usage_kind: 'script', + column_lineage: [ + { + column: 'amt', + inputs: [{ from_kind: 'ducklake', from_path: 'wh/orders', from_column: 'amount' }] + } + ] + }, + { + path: 's2', + usage_kind: 'script', + column_lineage: [ + { + column: 'total', + inputs: [{ from_kind: 'ducklake', from_path: 'wh/staging', from_column: 'amt' }] + }, + { + column: 'cust', + inputs: [{ from_kind: 'ducklake', from_path: 'wh/customers', from_column: 'name' }] + } + ] + } + ], + edges: [ + { + runnable_path: 's1', + runnable_kind: 'script', + asset_kind: 'ducklake', + asset_path: 'wh/staging', + access_type: 'w' + }, + { + runnable_path: 's2', + runnable_kind: 'script', + asset_kind: 'ducklake', + asset_path: 'wh/daily', + access_type: 'w' + } + ] + } +} + +const ORDERS_AMOUNT = colNodeId('ducklake', 'wh/orders', 'amount') +const STAGING_AMT = colNodeId('ducklake', 'wh/staging', 'amt') +const DAILY_TOTAL = colNodeId('ducklake', 'wh/daily', 'total') +const DAILY_CUST = colNodeId('ducklake', 'wh/daily', 'cust') +const CUSTOMERS_NAME = colNodeId('ducklake', 'wh/customers', 'name') + +describe('buildColumnGraph', () => { + it('stitches per-script lineage into a transitive graph via shared columns', () => { + const g = buildColumnGraph(chainGraph()) + // orders.amount feeds staging.amt feeds daily.total + expect(g.up.get(STAGING_AMT)).toEqual(new Set([ORDERS_AMOUNT])) + expect(g.up.get(DAILY_TOTAL)).toEqual(new Set([STAGING_AMT])) + expect(g.down.get(ORDERS_AMOUNT)).toEqual(new Set([STAGING_AMT])) + expect(g.down.get(STAGING_AMT)).toEqual(new Set([DAILY_TOTAL])) + }) + + it('anchors to the // materialize target, not a guessed write-edge', () => { + const graph = chainGraph() + // s1 declares its materialize target and also has unordered extra + // ducklake writes; the lineage must anchor to the declared target. + const s1 = graph.runnables.find((r) => r.path === 's1')! + s1.materialize_target = { kind: 'ducklake', path: 'wh/staging' } + graph.edges.unshift({ + runnable_path: 's1', + runnable_kind: 'script', + asset_kind: 'ducklake', + asset_path: 'wh/other', + access_type: 'w' + }) + const g = buildColumnGraph(graph) + expect(g.nodes.has(STAGING_AMT)).toBe(true) // anchored to the declared target + expect(g.nodes.has(colNodeId('ducklake', 'wh/other', 'amt'))).toBe(false) + }) + + it('falls back to a ducklake write-edge when there is no materialize target', () => { + // chainGraph's runnables carry no materialize_target, so s1's lineage is + // anchored via its (single) ducklake write-edge. + const g = buildColumnGraph(chainGraph()) + expect(g.nodes.has(STAGING_AMT)).toBe(true) + }) + + it('node ids are collision-proof across `#` / `:` in paths and columns', () => { + // A delimiter-concatenated id would merge these; the JSON-encoded id must not. + expect(colNodeId('ducklake', 'a#b', 'c')).not.toBe(colNodeId('ducklake', 'a', 'b#c')) + expect(colNodeId('ducklake', 'a:b', 'c')).not.toBe(colNodeId('ducklake', 'a', 'b:c')) + }) + + it('skips producers with no ducklake output asset (columns unanchorable)', () => { + const graph = chainGraph() + graph.edges = graph.edges.filter((e) => e.runnable_path !== 's1') // s1 loses its output edge + const g = buildColumnGraph(graph) + // staging.amt is no longer produced as a node by s1... + expect(g.up.has(STAGING_AMT)).toBe(false) + // ...but s2 still anchors daily.total ← staging.amt (staging.amt as a source). + expect(g.up.get(DAILY_TOTAL)).toEqual(new Set([STAGING_AMT])) + }) +}) + +describe('traceColumn', () => { + it('returns the full upstream + downstream impact set of a source column', () => { + const g = buildColumnGraph(chainGraph()) + // from the root source, the whole chain downstream is impacted + expect(traceColumn(ORDERS_AMOUNT, g)).toEqual( + new Set([ORDERS_AMOUNT, STAGING_AMT, DAILY_TOTAL]) + ) + }) + + it('traces backward from a sink to every contributing source', () => { + const g = buildColumnGraph(chainGraph()) + expect(traceColumn(DAILY_TOTAL, g)).toEqual(new Set([DAILY_TOTAL, STAGING_AMT, ORDERS_AMOUNT])) + // the sibling output `cust` and its source are NOT in total's trace + expect(traceColumn(DAILY_TOTAL, g).has(DAILY_CUST)).toBe(false) + expect(traceColumn(DAILY_TOTAL, g).has(CUSTOMERS_NAME)).toBe(false) + }) + + it('traces an intermediate column both directions', () => { + const g = buildColumnGraph(chainGraph()) + expect(traceColumn(STAGING_AMT, g)).toEqual(new Set([STAGING_AMT, ORDERS_AMOUNT, DAILY_TOTAL])) + }) +}) + +describe('connectedComponent + depths', () => { + it('collects the neighborhood of an asset and lays it out by hop depth', () => { + const g = buildColumnGraph(chainGraph()) + const seeds = assetColumnNodes(g, 'ducklake', 'wh/daily') // the sink asset + expect(new Set(seeds)).toEqual(new Set([DAILY_TOTAL, DAILY_CUST])) + const comp = connectedComponent(seeds, g) + expect(comp).toEqual( + new Set([DAILY_TOTAL, DAILY_CUST, STAGING_AMT, ORDERS_AMOUNT, CUSTOMERS_NAME]) + ) + const depths = computeDepths(comp, g) + expect(depths.get(ORDERS_AMOUNT)).toBe(0) + expect(depths.get(STAGING_AMT)).toBe(1) + expect(depths.get(DAILY_TOTAL)).toBe(2) + expect(depths.get(CUSTOMERS_NAME)).toBe(0) + expect(depths.get(DAILY_CUST)).toBe(1) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts new file mode 100644 index 0000000000..50fa1c4e7b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts @@ -0,0 +1,169 @@ +import type { AssetKind } from '$lib/gen' +import type { AssetGraphResponse } from './types' + +// A node in the column-level lineage graph: one column of one asset. +export type ColumnNode = { kind: AssetKind; path: string; column: string } +export type ColumnNodeId = string + +// Collision-proof node id — JSON-encoded tuple, so a `#`/`:` inside a path or +// (quoted) column name can't merge two distinct columns into one node. +export function colNodeId(kind: AssetKind, path: string, column: string): ColumnNodeId { + return JSON.stringify([kind, path, column]) +} + +// The pipeline-wide column-lineage graph, stitched across every producer. Each +// producer's `column_lineage` contributes single-hop edges (its output column ← +// its source columns); shared (asset,column) nodes chain those hops into the +// full transitive graph (`orders.amount → staging.amt → daily.total`). +export type ColumnLineageGraph = { + nodes: Map + // outputColumn → the source columns it derives from (walk upstream). + up: Map> + // sourceColumn → the output columns derived from it (walk downstream). + down: Map> +} + +// Build the column graph from a resolved asset graph. A producer's +// `column_lineage` describes the columns of the asset it materializes; that +// output asset is the ducklake target it writes (v1 materialize target), found +// from its write-edge. Producers without a known ducklake output are skipped +// (their columns can't be anchored to an asset node). +export function buildColumnGraph(graph: AssetGraphResponse): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + + const addNode = (n: ColumnNode): ColumnNodeId => { + const id = colNodeId(n.kind, n.path, n.column) + if (!nodes.has(id)) nodes.set(id, n) + return id + } + const addEdge = (src: ColumnNodeId, out: ColumnNodeId) => { + if (src === out) return + ;(up.get(out) ?? up.set(out, new Set()).get(out)!).add(src) + ;(down.get(src) ?? down.set(src, new Set()).get(src)!).add(out) + } + + // The output asset a runnable's `column_lineage` describes. The declared + // `// materialize` target is authoritative (a multi-output script writes + // several ducklake tables, and the deployed write-edges are unordered, so + // picking "a" write-edge can anchor to the wrong asset). Fall back to a + // ducklake write-edge only for producers with no materialize annotation + // (e.g. a literal single-output CTAS). + const outputAsset = new Map() + for (const r of graph.runnables ?? []) { + if (r.materialize_target) { + outputAsset.set(`${r.usage_kind}:${r.path}`, r.materialize_target) + } + } + for (const e of graph.edges ?? []) { + const access = e.access_type ?? 'r' + const key = `${e.runnable_kind}:${e.runnable_path}` + if ( + (access === 'w' || access === 'rw') && + e.asset_kind === 'ducklake' && + !outputAsset.has(key) + ) { + outputAsset.set(key, { kind: e.asset_kind, path: e.asset_path }) + } + } + + for (const r of graph.runnables ?? []) { + const lineage = r.column_lineage + if (!lineage || lineage.length === 0) continue + const out = outputAsset.get(`${r.usage_kind}:${r.path}`) + if (!out) continue + for (const cl of lineage) { + const outId = addNode({ kind: out.kind, path: out.path, column: cl.column }) + for (const inp of cl.inputs) { + const srcId = addNode({ + kind: inp.from_kind, + path: inp.from_path, + column: inp.from_column + }) + addEdge(srcId, outId) + } + } + } + + return { nodes, up, down } +} + +// Every node reachable from `start` by following `adj` (transitive closure, +// excluding `start` itself). Iterative to avoid deep-recursion limits. +function reach(start: ColumnNodeId, adj: Map>): Set { + const seen = new Set() + const stack = [start] + while (stack.length) { + const n = stack.pop()! + for (const m of adj.get(n) ?? []) { + if (!seen.has(m)) { + seen.add(m) + stack.push(m) + } + } + } + return seen +} + +// The full transitive trace of a column: itself + all upstream ancestors + all +// downstream descendants. This is the impact set — "everything that feeds, or +// is fed by, this column". +export function traceColumn(id: ColumnNodeId, g: ColumnLineageGraph): Set { + const out = new Set([id]) + for (const a of reach(id, g.up)) out.add(a) + for (const d of reach(id, g.down)) out.add(d) + return out +} + +// The connected neighborhood of a set of seed columns (an asset's columns): +// the seeds plus everything upstream and downstream of any of them. This is the +// subgraph the trace view renders around a selected asset. +export function connectedComponent( + seeds: ColumnNodeId[], + g: ColumnLineageGraph +): Set { + const out = new Set() + for (const s of seeds) { + if (!g.nodes.has(s)) continue + out.add(s) + for (const a of reach(s, g.up)) out.add(a) + for (const d of reach(s, g.down)) out.add(d) + } + return out +} + +// All column-node ids belonging to one asset (its seed set for a trace). +export function assetColumnNodes( + g: ColumnLineageGraph, + kind: AssetKind, + path: string +): ColumnNodeId[] { + const ids: ColumnNodeId[] = [] + for (const [id, n] of g.nodes) if (n.kind === kind && n.path === path) ids.push(id) + return ids +} + +// Longest-path depth of each node within `ids`, sources at depth 0 and depth +// increasing downstream — so a left→right layout reads upstream→downstream. +// Cycle-guarded (lineage is a DAG, but be defensive). +export function computeDepths( + ids: Set, + g: ColumnLineageGraph +): Map { + const depth = new Map() + const visiting = new Set() + const d = (id: ColumnNodeId): number => { + const memo = depth.get(id) + if (memo !== undefined) return memo + if (visiting.has(id)) return 0 + visiting.add(id) + let m = 0 + for (const u of g.up.get(id) ?? []) if (ids.has(u)) m = Math.max(m, d(u) + 1) + visiting.delete(id) + depth.set(id, m) + return m + } + for (const id of ids) d(id) + return depth +} diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts index 008b903ad9..3eb1e4e574 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts @@ -19,7 +19,8 @@ const ASSERTED_TS_FIELDS: Record = { tag: true, retry: true, materialize: true, - dataTests: true + dataTests: true, + columnLineage: true } // Parser-parity guard: this TS parser (drives the live graph preview) and @@ -71,6 +72,9 @@ type Fixture = { // corpus drives both sides. The TS parser emits this shape verbatim // (snake_case fields), so the comparison is 1:1. Absent === []. data_tests?: Array> + // Snake_case `ColumnLineage` serde shape — TS parser emits it verbatim, + // so the comparison is 1:1. Absent === []. + column_lineage?: Array> } } @@ -161,6 +165,8 @@ describe('parsePipelineAnnotations matches the shared Rust fixture corpus', () = } expect(got.dataTests, 'data tests').toEqual(f.expected.data_tests ?? []) + + expect(got.columnLineage, 'column lineage').toEqual(f.expected.column_lineage ?? []) }) } }) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts index 15b12b959d..5f4b063e3f 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import { + mergeColumnLineage, + parsePipelineAnnotations, + type ColumnLineage +} from './parsePipelineAnnotations' // Unit-test the TS mirror of the backend `parse_pipeline_annotations`. The // two implementations MUST stay behaviorally identical — these tests are @@ -131,3 +135,29 @@ describe('parsePipelineAnnotations: data_upload', () => { expect(out.nativeTriggers).toEqual([]) }) }) + +describe('mergeColumnLineage', () => { + const ref = (path: string, col: string): ColumnLineage['inputs'][number] => ({ + from_kind: 'ducklake', + from_path: path, + from_column: col + }) + + it('annotation wins per output column; inferred fills the rest (mirrors Rust)', () => { + const inferred: ColumnLineage[] = [ + { column: 'total', inputs: [ref('w/o', 'amount')] }, + { column: 'qty', inputs: [ref('w/o', 'qty')] } + ] + const annotated: ColumnLineage[] = [{ column: 'total', inputs: [ref('w/manual', 'grand')] }] + const merged = mergeColumnLineage(inferred, annotated) + expect(merged).toEqual([ + { column: 'total', inputs: [ref('w/manual', 'grand')] }, // annotation, first + authoritative + { column: 'qty', inputs: [ref('w/o', 'qty')] } // inferred, not overridden + ]) + }) + + it('returns annotations unchanged when there is no inferred lineage', () => { + const annotated: ColumnLineage[] = [{ column: 'a', inputs: [ref('w/o', 'a')] }] + expect(mergeColumnLineage([], annotated)).toEqual(annotated) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index a788acdbe0..ccbc746b04 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -113,6 +113,44 @@ export type DataTest = } | { type: 'custom'; path: string } +// `// column <- .[, …]` — declared column-level +// lineage: one output column and the upstream source columns it derives from. +// See backend `ColumnLineage`. A sibling of `DataTest` in the extensible +// annotation family — same parse shape, accumulating (one line per output +// column) — but pure metadata: drives the column-lineage graph view, runs no +// probe. Field names are snake_case to match the Rust `ColumnLineage` serde +// output verbatim, so the live-draft parse and the backend graph endpoint +// (deployed nodes) produce wire-identical shapes. +export type ColumnRef = { + from_kind: AssetKind + from_path: string + from_column: string +} +export type ColumnLineage = { + column: string + inputs: ColumnRef[] +} + +// Combine body-inferred column lineage with `// column` annotations, the +// annotation winning per output column. Mirrors the Rust `merge_column_lineage` +// (`asset_parser.rs`) so the live-draft preview matches what deploys: the +// backend already merges inferred + annotated server-side, and the live graph +// must apply the same precedence to the WASM-inferred lineage. +export function mergeColumnLineage( + inferred: ColumnLineage[], + annotated: ColumnLineage[] +): ColumnLineage[] { + const seen = new Set(annotated.map((c) => c.column)) + const out = [...annotated] + for (const c of inferred) { + if (!seen.has(c.column)) { + seen.add(c.column) + out.push(c) + } + } + return out +} + export type PipelineAnnotations = { inPipeline: boolean triggerAssets: PipelineTriggerAsset[] @@ -125,6 +163,8 @@ export type PipelineAnnotations = { materialize?: MaterializeSpec // `// data_test …` — accumulating data-quality checks (multiple lines). dataTests: DataTest[] + // `// column <- .[, …]` — accumulating column lineage. + columnLineage: ColumnLineage[] } // Tokenize a `key=value [key="quoted value"] ...` option string. Bare @@ -261,6 +301,37 @@ function parseRelationships(s: string): DataTest | undefined { } } +// `.` upstream reference. The column is the segment after the +// final `.`; the rest is the asset URI (default-syntax shorthands enabled). +// Mirrors Rust `parse_column_ref`. +function parseColumnRef(s: string): ColumnRef | undefined { + const dot = s.lastIndexOf('.') + if (dot < 0) return undefined + const fromColumn = singleIdent(s.slice(dot + 1)) + if (!fromColumn) return undefined + const asset = parseAssetSyntaxDefault(s.slice(0, dot).trim()) + if (!asset || asset.path === '') return undefined + return { from_kind: asset.kind, from_path: asset.path, from_column: fromColumn } +} + +// ` <- [, …]`. Individually malformed refs are dropped; the +// line is kept iff ≥1 ref parses (mirrors `parseAcceptedValues`). A missing +// `<-`, a non-ident output column, or zero valid refs drops the line. +// Mirrors Rust `parse_column_lineage_spec`. +function parseColumnLineageSpec(s: string): ColumnLineage | undefined { + const arrow = s.indexOf('<-') + if (arrow < 0) return undefined + const column = singleIdent(s.slice(0, arrow)) + if (!column) return undefined + const inputs = s + .slice(arrow + 2) + .split(',') + .map((r) => parseColumnRef(r.trim())) + .filter((r): r is ColumnRef => r !== undefined) + if (inputs.length === 0) return undefined + return { column, inputs } +} + // Parse a `// data_test …` right-hand side into one `DataTest`. The // leading token selects the variant; anything not a built-in keyword is the // `custom` escape hatch (a single script-path token). Returns `undefined` for @@ -398,7 +469,8 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { inPipeline: false, triggerAssets: [], nativeTriggers: [], - dataTests: [] + dataTests: [], + columnLineage: [] } for (const rawLine of code.split('\n')) { @@ -475,6 +547,15 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { continue } + // `column` is a complete word; a body comment that merely starts with + // `column` has no `<-` and is dropped fail-safe. Accumulates. + const afterColumn = consumeKeyword(inner, 'column') + if (afterColumn !== undefined) { + const spec = parseColumnLineageSpec(afterColumn.trim()) + if (spec) out.columnLineage.push(spec) + continue + } + const afterOn = consumeKeyword(inner, 'on') if (afterOn !== undefined) { const specText = afterOn.trim() diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 0521051375..b7c82822fa 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -1,5 +1,10 @@ import type { AssetGraphResponse, NativeTriggerKind } from './types' -import { parsePipelineAnnotations, type PipelineAnnotations } from './parsePipelineAnnotations' +import { + mergeColumnLineage, + parsePipelineAnnotations, + type ColumnLineage, + type PipelineAnnotations +} from './parsePipelineAnnotations' import { extractWrites, extractReads, @@ -20,7 +25,12 @@ export type ResolveGraphInput = { /** In-flight drafts keyed by script path. */ drafts: Map /** Body assets inferred for the currently-open script (live keystrokes). */ - liveBodyAssets: { scriptPath: string | undefined; assets: AssetWithAltAccessType[] } + liveBodyAssets: { + scriptPath: string | undefined + assets: AssetWithAltAccessType[] + /** Body-inferred column lineage (DuckDB SQL AST) for the open script. */ + columnLineage?: ColumnLineage[] + } /** Pipeline annotations parsed from the currently-open buffer. */ liveAnnotations: { scriptPath: string | undefined; annotations: PipelineAnnotations } /** Sticky session caches of inferred body writes/reads per script path. */ @@ -222,6 +232,19 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { for (const [path, d] of drafts) { const parsed = parsePipelineAnnotations(d.script.content) + // For the open script, fold in the WASM-inferred column lineage (DuckDB + // SQL AST) under the same annotation-wins precedence the backend applies + // on deploy, so the live preview matches what deploys. Only the open + // script carries live inference (`liveBodyAssets`); other drafts stay + // annotation-only until they deploy (the backend infers then). + const inferredCL = + path === liveBodyAssets.scriptPath ? (liveBodyAssets.columnLineage ?? []) : [] + const mergedCL = mergeColumnLineage(inferredCL, parsed.columnLineage) + // The `// materialize` target this draft's column lineage describes, so + // the column graph anchors to it rather than guessing a write-edge. + const materializeTarget = parsed.materialize + ? { kind: parsed.materialize.targetKind, path: parsed.materialize.targetPath } + : undefined // A draft can coexist with a base entry — during save the refetch // lands before drafts cleanup, and a user re-editing a deployed // script also produces both. In that case we mutate the existing @@ -240,15 +263,20 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { tag: parsed.tag, retry: parsed.retry, data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, + column_lineage: mergedCL.length > 0 ? mergedCL : undefined, + materialize_target: materializeTarget, unsaved: true }) } else { // Refresh annotation-derived badges from the live parse too, so - // adding/removing `// data_test` lines on an already-deployed script - // updates the badge immediately (not only after redeploy/refetch). + // adding/removing `// data_test` / `// column` lines on an + // already-deployed script updates the badge immediately (not only + // after redeploy/refetch). runnables[baseIdx] = { ...runnables[baseIdx], data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, + column_lineage: mergedCL.length > 0 ? mergedCL : undefined, + materialize_target: materializeTarget, unsaved: true } } diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index c942d28934..9276f500af 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -1,5 +1,5 @@ import type { AssetKind } from '$lib/gen' -import type { DataTest } from './parsePipelineAnnotations' +import type { ColumnLineage, DataTest } from './parsePipelineAnnotations' export type GraphUsageKind = 'script' | 'flow' @@ -33,6 +33,14 @@ export interface AssetGraphRunnableNode { // asset. Surfaced as a count badge (with a per-test breakdown in the title) // so test coverage is visible on the node without opening the pane. data_tests?: DataTest[] + // `// column <- .` declared column-level lineage for this + // script's materialized output. Surfaced as a count badge on the write-edge + // and as a column-to-column diagram in the asset details pane. + column_lineage?: ColumnLineage[] + // `// materialize ` target — the asset `column_lineage` describes. + // Lets the column graph anchor lineage to the exact output instead of + // guessing a ducklake write-edge (a multi-output script writes several). + materialize_target?: { kind: AssetKind; path: string } // Managed `// materialize` write strategy. Absent for non-materializing or // `manual` scripts. Used (with `partition_kind`) to decide whether a // produced asset's schema can evolve: only whole-table `replace` can, since diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 7c8b83a64f..a70dd7c745 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -68,6 +68,7 @@ import wasmUrlWac from 'windmill-parser-wasm-wac/windmill_parser_wasm_bg.wasm?ur import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from 'windmill-utils-internal' import { type AssetWithAccessType } from './components/assets/lib.js' +import { type ColumnLineage } from './components/assets/AssetGraph/parsePipelineAnnotations' const loadSchemaLastRun = writable< | [ @@ -169,6 +170,10 @@ type InferAssetsResult = assets: AssetWithAccessType[] sql_queries?: InferAssetsSqlQueryDetails[] columns?: Record + // Body-inferred column lineage (DuckDB SQL AST). Present once the + // `windmill-parser-wasm-asset` package is rebuilt with the inference; + // the spread below already forwards it from the parser output. + column_lineage?: ColumnLineage[] } | { status: 'error' diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 3548ba6017..3056245e51 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -33,8 +33,13 @@ import PipelineModeToggle from '$lib/components/assets/AssetGraph/PipelineModeToggle.svelte' import { parsePipelineAnnotations, + type ColumnLineage, type PipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { + buildColumnGraph, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { resolveGraph } from '$lib/components/assets/AssetGraph/resolveGraph' import { computeDownstreamClosure, @@ -472,7 +477,8 @@ inPipeline: false, triggerAssets: [], nativeTriggers: [], - dataTests: [] + dataTests: [], + columnLineage: [] } }) @@ -485,6 +491,7 @@ let liveBodyAssets = $state<{ scriptPath: string | undefined assets: AssetWithAltAccessType[] + columnLineage?: ColumnLineage[] }>({ scriptPath: undefined, assets: [] }) // The open draft's live editor buffer, emitted by the pane on every @@ -502,7 +509,13 @@ const EMPTY_LIVE_ASSETS = { scriptPath: undefined, assets: [] } const EMPTY_LIVE_ANNOTATIONS = { scriptPath: undefined, - annotations: { inPipeline: false, triggerAssets: [], nativeTriggers: [], dataTests: [] } + annotations: { + inPipeline: false, + triggerAssets: [], + nativeTriggers: [], + dataTests: [], + columnLineage: [] + } } // Reset every live editor overlay (annotations / body assets / content) @@ -1190,14 +1203,18 @@ ) { liveAnnotations = { scriptPath, annotations } } - function handleAssetsChange(scriptPath: string | undefined, assets: AssetWithAltAccessType[]) { + function handleAssetsChange( + scriptPath: string | undefined, + assets: AssetWithAltAccessType[], + columnLineage?: ColumnLineage[] + ) { // Single update site for the live overlay. `inferredWritesByPath` // / `inferredReadsByPath` are now derived from `liveBodyAssets` // (for the open script) + `inferredAssetsByPath` (prefetched // snapshot for every other script), so we don't have to write // into those caches here — the derive picks up our update on the // next reactive tick. - liveBodyAssets = { scriptPath, assets } + liveBodyAssets = { scriptPath, assets, columnLineage } } function handleContentChange(scriptPath: string | undefined, content: string) { liveContent = { scriptPath, content } @@ -1958,6 +1975,27 @@ .map((e) => ({ kind: e.runnable_kind, path: e.runnable_path, unsaved: e.unsaved })) }) + // Empty graph reused when the trace isn't shown (no ducklake-asset selection, + // or a draft is actively edited) so the pane blanks out like the other + // selection overlays and `buildColumnGraph` doesn't run. + const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() + } + // Pipeline-wide column-lineage graph, stitched across every producer's + // (inferred + annotated) `column_lineage` and the asset write-edges. Drives + // the transitive column trace in the details pane. Built from `displayGraph` + // — the exact graph the canvas renders — so the trace matches it: draft + // overlays in edit / show-drafts, deployed-only in plain View. Gated to a + // ducklake-asset selection so it isn't rebuilt on every editor keystroke when + // the trace UI isn't even shown. + let columnGraph = $derived( + selection?.kind === 'asset' && selection.asset_kind === 'ducklake' + ? buildColumnGraph(displayGraph) + : EMPTY_COLUMN_GRAPH + ) + // Whether the selected ducklake asset's captured schema can *evolve* (drives // the asset panel's Schema tab: version history vs. a single fixed schema). // Only a whole-table `replace` producer (CREATE OR REPLACE) can change @@ -2552,6 +2590,7 @@ onRunByPath={runByPathLegit} selection={activeDraft ? undefined : selection} selectionProducers={activeDraft ? [] : selectionProducers} + selectionColumnGraph={activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} {schemaCanEvolve} {runsRefreshKey} {runsPendingJobId} From c479afab8ebceccbee050e923dc5c27a6712ea62 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 28 Jun 2026 11:25:59 +0200 Subject: [PATCH 117/273] fix: redeploy older app version from deployment history (#9826) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: redeploy older app version from deployment history Co-Authored-By: Claude Opus 4.8 (1M context) * fix: apply restored app version to low-code editor on redeploy Redeploying an older app version from Deployment History fired the restore callback (toast shown) but the canvas kept displaying the current version, and Deploy then shipped that current value. AppEditor seeds its working state from `appDraftHandle.draft ?? app`, preferring the per-path autosave over the freshly restored `app` prop. The remount triggered by the restore therefore re-read the stale pre-restore draft. `reloadDeployed` already clears the draft before remounting for the reset-to-deployed flow; `onRestore` was missing the same step. Drop the autosave in `onRestore` so the remounted editor seeds from the restored value. Raw apps are unaffected: RawAppEditor binds `files` directly (no draft precedence), and `extractRawApp` mutates that bound state in place. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(raw-apps): convert savedNewAppPath event forwarding to a callback prop `svelte-check` (CI `npm check`) failed with one error: forwarding the `savedNewAppPath` createEventDispatcher event through the runes-mode RawAppEditor → RawAppEditorHeader chain types as "not assignable to never". This is the same legacy-forwarding-through-runes pattern already removed for `restore` in this PR — `on:savedNewAppPath` would likewise be dropped at runtime, breaking navigation to the new path after a deploy that renames the app. Replace the `on:savedNewAppPath` forwarding with an `onSavedNewAppPath` callback prop threaded page → RawAppEditor → RawAppEditorHeader, matching `onRestore`. The header now invokes the callback instead of dispatching, and its now-unused createEventDispatcher is removed. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../components/apps/editor/AppEditor.svelte | 5 +++-- .../apps/editor/AppEditorHeader.svelte | 9 ++++++-- frontend/src/lib/components/apps/types.ts | 4 ++++ .../components/raw_apps/RawAppEditor.svelte | 16 +++++++++++--- .../raw_apps/RawAppEditorHeader.svelte | 22 +++++++++++++------ .../(logged)/apps/edit/[...path]/+page.svelte | 11 +++++++--- .../apps_raw/edit/[...path]/+page.svelte | 12 +++++----- 7 files changed, 56 insertions(+), 23 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 0cffdc2400..f6e3245c2c 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -83,7 +83,8 @@ onResetToDeployed, loadedFromDraft = false, othersDraftsCount = 0, - onOpenOthersDrafts + onOpenOthersDrafts, + onRestore }: AppEditorProps = $props() migrateApp(untrack(() => app)) @@ -890,7 +891,7 @@ {loadedFromDraft} {othersDraftsCount} {onOpenOthersDrafts} - on:restore + {onRestore} {policy} {fromHub} bind:this={appEditorHeader} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index 1de186c301..0ed389f52c 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -112,6 +112,10 @@ loadedFromDraft?: boolean othersDraftsCount?: number onOpenOthersDrafts?: () => void + // Restoring an older deployment from the history drawer. A callback prop + // (not `on:restore` forwarding): forwarding a `createEventDispatcher` + // event up through these runes-mode components silently drops it. + onRestore?: (restoredApp: any) => void } let { @@ -137,7 +141,8 @@ onResetToDeployed, loadedFromDraft = false, othersDraftsCount = 0, - onOpenOthersDrafts + onOpenOthersDrafts, + onRestore }: Props = $props() /** Mirror of the path the user is editing in the pen popover. Initialized @@ -862,7 +867,7 @@ (historyBrowserDrawerOpen = false)}> - + onRestore?.(e.detail)} appPath={$appPath} /> diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index de34f4676d..d8ac000ba5 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -176,6 +176,10 @@ export interface AppEditorProps { loadedFromDraft?: boolean othersDraftsCount?: number onOpenOthersDrafts?: () => void + // Restoring an older deployment from the history drawer. Threaded through + // AppEditorHeader as a callback prop rather than `on:restore` forwarding, + // which does not propagate through these runes-mode components. + onRestore?: (restoredApp: any) => void } export type App = { diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index c8d15e6e7e..18f9c1af0c 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -121,6 +121,14 @@ onOpenOthersDrafts?: () => void onRuntimeLogRequester?: (requester: RawAppRuntimeLogRequester | undefined) => void onRunsProvider?: (provider: RawAppRunsProvider | undefined) => void + // Restoring an older deployment from the history drawer. A callback prop + // (not `on:restore` forwarding): forwarding a `createEventDispatcher` + // event up through these runes-mode components silently drops it. + onRestore?: (restoredApp: any) => void + // Deploy created the app at a new path; the page navigates to it. Callback + // prop for the same reason as `onRestore` — `on:savedNewAppPath` forwarding + // through these runes-mode components is dropped. + onSavedNewAppPath?: (path: string) => void } let { @@ -148,7 +156,9 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRuntimeLogRequester = undefined, - onRunsProvider = undefined + onRunsProvider = undefined, + onRestore, + onSavedNewAppPath }: Props = $props() export const version: number | undefined = undefined @@ -1596,8 +1606,8 @@ bind:savedApp bind:summary bind:pendingDraftPath - on:restore - on:savedNewAppPath + {onRestore} + {onSavedNewAppPath} {policy} {diffDrawer} {newApp} diff --git a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte index 3963e1b16d..9853fb69bd 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte @@ -26,7 +26,7 @@ Undo, WandSparkles } from 'lucide-svelte' - import { createEventDispatcher, untrack } from 'svelte' + import { untrack } from 'svelte' import { orderedJsonStringify, type Value, replaceFalseWithUndefined } from '../../utils' import { random_adj } from '$lib/components/random_positive_adjetive' @@ -151,6 +151,14 @@ loadedFromDraft?: boolean othersDraftsCount?: number onOpenOthersDrafts?: () => void + // Restoring an older deployment from the history drawer. A callback prop + // (not `on:restore` forwarding): forwarding a `createEventDispatcher` + // event up through these runes-mode components silently drops it. + onRestore?: (restoredApp: any) => void + // Deploy created the app at a new path; the page navigates to it. Callback + // prop for the same reason as `onRestore` — `on:savedNewAppPath` forwarding + // through these runes-mode components is dropped. + onSavedNewAppPath?: (path: string) => void } let { @@ -184,7 +192,9 @@ onResetToDeployed, loadedFromDraft = false, othersDraftsCount = 0, - onOpenOthersDrafts + onOpenOthersDrafts, + onRestore, + onSavedNewAppPath }: Props = $props() // Set by the on-behalf-of selector when the publisher picks a user other than @@ -352,7 +362,7 @@ path: appPath }) } - dispatch('savedNewAppPath', path) + onSavedNewAppPath?.(path) onDeploy?.({ path }) } catch (e) { sendUserToast(`Error creating app: ${e.body ?? e.message}`, true) @@ -475,7 +485,7 @@ }) } if (appPath !== npath) { - dispatch('savedNewAppPath', npath) + onSavedNewAppPath?.(npath) } onDeploy?.({ path: npath }) } @@ -575,8 +585,6 @@ } ]) - const dispatch = createEventDispatcher() - let customPath = $state(savedApp?.custom_path) let customPathError = $state('') @@ -688,7 +696,7 @@ (historyBrowserDrawerOpen = false)}> - + onRestore?.(e.detail)} {appPath} /> diff --git a/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte index 8c0efa0af4..ea87409171 100644 --- a/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte @@ -388,9 +388,14 @@ let diffDrawer: DiffDrawer | undefined = $state() - function onRestore(ev: any) { + function onRestore(restoredApp: any) { sendUserToast('App restored from previous deployment') - app = ev.detail + // Drop the stale pre-restore autosave. The remounted AppEditor seeds its + // state from `appDraftHandle.draft ?? app`, so without this it keeps showing + // the old draft instead of the restored version. Same reason `reloadDeployed` + // removes the draft before remounting. + UserDraft.remove('app', path) + app = restoredApp // Re-pin the stale-draft fork base to the current head. A restored value // carries the `parent_version` baked in when that older version was deployed, // which would make the deploy guard (`compareVersions`) falsely report "not on @@ -470,7 +475,7 @@ app.path = url } }} - on:restore={onRestore} + {onRestore} summary={app.summary} app={app.value} {deployedBaseline} diff --git a/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte index 94c6d72e71..0cf374f0b1 100644 --- a/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte @@ -449,9 +449,9 @@ let diffDrawer: DiffDrawer | undefined = $state(undefined) - function onRestore(ev: any) { + function onRestore(restoredApp: any) { sendUserToast('App restored from previous deployment') - let prev = ev.detail + let prev = restoredApp extractRawApp(prev) savedApp = { summary: prev.summary, @@ -536,12 +536,12 @@ {#key redraw}
{ + onSavedNewAppPath={(savedPath) => { draftSync.remove() - goto(`/apps_raw/edit/${event.detail}`) - newPath = event.detail + goto(`/apps_raw/edit/${savedPath}`) + newPath = savedPath }} - on:restore={onRestore} + {onRestore} bind:files bind:runnables bind:data From da45e699c8aefeede172c90769ef4f4b182fec0c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 28 Jun 2026 11:43:09 +0200 Subject: [PATCH 118/273] feat(apps): add labels input to app editor deploy drawer (#9828) * feat(apps): add labels input to app editor deploy drawer The labels feature (c4c9ef5fd) wired LabelsInput into the script, flow, schedule, resource and variable editors but left the app editor out: it had no labels state and createApp/updateApp never sent labels, so apps could not be labeled from the UI despite full backend support. Thread the deployed app's labels from the edit page through AppEditor into AppEditorHeader, render LabelsInput in AppEditorHeaderDeploy after the summary field (matching ScriptBuilder/FlowSettings), and include labels in the create/update request bodies, the savedApp snapshot, and the diff/deploy comparison values. The raw-app editor shares the deploy drawer, so it is wired symmetrically (createAppRaw/updateAppRaw + the raw page loader) to avoid leaking a non-functional input there. Fixes WIN-2107 Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(apps): drop redundant labels cast in app edit restore path Co-Authored-By: Claude Opus 4.8 (1M context) * fix(apps): include labels in deploy-drawer Diff current value The Diff button inside the deploy drawer built its current value without labels, so the approval preview could hide label changes that would be deployed. (Identified by cubic.) Co-Authored-By: Claude Opus 4.8 (1M context) * fix(apps): reset raw-app labels on new-draft seed The raw-app edit route keeps labels as route-level state and the ?new_draft=true seed-template branch never cleared it. Since the route is reused across raw-app navigations, opening a labeled raw app then creating a fresh one could remount RawAppEditor with the previous app's labels and deploy them via createAppRaw. Reset labels with the other bleed-prevention resets at the top of the new-draft branch. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../components/apps/editor/AppEditor.svelte | 2 ++ .../apps/editor/AppEditorHeader.svelte | 30 ++++++++++++++----- .../apps/editor/AppEditorHeaderDeploy.svelte | 5 ++++ frontend/src/lib/components/apps/types.ts | 3 ++ .../components/raw_apps/RawAppEditor.svelte | 5 ++++ .../raw_apps/RawAppEditorHeader.svelte | 21 +++++++++---- .../(logged)/apps/edit/[...path]/+page.svelte | 8 +++-- .../apps_raw/edit/[...path]/+page.svelte | 12 +++++++- 8 files changed, 70 insertions(+), 16 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index f6e3245c2c..64cc94f551 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -69,6 +69,7 @@ path, policy, summary, + labels, deployedBaseline = undefined, fromHub = false, diffDrawer = undefined, @@ -886,6 +887,7 @@ void @@ -129,6 +132,7 @@ bottomPanelHidden = false, newApp, newPath = '', + labels: initialLabels = undefined, userDraftPath = '', onSavedNewAppPath, onShowLeftPanel, @@ -261,7 +265,8 @@ policy, deployment_message: deploymentMsg, custom_path: customPath, - preserve_on_behalf_of: preserveOnBehalfOf || undefined + preserve_on_behalf_of: preserveOnBehalfOf || undefined, + labels } }) // New path now exists server-side — drop the autocomplete cache so @@ -272,7 +277,8 @@ value: structuredClone($state.snapshot($app)), path: path, policy: policy, - custom_path: customPath + custom_path: customPath, + labels: $state.snapshot(labels) } closeSaveDrawer() sendUserToast('App deployed successfully') @@ -315,7 +321,8 @@ value: $app, path: newEditedPath || savedApp.path, policy, - custom_path: customPath + custom_path: customPath, + labels }) ) ) { @@ -366,7 +373,8 @@ // it also means that customPath needs to be set to '' instead of undefined to unset it (when admin) custom_path: $userStore?.is_admin || $userStore?.is_super_admin ? (customPath ?? '') : undefined, - preserve_on_behalf_of: preserveOnBehalfOf || undefined + preserve_on_behalf_of: preserveOnBehalfOf || undefined, + labels } }) invalidateWorkspacePaths($workspaceStore!) @@ -375,7 +383,8 @@ value: structuredClone($state.snapshot($app)), path: npath, policy, - custom_path: customPath + custom_path: customPath, + labels: $state.snapshot(labels) } const appHistory = await AppService.getAppHistoryByPath({ workspace: $workspaceStore!, @@ -629,7 +638,8 @@ value: $app, path: newEditedPath || savedApp.path, policy, - custom_path: customPath + custom_path: customPath, + labels } }) }, @@ -728,6 +738,7 @@ }) let customPath = $state(savedApp?.custom_path) + let labels = $state(untrack(() => initialLabels)) $effect(() => { if ($openDebugRun == undefined) { @@ -757,7 +768,8 @@ value: $app, path: newEditedPath || savedApp?.path, policy, - custom_path: customPath + custom_path: customPath, + labels }} /> @@ -801,7 +813,8 @@ value: $app, path: newEditedPath || savedApp.path, policy, - custom_path: customPath + custom_path: customPath, + labels }, button: { text: 'Looks good, deploy', @@ -854,6 +867,7 @@ bind:pathError bind:newEditedPath bind:preserveOnBehalfOf + bind:labels hideSecretUrl={false} /> diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index 0d5fc0c44a..01c0cb3028 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -16,6 +16,7 @@ import { isCloudHosted } from '$lib/cloud' import EEOnly from '$lib/components/EEOnly.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' + import LabelsInput from '$lib/components/LabelsInput.svelte' import OnBehalfOfSelector, { type OnBehalfOfChoice } from '$lib/components/OnBehalfOfSelector.svelte' @@ -38,6 +39,7 @@ newPath, hideSecretUrl = false, preserveOnBehalfOf = $bindable(false), + labels = $bindable(), rawApp = false, newApp = false }: { @@ -55,6 +57,7 @@ newPath: string hideSecretUrl?: boolean preserveOnBehalfOf?: boolean + labels?: string[] | undefined // Raw apps need cross-origin isolation (wm_coep) to be embeddable. Classic // (low-code) apps must NOT get the flag — it would force COEP on the // document and break no-CORP cross-origin subresources (external images, @@ -201,6 +204,8 @@ bind:value={summary} />
+
+
diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index d8ac000ba5..960e273ac3 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -144,6 +144,8 @@ export interface AppEditorProps { path: string policy: Policy summary: string + /** Initial labels for the app, threaded from the loaded app data. */ + labels?: string[] /** Deployed app value the autosave `discardIf` compares against, so an * edit reverting to deployed clears the draft instead of leaving a no-op. * `undefined` for draft-only paths (no deployed baseline). */ @@ -157,6 +159,7 @@ export interface AppEditorProps { summary: string policy: any custom_path?: string + labels?: string[] } | undefined version?: number | undefined diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 18f9c1af0c..d93da9a46c 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -71,6 +71,8 @@ summary?: string path: string newPath?: string | undefined + /** Initial labels for the app, threaded from the loaded app data. */ + labels?: string[] savedApp?: | { value: any @@ -82,6 +84,7 @@ /** No deployed counterpart exists (draft-only); disables Diff. */ no_deployed?: boolean custom_path?: string + labels?: string[] } | undefined diffDrawer?: DiffDrawer | undefined @@ -140,6 +143,7 @@ summary = $bindable(''), path, newPath = undefined, + labels = undefined, savedApp = $bindable(undefined), diffDrawer = undefined, onNavigate, @@ -1612,6 +1616,7 @@ {diffDrawer} {newApp} {newPath} + {labels} appPath={path} {liveEditorDraftStoragePath} {autosaveWorkspace} diff --git a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte index 9853fb69bd..2d7bed8333 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte @@ -102,6 +102,7 @@ summary: string policy: any custom_path?: string + labels?: string[] /** No deployed counterpart exists (draft-only); disables Diff. */ no_deployed?: boolean } @@ -109,6 +110,8 @@ version?: number | undefined newApp: boolean newPath?: string + /** Initial labels for the app, threaded from the loaded app data. */ + labels?: string[] appPath: string runnables: Record files: Record | undefined @@ -169,6 +172,7 @@ version = $bindable(undefined), newApp, newPath = '', + labels: initialLabels = undefined, appPath, runnables, data, @@ -331,7 +335,8 @@ policy, deployment_message: deploymentMsg, custom_path: customPath, - preserve_on_behalf_of: preserveOnBehalfOf || undefined + preserve_on_behalf_of: preserveOnBehalfOf || undefined, + labels }, js, css @@ -345,7 +350,8 @@ value: structuredClone(stateSnapshot(app)), path: path, policy: policy, - custom_path: customPath + custom_path: customPath, + labels: $state.snapshot(labels) } closeSaveDrawer() sendUserToast('App deployed successfully') @@ -454,7 +460,8 @@ // custom_path requires admin so to accept update without it, we need to send as undefined when non-admin (when undefined, it will be ignored) // it also means that customPath needs to be set to '' instead of undefined to unset it (when admin) custom_path: - $userStore?.is_admin || $userStore?.is_super_admin ? (customPath ?? '') : undefined + $userStore?.is_admin || $userStore?.is_super_admin ? (customPath ?? '') : undefined, + labels }, js, css @@ -466,7 +473,8 @@ value: structuredClone(stateSnapshot(app)), path: npath, policy, - custom_path: customPath + custom_path: customPath, + labels: $state.snapshot(labels) } const appHistory = await AppService.getAppHistoryByPath({ workspace: $workspaceStore!, @@ -587,6 +595,7 @@ let customPath = $state(savedApp?.custom_path) let customPathError = $state('') + let labels = $state(untrack(() => initialLabels)) let jobsDrawerOpen = $state(false) @@ -600,7 +609,8 @@ value: app, path: newEditedPath || savedApp?.path, policy, - custom_path: customPath + custom_path: customPath, + labels }) ) @@ -690,6 +700,7 @@ bind:pathError bind:newEditedPath bind:preserveOnBehalfOf + bind:labels /> diff --git a/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte index ea87409171..7eb5d30106 100644 --- a/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/edit/[...path]/+page.svelte @@ -33,6 +33,7 @@ summary: string policy: any custom_path?: string + labels?: string[] } | undefined = $state(undefined) let redraw = $state(0) @@ -297,7 +298,8 @@ value: backendApp_.value as App, path: backendApp_.path, policy: backendApp_.policy, - custom_path: backendApp_.custom_path + custom_path: backendApp_.custom_path, + labels: backendApp_.labels } // "Load another user's draft" handoff: render their value. Overlay mode (we // have our own draft) hard-locks saves until the user confirms overwriting @@ -409,7 +411,8 @@ value: app_.value as App, path: app_.path, policy: app_.policy, - custom_path: app_.custom_path + custom_path: app_.custom_path, + labels: app_.labels } redraw++ } @@ -478,6 +481,7 @@ {onRestore} summary={app.summary} app={app.value} + labels={app.labels} {deployedBaseline} newPath={app.value?.draft_path ?? app.path} path={page.params.path ?? ''} diff --git a/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte index 0cf374f0b1..6227776ec4 100644 --- a/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps_raw/edit/[...path]/+page.svelte @@ -57,6 +57,7 @@ // let lastVersion = 0 let policy: any = $state({}) let summary = $state('') + let labels = $state(undefined) /** User-typed path from `RawAppEditorHeader` when it differs from * `savedApp.path`; mirrored into the draft below as `draft_path` for the * home list's friendly name. */ @@ -72,6 +73,7 @@ summary: string policy: any custom_path?: string + labels?: string[] no_deployed?: boolean } | undefined = $state(undefined) @@ -140,6 +142,7 @@ if (extractedData) data = extractedData files = app.value.files summary = app.summary + labels = app.labels // lastVersion = app.version policy = app.policy // Prefer the saved `draft_path` so the topbar shows the pending name, not @@ -179,6 +182,10 @@ loadedFromDraft = false draftSavedAt = undefined deployedAt = undefined + // `labels` is route-level state; reset it too so a fresh draft doesn't + // inherit (and then deploy) the previously-opened app's labels. The + // import branch re-seeds it via extractRawApp below. + labels = undefined // Brand-new raw app: no deployed baseline, so never discard-on-equal. deployedBaseline = undefined // Suspend autosave across the bootstrap: the seed template and the @@ -355,6 +362,7 @@ path: backendApp_.path, policy: backendApp_.policy, custom_path: backendApp_.custom_path, + labels: backendApp_.labels, no_deployed: backendApp_.no_deployed } // Extract the effective raw app into the editor's local pieces. The bundle @@ -458,7 +466,8 @@ value: structuredClone(stateSnapshot(prev.value)), path: prev.path, policy: structuredClone(stateSnapshot(policy)), - custom_path: prev.custom_path + custom_path: prev.custom_path, + labels: prev.labels } redraw++ } @@ -548,6 +557,7 @@ bind:summary bind:pendingDraftPath {newPath} + {labels} path={page.params.path ?? ''} liveEditorDraftStoragePath={path} {policy} From c0768de0acdf63eaba5fb97d04bfc64f2f03b93d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 28 Jun 2026 11:49:27 +0200 Subject: [PATCH 119/273] fix: close unauthenticated DAP debugger program-mode launch bypass (#9829) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /ws_debug debugger WebSocket gated JWT signature verification on inline `code` being present (`if (code && REQUIRE_SIGNED_REQUESTS)`), so a `program`-mode launch (naming an arbitrary server-side file path that is read and executed) skipped verification entirely — even with REQUIRE_SIGNED_DEBUG_REQUESTS=true. The WS handshake also performed no Origin check, allowing cross-origin (CSWSH) drive-by from a malicious page. - Enforce signing on every launch in both handlers (Python + Bun/TS): reject program-mode outright and require+verify a token for inline code. - Add opt-in DEBUG_ALLOWED_ORIGINS allowlist enforced at the WS handshake. - Default docker-compose REQUIRE_SIGNED_DEBUG_REQUESTS to true. - Update THREAT_MODEL T8/EP15 to reflect the root cause and mitigation. Co-authored-by: Claude Opus 4.8 (1M context) --- backend/THREAT_MODEL.md | 4 +-- debugger/dap_debug_service.ts | 38 +++++++++++++++++++++++++--- debugger/dap_websocket_server_bun.ts | 36 +++++++++++++++++++++++--- docker-compose.yml | 3 ++- 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index dc98a5a625..9501b5bf0d 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -88,7 +88,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | | EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | | EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | -| EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS=false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | +| EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS` now defaults to `true`; can still be overridden to `false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | | EP16 Supply chain | Cached hub scripts, GitHub workflow actions, vendored deps, Docker base image | build/update-time input → host & build integrity | Worker host, build integrity | | EP17 Token lifecycle | Token create/rescope/refresh, script-issued JWTs | scoped caller → broader privilege | Tokens, accounts, isolation | @@ -103,7 +103,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e | | T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b | | T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j | -| T8 | Unauthenticated RCE via the Debugger WebSocket in the default `windmill_extra` configuration | remote_unauth | EP15 | Worker host, all assets | critical | possible | unmitigated | `REQUIRE_SIGNED_DEBUG_REQUESTS` exists but defaults to false | GHSA-725h-99vx-9xr4 | +| T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | | T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | | T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | diff --git a/debugger/dap_debug_service.ts b/debugger/dap_debug_service.ts index 0486814ff3..2b9ce46272 100644 --- a/debugger/dap_debug_service.ts +++ b/debugger/dap_debug_service.ts @@ -149,6 +149,22 @@ const logger = { const WINDMILL_BASE_URL = process.env.WINDMILL_BASE_URL || process.env.BASE_INTERNAL_URL // e.g., http://localhost:8000 const REQUIRE_SIGNED_REQUESTS = process.env.REQUIRE_SIGNED_DEBUG_REQUESTS !== 'false' +// Opt-in cross-origin protection (CSWSH defense-in-depth). When +// DEBUG_ALLOWED_ORIGINS is set (comma-separated list of origins), browser +// requests carrying a non-matching Origin header are rejected at the +// handshake. Non-browser clients send no Origin and are unaffected; code +// execution is independently gated by signed-token verification on launch. +const ALLOWED_ORIGINS = (process.env.DEBUG_ALLOWED_ORIGINS || '') + .split(',') + .map(o => o.trim()) + .filter(Boolean) + +function isOriginRejected(req: Request): boolean { + const origin = req.headers.get('origin') + if (!origin || ALLOWED_ORIGINS.length === 0) return false + return !ALLOWED_ORIGINS.includes(origin) +} + interface JWK { kty: string crv: string @@ -897,9 +913,18 @@ class PythonDebugSession extends BaseDebugSession { this.mainArgs = (args.args as Record) || {} this.envVars = (args.env as Record) || {} - // Verify JWT token if code is provided (signed debug request) - // The token is passed in the launch arguments - if (code && REQUIRE_SIGNED_REQUESTS) { + // Enforce signing on every launch. The token is passed in the launch + // arguments and is verified against the inline `code` (see windmill-api-debug). + if (REQUIRE_SIGNED_REQUESTS) { + // The backend only signs inline `code`; a `program`-mode launch names an + // arbitrary server-side file path that gets read and executed and is never + // signed. Refuse it so it cannot bypass token verification entirely. + if (this.scriptPath) { + logger.error('Rejected program-mode launch: only signed inline code is permitted') + this.sendResponse(request, false, {}, 'program-mode launch is not permitted; submit signed code instead') + return + } + const token = args.token as string | undefined if (!token) { logger.error('No debug token provided but signed requests are required') @@ -907,7 +932,7 @@ class PythonDebugSession extends BaseDebugSession { return } - const verificationError = await verifyDebugToken(token, code) + const verificationError = await verifyDebugToken(token, code ?? '') if (verificationError) { logger.error(`Token verification failed: ${verificationError}`) this.sendResponse(request, false, {}, `Token verification failed: ${verificationError}`) @@ -1067,6 +1092,11 @@ const server = Bun.serve({ const url = new URL(req.url) const path = url.pathname + if (isOriginRejected(req)) { + logger.warn(`Rejected request from disallowed origin: ${req.headers.get('origin')}`) + return new Response('Forbidden origin', { status: 403 }) + } + // Handle WebSocket upgrade with path-based routing if (server.upgrade(req, { data: { path } })) { logger.info(`WS upgrade: ${path}`) diff --git a/debugger/dap_websocket_server_bun.ts b/debugger/dap_websocket_server_bun.ts index 9caa0d4e61..d6ed8967ad 100644 --- a/debugger/dap_websocket_server_bun.ts +++ b/debugger/dap_websocket_server_bun.ts @@ -222,6 +222,21 @@ function generateMainCallArgs(code: string, args: Record): stri const WINDMILL_BASE_URL = process.env.WINDMILL_BASE_URL || process.env.BASE_INTERNAL_URL // e.g., http://localhost:8000 const REQUIRE_SIGNED_REQUESTS = process.env.REQUIRE_SIGNED_DEBUG_REQUESTS !== 'false' +// Opt-in cross-origin protection (CSWSH defense-in-depth); see +// dap_debug_service.ts for the rationale. Only enforced for this file's +// standalone Bun.serve entrypoint (the windmill-extra runtime imports the +// DebugSession class and runs the guarded server in dap_debug_service.ts). +const ALLOWED_ORIGINS = (process.env.DEBUG_ALLOWED_ORIGINS || '') + .split(',') + .map(o => o.trim()) + .filter(Boolean) + +function isOriginRejected(req: Request): boolean { + const origin = req.headers.get('origin') + if (!origin || ALLOWED_ORIGINS.length === 0) return false + return !ALLOWED_ORIGINS.includes(origin) +} + interface JWK { kty: string crv: string @@ -1428,9 +1443,18 @@ export class DebugSession { this.mainArgs = (args.args as Record) || {} this.envVars = (args.env as Record) || {} - // Verify JWT token if code is provided (signed debug request) - // The token is passed in the launch arguments - if (code && REQUIRE_SIGNED_REQUESTS) { + // Enforce signing on every launch. The token is passed in the launch + // arguments and is verified against the inline `code` (see windmill-api-debug). + if (REQUIRE_SIGNED_REQUESTS) { + // The backend only signs inline `code`; a `program`-mode launch names an + // arbitrary server-side file path that gets read and executed and is never + // signed. Refuse it so it cannot bypass token verification entirely. + if (this.scriptPath) { + logger.error('Rejected program-mode launch: only signed inline code is permitted') + this.sendResponse(request, false, {}, 'program-mode launch is not permitted; submit signed code instead') + return + } + const token = args.token as string | undefined if (!token) { logger.error('No debug token provided but signed requests are required') @@ -1438,7 +1462,7 @@ export class DebugSession { return } - const verificationError = await verifyDebugToken(token, code) + const verificationError = await verifyDebugToken(token, code ?? '') if (verificationError) { logger.error(`Token verification failed: ${verificationError}`) this.sendResponse(request, false, {}, `Token verification failed: ${verificationError}`) @@ -2535,6 +2559,10 @@ if (import.meta.main) { hostname: host, port, fetch(req, server) { + if (isOriginRejected(req)) { + logger.warn(`Rejected request from disallowed origin: ${req.headers.get('origin')}`) + return new Response('Forbidden origin', { status: 403 }) + } // Upgrade to WebSocket if (server.upgrade(req)) { return undefined as unknown as Response diff --git a/docker-compose.yml b/docker-compose.yml index 8b30479481..60be2185b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -182,8 +182,9 @@ services: - ENABLE_DEBUGGER=true # Set to true to enable debugger - DEBUGGER_PORT=3003 # Debugger service port - ENABLE_NSJAIL=false # Set to true for nsjail sandboxing (requires privileged: true) - - REQUIRE_SIGNED_DEBUG_REQUESTS=false # Set to true to require JWT tokens for debug sessions + - REQUIRE_SIGNED_DEBUG_REQUESTS=true # Require backend-signed JWT tokens for debug sessions. Do NOT set to false on any internet-reachable deployment: it exposes an unauthenticated code-execution debugger. - WINDMILL_BASE_URL=http://windmill_server:8000 + # - DEBUG_ALLOWED_ORIGINS=https://your-windmill-host # Optional CSWSH hardening: comma-separated allowlist of browser Origins permitted to open debug WebSockets volumes: - lsp_cache:/pyls/.cache logging: *default-logging From 75ba81b2d27fb0722095780312064cb93d20287e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 28 Jun 2026 14:21:38 +0200 Subject: [PATCH 120/273] fix(audit): don't read pg_authid from an elevated context in S3 export migration (#9832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(audit): don't read pg_authid from an elevated context in S3 export migration Migration 20260626132251 aborted instance startup on managed Postgres (e.g. Cloud SQL) with "Modifying pg_authid or pg_auth_members is not allowed in elevated context": the audit S3 export "oldest in-flight xact_start" floor probe calls pg_has_role(...), which reads pg_authid, and managed providers forbid that read from an elevated context. The migration ran the probe inline in its UPDATE, so the whole migration — and the instance boot — failed. Extract the probe into a shared SQL function audit_logs_s3_oldest_inflight_ts() that returns the oldest in-flight xact_start (when cluster-wide stats are visible) or NULL otherwise. The pg_has_role read is wrapped in a plpgsql BEGIN/EXCEPTION subtransaction, so a pg_authid failure returns NULL (callers fall back to a conservative 7-day window / reject) instead of aborting. is_superuser (a GUC, no catalog read) is checked first to short-circuit. The migration's trigger and UPDATE, the OSS backfill try_start, and the EE exporter/startup anchor (companion windmill-ee-private PR) all route through it. Because 20260626132251 already shipped, it is added to the potentially_stale list in windmill-api/src/db.rs: on startup the stale _sqlx_migrations row (checksum mismatch) is deleted and the fixed, idempotent migration re-applies, so already-migrated instances upgrade without a checksum-mismatch boot failure. Fixes WIN-2108 Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 95352c13c4c82247d8cfd80936f9203aeb079802 This commit updates the EE repository reference after PR #635 was merged in windmill-ee-private. Previous ee-repo-ref: 136f49a52af922868acac33abf8198913a9e835c New ee-repo-ref: 95352c13c4c82247d8cfd80936f9203aeb079802 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...100e926bb678171947e71d6d87fdd0c3f9299.json | 15 ------ ...01699bf9968a02675f9f598c97a19c6df089b.json | 20 -------- ...41b344a1710715fb230ac97aca875a9ef38a5.json | 20 ++++++++ ...f172c89da5b216d3530c2743319ab1f4ca853.json | 20 -------- ...fb76daed7d908be80f664a8a30e9896ab1f7e.json | 20 ++++++++ ...95eadc68642592e3c56d4bf0b89b6440869b1.json | 15 ++++++ backend/ee-repo-ref.txt | 2 +- ..._audit_logs_s3_reanchor_on_enable.down.sql | 2 + ...51_audit_logs_s3_reanchor_on_enable.up.sql | 50 ++++++++++++------- .../src/audit_logs_s3_backfill.rs | 23 ++++----- backend/windmill-api/src/db.rs | 6 +++ 11 files changed, 108 insertions(+), 85 deletions(-) delete mode 100644 backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json delete mode 100644 backend/.sqlx/query-5d84f1ede2fbe09923a36d80d7c01699bf9968a02675f9f598c97a19c6df089b.json create mode 100644 backend/.sqlx/query-7b6fa8b999a0160ca349e70e1cd41b344a1710715fb230ac97aca875a9ef38a5.json delete mode 100644 backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json create mode 100644 backend/.sqlx/query-b4c8f7ee9b1d065e1ab34ffe0dcfb76daed7d908be80f664a8a30e9896ab1f7e.json create mode 100644 backend/.sqlx/query-cd7c651f33629af0eb74362525395eadc68642592e3c56d4bf0b89b6440869b1.json diff --git a/backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json b/backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json deleted file mode 100644 index 54db7ff310..0000000000 --- a/backend/.sqlx/query-3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', now(),\n 'last_oldest_inflight_ts', COALESCE(\n CASE WHEN (current_setting('is_superuser') = 'on'\n OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE'))\n AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts)\n THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL)\n ELSE NULL END,\n now() - interval '7 days'))\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3accb7e0eab75fcd34bf5b6d75e100e926bb678171947e71d6d87fdd0c3f9299" -} diff --git a/backend/.sqlx/query-5d84f1ede2fbe09923a36d80d7c01699bf9968a02675f9f598c97a19c6df089b.json b/backend/.sqlx/query-5d84f1ede2fbe09923a36d80d7c01699bf9968a02675f9f598c97a19c6df089b.json deleted file mode 100644 index ae27ba252d..0000000000 --- a/backend/.sqlx/query-5d84f1ede2fbe09923a36d80d7c01699bf9968a02675f9f598c97a19c6df089b.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT CASE WHEN (current_setting('is_superuser') = 'on'\n OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE'))\n AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts)\n THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL)\n ELSE NULL END AS \"x\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "x", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "5d84f1ede2fbe09923a36d80d7c01699bf9968a02675f9f598c97a19c6df089b" -} diff --git a/backend/.sqlx/query-7b6fa8b999a0160ca349e70e1cd41b344a1710715fb230ac97aca875a9ef38a5.json b/backend/.sqlx/query-7b6fa8b999a0160ca349e70e1cd41b344a1710715fb230ac97aca875a9ef38a5.json new file mode 100644 index 0000000000..f909a33b4c --- /dev/null +++ b/backend/.sqlx/query-7b6fa8b999a0160ca349e70e1cd41b344a1710715fb230ac97aca875a9ef38a5.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT audit_logs_s3_oldest_inflight_ts() AS \"x\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "x", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "7b6fa8b999a0160ca349e70e1cd41b344a1710715fb230ac97aca875a9ef38a5" +} diff --git a/backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json b/backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json deleted file mode 100644 index 4c0d0ae14f..0000000000 --- a/backend/.sqlx/query-a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT CASE WHEN (current_setting('is_superuser') = 'on'\n OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE'))\n AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts)\n THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL)\n ELSE NULL END AS \"cutoff?\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "cutoff?", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "a54f686b1bfb16e4e1da2bc143ef172c89da5b216d3530c2743319ab1f4ca853" -} diff --git a/backend/.sqlx/query-b4c8f7ee9b1d065e1ab34ffe0dcfb76daed7d908be80f664a8a30e9896ab1f7e.json b/backend/.sqlx/query-b4c8f7ee9b1d065e1ab34ffe0dcfb76daed7d908be80f664a8a30e9896ab1f7e.json new file mode 100644 index 0000000000..dc82db7591 --- /dev/null +++ b/backend/.sqlx/query-b4c8f7ee9b1d065e1ab34ffe0dcfb76daed7d908be80f664a8a30e9896ab1f7e.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT audit_logs_s3_oldest_inflight_ts() AS \"cutoff?\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cutoff?", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b4c8f7ee9b1d065e1ab34ffe0dcfb76daed7d908be80f664a8a30e9896ab1f7e" +} diff --git a/backend/.sqlx/query-cd7c651f33629af0eb74362525395eadc68642592e3c56d4bf0b89b6440869b1.json b/backend/.sqlx/query-cd7c651f33629af0eb74362525395eadc68642592e3c56d4bf0b89b6440869b1.json new file mode 100644 index 0000000000..2c775af0b1 --- /dev/null +++ b/backend/.sqlx/query-cd7c651f33629af0eb74362525395eadc68642592e3c56d4bf0b89b6440869b1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', now(),\n 'last_oldest_inflight_ts',\n COALESCE(audit_logs_s3_oldest_inflight_ts(), now() - interval '7 days'))\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cd7c651f33629af0eb74362525395eadc68642592e3c56d4bf0b89b6440869b1" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 294707d466..ce3db8ebcb 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -b821fecccbcba2efed544890576bf2b84321d70d +95352c13c4c82247d8cfd80936f9203aeb079802 diff --git a/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql index 2c9e78659e..1ce2c852b7 100644 --- a/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql +++ b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.down.sql @@ -17,3 +17,5 @@ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql; + +DROP FUNCTION IF EXISTS audit_logs_s3_oldest_inflight_ts(); diff --git a/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql index 3fd759da9e..ff2ed110c6 100644 --- a/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql +++ b/backend/migrations/20260626132251_audit_logs_s3_reanchor_on_enable.up.sql @@ -17,27 +17,48 @@ -- The task name literal must match -- `windmill_common::global_settings::AUDIT_LOGS_S3_EXPORT_TASK`. +-- Oldest in-flight `xact_start` when this role can observe *all* sessions (so the +-- min is a true cluster-wide bound), else NULL — callers substitute a conservative +-- window. The stats-visibility check goes through `is_superuser` (a preset GUC, no +-- catalog read) first, then a best-effort `pg_has_role` probe guarded by EXCEPTION: +-- `pg_has_role` reads `pg_authid`, which some managed providers (e.g. Cloud SQL) +-- forbid even to read from an elevated context, raising "Modifying pg_authid or +-- pg_auth_members is not allowed in elevated context". The EXCEPTION block runs in +-- its own subtransaction, so a failure there returns NULL (→ conservative fallback) +-- without aborting the caller — the migration-time UPDATE below, the trigger, or the +-- export task, none of which must fail just because the optimization is unavailable. +CREATE OR REPLACE FUNCTION audit_logs_s3_oldest_inflight_ts() +RETURNS timestamptz AS $$ +DECLARE + v_can_read_all_stats boolean := current_setting('is_superuser') = 'on'; +BEGIN + IF NOT v_can_read_all_stats THEN + BEGIN + v_can_read_all_stats := pg_has_role(current_user, 'pg_read_all_stats', 'USAGE'); + EXCEPTION WHEN OTHERS THEN + v_can_read_all_stats := false; + END; + END IF; + IF v_can_read_all_stats AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) THEN + RETURN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL); + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION audit_logs_s3_anchor_on_enable() RETURNS TRIGGER AS $$ -DECLARE - v_floor timestamptz; BEGIN IF NEW.value = to_jsonb(true) AND (TG_OP = 'INSERT' OR OLD.value IS DISTINCT FROM NEW.value) THEN - v_floor := COALESCE( - CASE WHEN (current_setting('is_superuser') = 'on' - OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')) - AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) - THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL) - ELSE NULL END, - now() - interval '7 days'); INSERT INTO background_task_state (name, value) VALUES ( 'audit_logs_s3_export', jsonb_build_object( 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint, 'last_ts', now(), - 'last_oldest_inflight_ts', v_floor + 'last_oldest_inflight_ts', + COALESCE(audit_logs_s3_oldest_inflight_ts(), now() - interval '7 days') ) ) ON CONFLICT (name) DO UPDATE @@ -59,12 +80,7 @@ UPDATE background_task_state SET value = jsonb_build_object( 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint, 'last_ts', to_jsonb(now()), - 'last_oldest_inflight_ts', to_jsonb(COALESCE( - CASE WHEN (current_setting('is_superuser') = 'on' - OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')) - AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) - THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL) - ELSE NULL END, - now() - interval '7 days'))) + 'last_oldest_inflight_ts', + to_jsonb(COALESCE(audit_logs_s3_oldest_inflight_ts(), now() - interval '7 days'))) WHERE name = 'audit_logs_s3_export' AND (value->>'last_ts')::timestamptz <= 'epoch'::timestamptz; diff --git a/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs b/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs index e86ceb6448..0c5c55f7ed 100644 --- a/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs +++ b/backend/windmill-api-settings/src/audit_logs_s3_backfill.rs @@ -186,18 +186,17 @@ pub async fn try_start(db: &DB, from: DateTime, to: DateTime) -> error // without pg_read_all_stats/superuser sees only its own sessions, and a prepared // (2PC) transaction is invisible to pg_stat_activity — in either case an old // transaction could still commit rows inside an accepted window after our scan ends. - // Since a backfill asserts completeness, we REJECT in those cases rather than fall - // back to a best-effort margin (NULL below). (The continuous exporter, which only - // claims bounded lag, keeps the 7-day fallback instead.) - let settled_cutoff: Option> = sqlx::query_scalar!( - r#"SELECT CASE WHEN (current_setting('is_superuser') = 'on' - OR pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')) - AND NOT EXISTS (SELECT 1 FROM pg_prepared_xacts) - THEN (SELECT min(xact_start) FROM pg_stat_activity WHERE xact_start IS NOT NULL) - ELSE NULL END AS "cutoff?""# - ) - .fetch_one(db) - .await?; + // Since a backfill asserts completeness, we REJECT in those cases (the function + // returns NULL). (The continuous exporter, which only claims bounded lag, keeps the + // 7-day fallback instead.) The probe lives in the `audit_logs_s3_oldest_inflight_ts()` + // SQL function (migration 20260626132251) so its `pg_has_role`/`pg_authid` read is + // wrapped in a subtransaction EXCEPTION: managed providers (e.g. Cloud SQL) forbid + // reading pg_authid from an elevated context, which would otherwise surface here as + // an opaque error instead of NULL → the actionable rejection below. + let settled_cutoff: Option> = + sqlx::query_scalar!(r#"SELECT audit_logs_s3_oldest_inflight_ts() AS "cutoff?""#) + .fetch_one(db) + .await?; let Some(settled_cutoff) = settled_cutoff else { return Err(error::Error::BadRequest( "audit backfill: cannot determine a trustworthy settled-time boundary, so completeness \ diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index df6c8cc6b5..b81a9dbb84 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -294,6 +294,12 @@ pub async fn migrate( // idempotent, so re-applying on an already-migrated DB is a no-op. 20260423050000, 20260523055641, + // Reworked to stop reading pg_authid (via pg_has_role) from an elevated + // context, which managed providers (e.g. Cloud SQL) forbid — the original + // aborted startup. The new file is idempotent (CREATE OR REPLACE + an + // epoch-guarded UPDATE that no-ops once anchored), so re-applying on an + // already-migrated DB is safe. + 20260626132251, ]; for m in migrator.migrations.iter() { if m.migration_type.is_down_migration() { From 96c0ff65bddd90c3f306b4803a4322d2e1e064fc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 28 Jun 2026 14:33:27 +0200 Subject: [PATCH 121/273] chore(main): release 1.742.0 (#9830) * chore(main): release 1.742.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 15 ++ backend/Cargo.lock | 176 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 54 +++++- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 191 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f07f85012..361b6fbd91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28) + + +### Features + +* **apps:** add labels input to app editor deploy drawer ([#9828](https://github.com/windmill-labs/windmill/issues/9828)) ([da45e69](https://github.com/windmill-labs/windmill/commit/da45e699c8aefeede172c90769ef4f4b182fec0c)) +* column-level lineage for DuckLake pipelines (SQL-AST inferred + traceable) ([#9814](https://github.com/windmill-labs/windmill/issues/9814)) ([003a262](https://github.com/windmill-labs/windmill/commit/003a262a4e9d6c2a63ada01aa8429aea1fbb6031)) + + +### Bug Fixes + +* **audit:** don't read pg_authid from an elevated context in S3 export migration ([#9832](https://github.com/windmill-labs/windmill/issues/9832)) ([75ba81b](https://github.com/windmill-labs/windmill/commit/75ba81b2d27fb0722095780312064cb93d20287e)) +* close unauthenticated DAP debugger program-mode launch bypass ([#9829](https://github.com/windmill-labs/windmill/issues/9829)) ([c0768de](https://github.com/windmill-labs/windmill/commit/c0768de0acdf63eaba5fb97d04bfc64f2f03b93d)) +* redeploy older app version from deployment history ([#9826](https://github.com/windmill-labs/windmill/issues/9826)) ([c479afa](https://github.com/windmill-labs/windmill/commit/c479afab8ebceccbee050e923dc5c27a6712ea62)) + ## [1.741.0](https://github.com/windmill-labs/windmill/compare/v1.740.0...v1.741.0) (2026-06-26) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c970648000..7072389c21 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1855,9 +1855,9 @@ dependencies = [ [[package]] name = "byte-unit" -version = "5.2.3" +version = "5.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37bcaa4a0975bed4a760af3efe4368825098ce5f9d37a30c5a021d635dc63d8f" +checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" dependencies = [ "rust_decimal", "schemars 1.2.1", @@ -4371,18 +4371,18 @@ dependencies = [ [[package]] name = "enum-ordinalize" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", @@ -8962,13 +8962,13 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.24" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" dependencies = [ - "ahash 0.8.12", "equivalent", - "hashbrown 0.16.1", + "foldhash 0.2.0", + "hashbrown 0.17.1", "parking_lot", ] @@ -13734,7 +13734,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-nats", @@ -13816,7 +13816,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.741.0" +version = "1.742.0" dependencies = [ "async-stream", "async-trait", @@ -13849,7 +13849,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13862,7 +13862,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "argon2", @@ -14000,7 +14000,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14023,7 +14023,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14038,7 +14038,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14064,7 +14064,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.741.0" +version = "1.742.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14074,7 +14074,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14091,7 +14091,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14113,7 +14113,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14136,7 +14136,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14152,7 +14152,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14173,7 +14173,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14194,7 +14194,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14208,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-nats", @@ -14243,7 +14243,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14268,7 +14268,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14286,7 +14286,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14308,7 +14308,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14328,7 +14328,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14365,7 +14365,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14393,7 +14393,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.741.0" +version = "1.742.0" dependencies = [ "lazy_static", "serde", @@ -14405,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.741.0" +version = "1.742.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14430,7 +14430,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14444,7 +14444,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.741.0" +version = "1.742.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14477,7 +14477,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.741.0" +version = "1.742.0" dependencies = [ "chrono", "lazy_static", @@ -14491,7 +14491,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14510,7 +14510,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.741.0" +version = "1.742.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14612,7 +14612,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.741.0" +version = "1.742.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14631,7 +14631,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.741.0" +version = "1.742.0" dependencies = [ "regex", "serde", @@ -14646,7 +14646,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14670,7 +14670,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "futures", @@ -14687,7 +14687,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.741.0" +version = "1.742.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14703,7 +14703,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -14724,7 +14724,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -14755,7 +14755,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "arc-swap", @@ -14780,7 +14780,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-stream", @@ -14814,7 +14814,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "futures", @@ -14832,7 +14832,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.741.0" +version = "1.742.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14841,7 +14841,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -14853,7 +14853,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde_json", @@ -14865,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "gosyn", @@ -14877,7 +14877,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -14889,7 +14889,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde_json", @@ -14901,7 +14901,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "nu-parser", @@ -14912,7 +14912,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14923,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14935,7 +14935,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14946,7 +14946,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-recursion", @@ -14968,7 +14968,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde_json", @@ -14980,7 +14980,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -14994,7 +14994,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15011,7 +15011,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -15024,7 +15024,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde", @@ -15036,7 +15036,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -15054,7 +15054,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15070,7 +15070,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15086,7 +15086,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-recursion", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "const_format", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.741.0" +version = "1.742.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15186,7 +15186,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-recursion", @@ -15220,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15244,7 +15244,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15277,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15310,7 +15310,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15330,7 +15330,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15364,7 +15364,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15423,7 +15423,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15447,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-nats", @@ -15471,7 +15471,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15506,7 +15506,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15534,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-trait", @@ -15559,7 +15559,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15578,7 +15578,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-once-cell", @@ -15688,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.741.0" +version = "1.742.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d721dcc272..45682215cf 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.741.0" +version = "1.742.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.741.0" +version = "1.742.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 78db9dd2b0..096d93163b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.741.0" +version = "1.742.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.741.0" +version = "1.742.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.741.0" +version = "1.742.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.741.0" +version = "1.742.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 6faadc1cec..e2afd2483b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.741.0" +version = "1.742.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e9c57c3fb0..f9a15ef72d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.741.0 + version: 1.742.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2b3b23746a..f8cc748bdb 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.741.0"; +export const VERSION = "v1.742.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 1e40e04ac7..42d8109c49 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.741.0"; +export const VERSION = "1.742.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bd2619a54e..4a3fcf0696 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.741.0", + "version": "1.742.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.741.0", + "version": "1.742.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -878,6 +878,7 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -889,6 +890,7 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -899,6 +901,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1414,6 +1417,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1562,6 +1566,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1578,6 +1583,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1594,6 +1600,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1610,6 +1617,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1626,6 +1634,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1642,6 +1651,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1658,6 +1668,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1674,6 +1685,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1690,6 +1702,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1706,6 +1719,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1722,6 +1736,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1738,6 +1753,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1754,6 +1770,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1772,6 +1789,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1788,6 +1806,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2093,6 +2112,7 @@ "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7328,7 +7348,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "devOptional": true, + "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7863,6 +7883,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7883,6 +7904,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7903,6 +7925,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7923,6 +7946,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7943,6 +7967,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7963,6 +7988,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7983,6 +8009,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8003,6 +8030,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8023,6 +8051,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8043,6 +8072,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8063,6 +8093,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12750,6 +12781,21 @@ } } }, + "node_modules/svelte-check/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13489,7 +13535,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/frontend/package.json b/frontend/package.json index 8bdf440a00..e9ec640b02 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.741.0", + "version": "1.742.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index cda87ce8d7..7f9d00903a 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.741.0" +wmill = ">=1.742.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 9ccfa3ceab..f4342930e9 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.741.0 + version: 1.742.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index ebdfaa8215..e5b553e6f3 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.741.0' + ModuleVersion = '1.742.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index d0942b025c..5310b72896 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.741.0" +version = "1.742.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 13d6388e30..7ae2023a7c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.741.0", + "version": "1.742.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 42ebc2096d..8edc248c45 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.741.0", + "version": "1.742.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index b27f19abcc..bee795f5f0 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.741.0 +1.742.0 From 2493eaf031f30072637a297398674e761f039005 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 29 Jun 2026 18:19:31 +0200 Subject: [PATCH 122/273] feat(home): redesign create-new popover and home header (#9827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(home): redesign create-new popover and home header Replace the home page "Home" title with a hover-driven "New" popover (CreateActionsMenu) listing Script / Flow / Workflow-as-Code / Apps with a description pane. Workflow-as-Code offers a Python / TypeScript choice; other entries are created by clicking the list row. Move CLI/MCP to the header far right, add a Hub link button, and drop the Workspace/Hub tab switcher so the home page shows only the workspace list. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): keep Home title, move New to the right, swap popover panes Reintroduce the "Home" header title on the left and place the New popover on the right alongside the Hub and CLI/MCP buttons (top-aligned, with extra gap before New). Swap the popover panes so the description is on the left and the option list on the right; the menu opens leftward again. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): make Hub, CLI/MCP and New header buttons uniform md size Set all three header buttons to unifiedSize="md" (New keeps the accent variant to stand out) and re-center the right group now that heights match. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): remove divider between popover panes Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): badge Workflow-as-Code as Advanced and low-code App as Legacy Add inline pills (Advanced / Legacy) next to the option label and in the description header, and widen the option list so the longest label plus badge fits without truncating. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): reorder create options and share App icon Order is now Script, Flow, App (full-code), Workflow-as-Code, App (low-code). Full-code App reuses the low-code App dashboard icon (distinguished by accent). Broaden Option.icon to also accept the BarsStaggered (Flow) component. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): add import / pipeline secondary actions to popover detail panels Surface the previous create-menu extras in the matching detail panel: Flow → Import flow + Pipeline (alpha); Workflow-as-Code → Import Workflow-as-Code; App (full/low-code) → Import full/low-code app. A shared YAML/JSON import drawer parses the pasted source into the relevant store (or sessionStorage for the full-reload apps_raw route) and navigates. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): add Data pipelines editor option (alpha) to create popover Add a first-class "Data pipelines editor" entry right after Workflow-as-Code (indigo accent, Workflow icon, emerald Alpha badge) routing to /pipeline, and drop the now-redundant "Pipeline (alpha)" secondary action from the Flow panel. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): emphasize import buttons and group the badged options Render the detail-panel import actions as default (bordered) buttons with an import icon instead of subtle text, and add a separator in the option list between the three plain options and the three badged (Advanced/Alpha/Legacy) ones. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): increase the y gap around the option-group separator Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): rename Data pipelines editor option to Data pipelines Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): click-open create popover with import submenu and toggleable docs Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): keyboard-navigable create popover via melt dropdown with looping Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Guilhem Lemouel --- .../components/home/CreateActionsMenu.svelte | 547 ++++++++++++++++++ .../src/lib/components/home/ItemsList.svelte | 3 +- .../src/routes/(root)/(logged)/+page.svelte | 77 +-- 3 files changed, 579 insertions(+), 48 deletions(-) create mode 100644 frontend/src/lib/components/home/CreateActionsMenu.svelte diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte new file mode 100644 index 0000000000..146e390419 --- /dev/null +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -0,0 +1,547 @@ + + +
+ + + {#if $open && active} +
+ {#if showDoc} + +
+
+
+ +
+
+
+

{active.label}

+ {#if active.badge} + + {active.badge.label} + + {/if} +
+

{active.tagline}

+
+
+ +

{active.description}

+ +
    + {#each active.bullets as bullet (bullet)} +
  • + + {bullet} +
    • + {/each} +
+ + +
+ {/if} + + +
+ {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} +
+ +
+ + {option.label} + + {#if option.badge} + + {option.badge.label} + + {/if} + {/snippet} + {#each allOptions as option (option.key)} + {@const ac = accentClasses[option.accent]} + {@const rowClass = + 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} + {#if option.variants} + + {#if $wacSubOpen} +
+ {#each option.variants ?? [] as variant (variant.label)} + {@const VariantIcon = variant.icon} + + {/each} +
+ {/if} + {:else} + + {/if} + {/each} + + +
+ + {#if $importSubOpen} +
+ {#each importActions as action (action.label)} + + {/each} +
+ {/if} + + {#if !showDoc} + + {/if} +
+
+ {/if} +
+ + + + importDrawer?.closeDrawer?.()}> + + + + {#snippet content()} +
+ {#key importType} + {#await import('$lib/components/SimpleEditor.svelte')} + + {:then Module} + + {/await} + {/key} +
+ {/snippet} + + {#snippet actions()} + + {/snippet} + + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 256b2277f5..b6ae577d4b 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -506,7 +506,8 @@ if (menuItem) { if (e.key === 'ArrowUp' || e.key === 'ArrowDown') { const menu = menuItem.closest('[role="menu"]') - if (menu) { + // menus marked data-arrow-loop keep melt's cyclic wrap instead of exiting + if (menu && !menu.hasAttribute('data-arrow-loop')) { const items = Array.from(menu.querySelectorAll('[role="menuitem"]')) const idx = items.indexOf(menuItem) const isFirst = idx === 0 diff --git a/frontend/src/routes/(root)/(logged)/+page.svelte b/frontend/src/routes/(root)/(logged)/+page.svelte index b070bd5fef..e326bc87f7 100644 --- a/frontend/src/routes/(root)/(logged)/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/+page.svelte @@ -1,13 +1,11 @@ + + + +
+ + {#if boundBar}{@render boundBar()}{/if} + {#if mode === 'edit'} + + {/if} + {#if prefetchingAssets} +
+ + Parsing assets… +
+ {/if} + {#if onTogglePanelHidden && (mode !== 'edit' || editor.selection != undefined || editor.activeDraftPath != undefined)} +
+
+ {/if} +
+ {#if detailsPaneOpen && workspace} + + {#if idleView && idlePane} + {@render idlePane()} + {:else} + onStartBoundedRunForOpen?.(editor.openScriptPath!) + : undefined} + {onRunCompleted} + {onTestStateChange} + {requestRemoveSignal} + {requestRunSignal} + {requestRunCascadeSignal} + {focusUploadSignal} + draftScript={activeDraft?.script} + {pathPrefix} + {onDraftPathChange} + {workspace} + onAnnotationsChange={editor.handleAnnotationsChange} + onAssetsChange={editor.handleAssetsChange} + onContentChange={editor.handleContentChange} + onDraftPersist={editor.handleDraftPersist} + onclose={onClose} + onHide={onTogglePanelHidden} + {onDiscard} + {onDraftSaved} + {onPersistedSaved} + {onScriptRenamed} + {onScriptRemoved} + /> + {/if} + + {/if} + diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts new file mode 100644 index 0000000000..a1a82870c8 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts @@ -0,0 +1,143 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { JobService, ScriptService } from '$lib/gen' +import { createPipelineAiHelpers, type PipelineDraft } from './pipelineAiHelpers' +import type { AssetGraphResponse } from './types' + +// Build a helper handle over an in-memory drafts Map, mirroring how the editor +// wires it. `getFolder` returns the bare folder name (as the route/session do). +function makeHandle( + initial: Array<[string, PipelineDraft]> = [], + runnables: Array<{ path: string }> = [] +) { + let drafts = new Map(initial) + let forgotten: string[] = [] + const handle = createPipelineAiHelpers({ + getFolder: () => 'x', + getWorkspace: () => 'w', + getResolvedGraph: () => + ({ assets: [], runnables, edges: [], triggers: [] }) as unknown as AssetGraphResponse, + getDrafts: () => drafts, + setDrafts: (next) => (drafts = next), + newDraftLocalId: () => 'id', + onForgetPath: (p) => forgotten.push(p) + }) + return { handle, drafts: () => drafts, forgotten: () => forgotten } +} + +afterEach(() => vi.restoreAllMocks()) + +const draft = (over: Partial = {}): PipelineDraft => + ({ localId: 'l', script: { content: '' } as any, ...over }) as PipelineDraft + +describe('pipeline AI direct-draft helpers', () => { + it('removeProposedNode discards the unsaved draft at a path', async () => { + const { handle, drafts, forgotten } = makeHandle([ + ['f/x/a', draft()], + ['f/x/b', draft()] + ]) + await handle.removeProposedNode('f/x/a') + expect(drafts().has('f/x/a')).toBe(false) + expect(drafts().has('f/x/b')).toBe(true) + expect(forgotten()).toContain('f/x/a') + }) + + it('removeProposedNode throws when there is no draft to discard', async () => { + const { handle } = makeHandle() + await expect(handle.removeProposedNode('f/x/missing')).rejects.toThrow() + }) + + it('getPipelineContext does not expose any pending/approval state', () => { + const { handle } = makeHandle([['f/x/a', draft()]]) + const ctx = handle.getPipelineContext() + expect(ctx).not.toHaveProperty('pendingProposals') + expect(handle).not.toHaveProperty('acceptAll') + expect(handle).not.toHaveProperty('rejectAll') + }) + + it('testNode on a deployed node never dispatches downstream subscribers', async () => { + // No draft at the path → runs the deployed version, which must carry + // `_wmill_skip_asset_dispatch` so a single-node test can't fire downstream. + const spy = vi.spyOn(JobService, 'runScriptByPath').mockResolvedValue('job-1' as any) + const { handle } = makeHandle() + await handle.testNode('f/x/deployed', { foo: 1 }) + expect(spy).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ _wmill_skip_asset_dispatch: true, foo: 1 }) + }) + ) + }) + + it('proposeNode rejects a path outside the open folder', async () => { + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ path: 'f/other/n', language: 'duckdb' as any, content: '' }) + ).rejects.toThrow(/open folder/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects content missing the pipeline annotation', async () => { + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ path: 'f/x/new', language: 'duckdb' as any, content: 'SELECT 1' }) + ).rejects.toThrow(/pipeline annotation/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects a path colliding with an existing draft', async () => { + const { handle } = makeHandle([['f/x/a', draft()]]) + await expect( + handle.proposeNode({ path: 'f/x/a', language: 'duckdb' as any, content: '-- pipeline' }) + ).rejects.toThrow(/already exists/) + }) + + it('proposeNode rejects a path colliding with an existing deployed node', async () => { + const { handle, drafts } = makeHandle([], [{ path: 'f/x/dep' }]) + await expect( + handle.proposeNode({ path: 'f/x/dep', language: 'duckdb' as any, content: '-- pipeline' }) + ).rejects.toThrow(/already exists/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects a path that is an already-deployed script when the graph has not hydrated', async () => { + // Empty graph (session preview can race open_preview), but a deployed script + // exists at the path — the backend probe must still catch it. + const spy = vi.spyOn(ScriptService, 'getScriptByPath').mockResolvedValue({} as any) + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ + path: 'f/x/deployed', + language: 'duckdb' as any, + content: '-- pipeline' + }) + ).rejects.toThrow(/already exists/) + expect(spy).toHaveBeenCalled() + expect(drafts().size).toBe(0) + }) + + it('editNode rejects a path outside the open folder', async () => { + const { handle, drafts } = makeHandle() + await expect(handle.editNode('f/other/foo', '-- pipeline')).rejects.toThrow(/open folder/) + expect(drafts().size).toBe(0) + }) + + it('editNode preserves the deployed script hash/metadata and replaces only content', async () => { + const deployed = { + hash: 'abc123', + path: 'f/x/node', + summary: 'My node', + description: 'desc', + tag: 'custom', + language: 'duckdb', + content: '-- pipeline\nSELECT 1' + } + vi.spyOn(ScriptService, 'getScriptByPath').mockResolvedValue(deployed as any) + const { handle, drafts } = makeHandle() + await handle.editNode('f/x/node', '-- pipeline\nSELECT 2') + const d = drafts().get('f/x/node') + expect(d?.script.hash).toBe('abc123') + expect(d?.script.summary).toBe('My node') + expect(d?.script.description).toBe('desc') + expect(d?.script.tag).toBe('custom') + expect(d?.script.content).toBe('-- pipeline\nSELECT 2') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts new file mode 100644 index 0000000000..959992543f --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts @@ -0,0 +1,319 @@ +import { JobService, ScriptService, type AssetKind, type Script, type ScriptLang } from '$lib/gen' +import { emptySchema, sendUserToast } from '$lib/utils' +import { inferAssets } from '$lib/infer' +import { extractWrites, type AssetWithAltAccessType } from '$lib/components/assets/lib' +import { assetUri, autoOutputAsset, type PipelineOutputKind } from './pipelineTemplates' +import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import type { AssetGraphResponse } from './types' +import type { + PipelineAIChatHelpers, + PipelineContext, + PipelineNodeSummary +} from '$lib/components/copilot/chat/pipeline/core' + +// ============================================================================ +// Shared data-pipeline AI helper layer. +// +// Both the full-page editor (/pipeline/[folder]) and the in-session preview +// (PipelineEditorView) drive the AI chat's pipeline tools through this factory, +// so the build/edit logic lives in exactly one place. Each caller injects +// accessors for its own draft Map and graph; this module owns the AI behaviour +// (build/edit/discard/test). AI edits apply directly as unsaved drafts — there +// is no separate approve/reject step. +// ============================================================================ + +/** + * An unsaved pipeline node draft. `localId` is a stable per-draft id preserved + * across renames (the page uses it to dedupe concurrent deploys). AI-built nodes + * and manually-created drafts are the same thing — an unsaved node on the canvas. + */ +export type PipelineDraft = { + localId: string + script: Script + outputAssets?: Array<{ kind: AssetKind; path: string }> +} + +export type PipelineAiHelperDeps = { + getFolder: () => string + getWorkspace: () => string | undefined + /** The draft-overlaid graph (resolveGraph output) the context summary reads. */ + getResolvedGraph: () => AssetGraphResponse + getDrafts: () => Map + setDrafts: (next: Map) => void + /** Stable id for a freshly-created draft (route page tracks deploys by it). */ + newDraftLocalId: () => string + /** Focus/select the node after it is staged (pan + open in the pane). */ + onProposeNode?: (path: string) => void + /** Throw (or switch to edit mode) when the surface can't accept AI edits. */ + ensureEditable?: () => void + /** Surface the draft overlay if it is hidden (the page's "show drafts" view). */ + onShowDrafts?: () => void + /** Forget per-path state when a draft is discarded. */ + onForgetPath?: (path: string) => void + /** Notify the caller a test run started so it can light up its run UI. */ + onRunStarted?: (jobId: string, path: string) => void +} + +export function makePipelineScript( + language: ScriptLang, + scriptPath: string, + content: string, + createdAt: string +): Script { + // Cast through unknown: a local draft only needs path/language/content/schema; + // the many readonly deployment fields on Script don't matter until createScript. + return { + hash: '', + path: scriptPath, + summary: '', + description: '', + content, + schema: emptySchema(), + is_template: false, + extra_perms: {}, + language, + kind: 'script', + created_by: '', + created_at: createdAt, + archived: false, + deleted: false, + starred: false + } as unknown as Script +} + +async function inferOutputAssets( + language: ScriptLang, + content: string +): Promise> { + try { + const inferred = await inferAssets(language, content) + if (inferred?.status === 'error') return [] + return extractWrites((inferred?.assets ?? []) as AssetWithAltAccessType[]) + } catch { + return [] + } +} + +export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIChatHelpers { + // A staged draft is always persisted into the OPEN folder's data_pipeline + // bundle, so a path outside the folder would silently land an unrelated script + // there. Both build and edit must stay scoped to the folder. + function assertInFolder(path: string) { + const folder = deps.getFolder() + if (folder && !path.startsWith(`f/${folder}/`)) { + throw new Error( + `Pipeline nodes must be in the open folder — use a path under 'f/${folder}/' (got '${path}').` + ) + } + } + + // A pipeline node IS its `// pipeline` annotation (it's what makes the deployed + // script a pipeline member). Reject content that lacks it so a staged draft + // isn't a non-member script the model can't see is broken until deploy. + function assertPipelineAnnotation(content: string) { + if (!parsePipelineAnnotations(content).inPipeline) { + throw new Error( + `Pipeline node content must declare the pipeline annotation on its own comment line ` + + `(\`// pipeline\`, or \`-- pipeline\` for SQL / \`# pipeline\` for Python).` + ) + } + } + + function buildContext(): PipelineContext { + const graph = deps.getResolvedGraph() + const drafts = deps.getDrafts() + const nodes: PipelineNodeSummary[] = graph.runnables + .filter((r) => r.usage_kind === 'script') + .map((r) => { + const draft = drafts.get(r.path) + const writes = graph.edges + .filter( + (e) => + e.runnable_kind === 'script' && + e.runnable_path === r.path && + (e.access_type === 'w' || e.access_type === 'rw') + ) + .map((e) => assetUri({ kind: e.asset_kind, path: e.asset_path })) + const reads = graph.edges + .filter( + (e) => + e.runnable_kind === 'script' && + e.runnable_path === r.path && + (e.access_type === 'r' || e.access_type === 'rw') + ) + .map((e) => assetUri({ kind: e.asset_kind, path: e.asset_path })) + const triggers = graph.triggers + .filter((t) => t.runnable_kind === 'script' && t.runnable_path === r.path) + .map((t) => + t.trigger_kind === 'asset' + ? assetUri({ kind: t.asset_kind, path: t.asset_path }) + : t.trigger_kind + ) + return { + path: r.path, + language: draft?.script.language, + unsaved: r.unsaved ?? false, + summary: draft?.script.summary || undefined, + writes: [...new Set(writes)], + reads: [...new Set(reads)], + triggers: [...new Set(triggers)] + } + }) + return { + folder: deps.getFolder(), + mode: 'edit', + nodes, + assets: graph.assets.map((a) => assetUri({ kind: a.kind, path: a.path })) + } + } + + const helpers: PipelineAIChatHelpers = { + getPipelineContext: buildContext, + getNodeBody: async (path) => { + const draft = deps.getDrafts().get(path) + if (draft) return { language: draft.script.language, content: draft.script.content } + const workspace = deps.getWorkspace() + if (!workspace) return undefined + try { + const deployed = await ScriptService.getScriptByPath({ workspace, path }) + return { language: deployed.language, content: deployed.content } + } catch { + return undefined + } + }, + proposeNode: async ({ path, language, content, outputKind }) => { + deps.ensureEditable?.() + // build_pipeline_node creates a NEW node in the OPEN folder. Reject a path + // outside the folder (it would silently stage into this folder's bundle) + // and a path that collides with an existing node (the model should use + // edit_pipeline_node instead of shadowing a deployed node as a draft). + assertInFolder(path) + assertPipelineAnnotation(content) + const drafts = deps.getDrafts() + if (drafts.has(path)) { + throw new Error( + `A draft already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + if (deps.getResolvedGraph().runnables.some((r) => r.path === path)) { + throw new Error( + `A pipeline node already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + // Authoritative new-node check: the resolved graph may not have hydrated yet + // (the session preview can race open_preview), and it only lists pipeline + // runnables — so probe the backend. ANY deployed script at this path means + // "build new" would shadow it on deploy; the model should edit instead. + const workspace = deps.getWorkspace() + if (workspace) { + let deployedExists = false + try { + await ScriptService.getScriptByPath({ workspace, path }) + deployedExists = true + } catch { + // 404 → no deployed script at this path, safe to create a new node. + } + if (deployedExists) { + throw new Error( + `A script already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + } + const inferred = await inferOutputAssets(language, content) + // Fall back to a seeded output (from the declared output_kind) when the + // body doesn't yet write anything inferable. + const seeded = + inferred[0] ?? + (outputKind + ? autoOutputAsset(outputKind as PipelineOutputKind, deps.getFolder(), language) + : undefined) + const next = new Map(drafts) + next.set(path, { + localId: deps.newDraftLocalId(), + script: makePipelineScript(language, path, content, new Date().toISOString()), + outputAssets: inferred.length > 0 ? inferred : seeded ? [seeded] : undefined + }) + deps.setDrafts(next) + deps.onShowDrafts?.() + deps.onProposeNode?.(path) + return { path } + }, + editNode: async (path, content) => { + deps.ensureEditable?.() + assertInFolder(path) + assertPipelineAnnotation(content) + const drafts = deps.getDrafts() + const existing = drafts.get(path) + // Base the edit on the existing draft's / deployed script object and replace + // ONLY the content — preserving hash, summary, description, tag, schema, and + // settings. Rebuilding a fresh script would wipe that metadata: deploying + // from the pane (auto_parent) would update the script while clearing it, and + // the route "Save all" path (no parent_hash) could hit the path-conflict + // branch on the occupied path. + let baseScript: Script + if (existing) { + baseScript = existing.script + } else { + const workspace = deps.getWorkspace() + if (!workspace) throw new Error('No workspace is selected.') + baseScript = await ScriptService.getScriptByPath({ workspace, path }) + } + const inferred = await inferOutputAssets(baseScript.language, content) + const next = new Map(drafts) + next.set(path, { + localId: existing?.localId ?? deps.newDraftLocalId(), + script: { ...baseScript, content }, + outputAssets: inferred.length > 0 ? inferred : existing?.outputAssets + }) + deps.setDrafts(next) + deps.onShowDrafts?.() + deps.onProposeNode?.(path) + }, + removeProposedNode: async (path) => { + if (!deps.getDrafts().has(path)) { + throw new Error(`No unsaved draft at '${path}' to discard.`) + } + const next = new Map(deps.getDrafts()) + next.delete(path) + deps.setDrafts(next) + deps.onForgetPath?.(path) + }, + testNode: async (path, args) => { + const workspace = deps.getWorkspace() + if (!workspace) return undefined + const draft = deps.getDrafts().get(path) + try { + let jobId: string + if (draft) { + // Un-deployed/edited body: preview-run the draft content so it can be + // tested before deploying. + jobId = await JobService.runScriptPreview({ + workspace, + requestBody: { + path, + content: draft.script.content, + language: draft.script.language, + args: args ?? {} + } + }) + } else { + // test_pipeline_node previews ONE node — never fan out to downstream + // deployed subscribers via the backend asset dispatcher (which would + // run side-effecting deployed scripts the user didn't ask for). + jobId = await JobService.runScriptByPath({ + workspace, + path, + requestBody: { ...(args ?? {}), _wmill_skip_asset_dispatch: true } + }) + } + deps.onRunStarted?.(jobId, path) + return jobId + } catch (e: any) { + sendUserToast(`Run failed: ${e?.body ?? e?.message ?? e}`, true) + return undefined + } + } + } + + return helpers +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts new file mode 100644 index 0000000000..bdecadff91 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts @@ -0,0 +1,184 @@ +import type { AssetKind, Script } from '$lib/gen' +import type { AssetWithAltAccessType } from '$lib/components/assets/lib' +import type { AssetGraphSelection } from './types' +import { + parsePipelineAnnotations, + type ColumnLineage, + type PipelineAnnotations +} from './parsePipelineAnnotations' +import type { PipelineDraft } from './pipelineAiHelpers' + +// ============================================================================ +// Externalized pipeline-editor state — the data-pipeline analogue of the flow +// editor's `flowStore` / `flowStateStore`. It owns the in-flight draft Map, the +// live editor overlays, and the current selection: the substrate the route page +// editor and the in-session preview both render through (via the shared +// ). Persistence, graph resolution, run dispatch, and deploy +// stay with the consumer; this is a plain reactive bag so a consumer can +// read/mutate it without prop plumbing. +// ============================================================================ + +const EMPTY_ANNOTATIONS: PipelineAnnotations = parsePipelineAnnotations('') + +type LiveAnnotations = { scriptPath: string | undefined; annotations: PipelineAnnotations } +type LiveBodyAssets = { + scriptPath: string | undefined + assets: AssetWithAltAccessType[] + columnLineage?: ColumnLineage[] +} +type LiveContent = { scriptPath: string | undefined; content: string } + +export class PipelineEditorState { + /** In-flight drafts keyed by script path (manual + AI-staged). */ + drafts = $state>(new Map()) + /** Draft open in the details pane (mutually exclusive with `selection`). */ + activeDraftPath = $state(undefined) + /** The persisted node/asset selected on the canvas. */ + selection = $state(undefined) + + /** Live-parsed annotations of the open script (refreshed per keystroke). */ + liveAnnotations = $state({ + scriptPath: undefined, + annotations: EMPTY_ANNOTATIONS + }) + /** Live-inferred body read/write assets of the open script. */ + liveBodyAssets = $state({ scriptPath: undefined, assets: [] }) + /** The open draft's live editor buffer. */ + liveContent = $state({ scriptPath: undefined, content: '' }) + + /** Set true once a draft bundle was restored from the DB on load — drives the + * route toolbar's one-shot "Loaded from draft" hint. Written by the editor's + * autosave hydrate when persistence is enabled. */ + loadedFromDbDraft = $state(false) + + /** Folder this state is scoped to. Used by the in-session preview (where one + * instance is reused across editor hide/show) to detect a retarget to a + * different folder and reset, so stale drafts don't bleed across folders. */ + folder = $state(undefined) + + /** True once the DB draft bundle for the current folder has been hydrated + * into this instance. Gated per-instance (not per component mount) so the + * in-session preview hydrates ONCE when its runtime is fresh and then keeps + * the in-memory drafts across editor hide/show — re-reading the DB on every + * remount would race a not-yet-flushed autosave and drop a just-staged draft. + * Reset to false on a folder retarget so the new folder re-hydrates. */ + hydratedFromDb = $state(false) + + /** Clear all in-flight state. Used when the session preview retargets a + * different pipeline folder (a same-folder remount keeps the drafts). */ + reset = () => { + this.drafts = new Map() + this.activeDraftPath = undefined + this.selection = undefined + this.clearLiveOverlays() + this.loadedFromDbDraft = false + // Force a re-hydrate from the DB draft of the newly-targeted folder. + this.hydratedFromDb = false + } + + #nextDraftLocalId = 0 + // Arrow fields so `pe.method` can be passed straight as a callback (the + // details pane takes onDraftPersist / onAnnotationsChange / … by reference). + newDraftLocalId = (): string => { + this.#nextDraftLocalId += 1 + return `pe-${this.#nextDraftLocalId}` + } + + handleAnnotationsChange = (scriptPath: string | undefined, annotations: PipelineAnnotations) => { + this.liveAnnotations = { scriptPath, annotations } + } + handleAssetsChange = ( + scriptPath: string | undefined, + assets: AssetWithAltAccessType[], + columnLineage?: ColumnLineage[] + ) => { + this.liveBodyAssets = { scriptPath, assets, columnLineage } + } + handleContentChange = (scriptPath: string | undefined, content: string) => { + this.liveContent = { scriptPath, content } + } + + clearLiveOverlays = () => { + this.liveAnnotations = { scriptPath: undefined, annotations: EMPTY_ANNOTATIONS } + this.liveBodyAssets = { scriptPath: undefined, assets: [] } + this.liveContent = { scriptPath: undefined, content: '' } + } + + /** Drop per-path editor state when a path goes away. Does NOT touch + * consumer-owned per-path state (e.g. the route page's save errors — the + * route layers that on in its own wrapper). */ + forgetPath = (path: string) => { + if (this.activeDraftPath === path) this.activeDraftPath = undefined + if (this.selection?.kind === 'runnable' && this.selection.path === path) + this.selection = undefined + if (this.liveAnnotations.scriptPath === path) + this.liveAnnotations = { scriptPath: undefined, annotations: EMPTY_ANNOTATIONS } + if (this.liveBodyAssets.scriptPath === path) + this.liveBodyAssets = { scriptPath: undefined, assets: [] } + if (this.liveContent.scriptPath === path) + this.liveContent = { scriptPath: undefined, content: '' } + } + + discardDraft = (path: string) => { + if (!this.drafts.has(path)) return + const next = new Map(this.drafts) + next.delete(path) + this.drafts = next + this.forgetPath(path) + } + + /** Commit body edits + inferred outputs back into the drafts Map on pane + * teardown (deferred a microtask so a same-batch discard doesn't resurrect the + * entry). Verbatim port of the route page's `handleDraftPersist`. */ + handleDraftPersist = ( + p: string, + snapshot: { content: string; writes: { kind: AssetKind; path: string }[]; script?: Script } + ) => { + queueMicrotask(() => { + const d = this.drafts.get(p) + if (!d) { + if (!snapshot.script) return + const next = new Map(this.drafts) + next.set(p, { + localId: this.newDraftLocalId(), + script: snapshot.script, + outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined + }) + this.drafts = next + return + } + // `?? 0` is load-bearing: an undefined `outputAssets` (a no-output draft) + // vs an empty inferred `writes` both mean "no writes". Without the + // coalesce, `undefined === 0` is false, so this never short-circuits — + // every persist re-writes the drafts Map with an equivalent object, + // re-triggering the pane's emit → graph re-derive → persist, an infinite + // microtask loop (hangs the tab without an effect-depth throw). + const writesEqual = + (d.outputAssets?.length ?? 0) === snapshot.writes.length && + (d.outputAssets ?? []).every( + (a, i) => a.kind === snapshot.writes[i]?.kind && a.path === snapshot.writes[i]?.path + ) + if (d.script.content === snapshot.content && writesEqual) return + const next = new Map(this.drafts) + next.set(p, { + ...d, + script: { ...d.script, content: snapshot.content }, + outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined + }) + this.drafts = next + }) + } + + /** The draft open in the pane, if any. */ + get activeDraft(): PipelineDraft | undefined { + return this.activeDraftPath ? this.drafts.get(this.activeDraftPath) : undefined + } + + /** Whichever script is open — the active draft, or a selected persisted script. */ + get openScriptPath(): string | undefined { + if (this.activeDraftPath) return this.activeDraftPath + if (this.selection?.kind === 'runnable' && this.selection.runnable_kind === 'script') + return this.selection.path + return undefined + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts new file mode 100644 index 0000000000..0fae3f836d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect } from 'vitest' +import { PipelineEditorState } from './pipelineEditorState.svelte' +import type { PipelineDraft } from './pipelineAiHelpers' +import type { AssetKind } from '$lib/gen' + +// handleDraftPersist defers its commit a microtask (so a same-batch discard can +// win); flush that microtask before asserting. +const flushMicrotasks = () => new Promise((resolve) => queueMicrotask(() => resolve())) + +function draft(content: string, outputAssets?: { kind: AssetKind; path: string }[]): PipelineDraft { + return { + localId: 'pe-1', + script: { path: 'f/x/n', language: 'duckdb', content } as PipelineDraft['script'], + outputAssets + } +} + +describe('PipelineEditorState.handleDraftPersist', () => { + // Regression: a no-output draft has `outputAssets: undefined`; the details pane + // infers an empty `writes: []`. Both mean "no writes", so persisting unchanged + // content+writes must be a no-op. The earlier `undefined === 0` length check made + // it false, so every persist re-wrote the drafts Map with an equivalent object — + // re-triggering the pane's emit → graph re-derive → persist, an infinite + // microtask loop that froze the tab. The Map reference must stay identical. + it('is idempotent for a no-output draft (undefined outputAssets vs empty inferred writes)', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { content: 'SELECT 1', writes: [] }) + await flushMicrotasks() + expect(pe.drafts).toBe(before) + }) + + it('re-writes the drafts Map when the content actually changes', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { content: 'SELECT 2', writes: [] }) + await flushMicrotasks() + expect(pe.drafts).not.toBe(before) + expect(pe.drafts.get('f/x/n')?.script.content).toBe('SELECT 2') + }) + + it('re-writes the drafts Map when the inferred writes actually change', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { + content: 'SELECT 1', + writes: [{ kind: 'resource' as AssetKind, path: 'f/x/out' }] + }) + await flushMicrotasks() + expect(pe.drafts).not.toBe(before) + expect(pe.drafts.get('f/x/n')?.outputAssets).toEqual([{ kind: 'resource', path: 'f/x/out' }]) + }) + + it('stays idempotent when outputAssets and inferred writes match (non-empty)', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([ + ['f/x/n', draft('SELECT 1', [{ kind: 'resource' as AssetKind, path: 'f/x/out' }])] + ]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { + content: 'SELECT 1', + writes: [{ kind: 'resource' as AssetKind, path: 'f/x/out' }] + }) + await flushMicrotasks() + expect(pe.drafts).toBe(before) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index dd4c64b225..378c4e1469 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -66,11 +66,14 @@ describe('resolveGraph', () => { expect(resolveGraph(input({ base }))).toEqual(base) }) - it('draft: adds an unsaved runnable + write edge from the static outputAsset', () => { + it('draft: adds an unsaved runnable + write edge from outputAssets', () => { const drafts = new Map([ [ 'f/x/d', - { script: { content: '' }, outputAsset: { kind: 's3object' as const, path: '/out.json' } } + { + script: { content: '' }, + outputAssets: [{ kind: 's3object' as const, path: '/out.json' }] + } ] ]) const r = resolveGraph(input({ drafts })) @@ -93,22 +96,6 @@ describe('resolveGraph', () => { }) }) - it('draft: outputAssets snapshot wins over the static outputAsset', () => { - const drafts = new Map([ - [ - 'f/x/d', - { - script: { content: '' }, - outputAsset: { kind: 's3object' as const, path: '/old.json' }, - outputAssets: [{ kind: 's3object' as const, path: '/new.json' }] - } - ] - ]) - const r = resolveGraph(input({ drafts })) - expect(r.edges.map((e) => e.asset_path)).toContain('/new.json') - expect(r.edges.map((e) => e.asset_path)).not.toContain('/old.json') - }) - it('active draft: live body writes are authoritative over the snapshot', () => { const drafts = new Map([ [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index b7c82822fa..e223379adb 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -15,7 +15,6 @@ import { /** Minimal structural shape of a pipeline draft `resolveGraph` needs. */ export type GraphDraft = { script: { content: string } - outputAsset?: { kind: AssetKind; path: string } outputAssets?: Array<{ kind: AssetKind; path: string }> } @@ -280,20 +279,15 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { unsaved: true } } - // Output asset(s): three-tier resolution. + // Output asset(s): two-tier resolution. // 1. Active draft (the body the user is editing right now): // live body inference is authoritative — renaming a // CREATE TABLE target or writeS3File path retires the // old output node and surfaces the new one as the user // types. - // 2. Inactive draft with a captured `outputAssets` snapshot - // (taken on the last pane transition): use those, so a - // draft the user already edited keeps its renamed outputs - // after they've clicked elsewhere. - // 3. Fallback to the static `outputAsset` seeded at draft - // creation — covers fresh drafts and parser misses (e.g. - // WIN-1943: wmill.writeS3File({s3, storage}) object form - // not yet detected by the TS parser). + // 2. Inactive draft: its captured `outputAssets` (inferred at + // creation/last edit, or the seeded output for a fresh draft + // whose body doesn't yet write anything inferable). const liveForThisDraft = liveBodyAssets.scriptPath === path const writeOuts: Array<{ kind: AssetKind; path: string }> = [] if (liveForThisDraft) { @@ -301,9 +295,6 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { } else if (d.outputAssets) { writeOuts.push(...d.outputAssets) } - if (writeOuts.length === 0 && d.outputAsset) { - writeOuts.push(d.outputAsset) - } // `// materialize ` declares a write output via annotation, not // the SQL body, so the body-inference tiers above miss it. Add it from // the live-parsed annotations so an edited materialize script keeps its diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 9276f500af..274d218e82 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -49,6 +49,7 @@ export interface AssetGraphRunnableNode { // Synthesized by the page from a local draft; the script doesn't exist // in the DB yet. Drives a dashed/lower-opacity rendering to mirror how // unsaved triggers are styled — visually distinct from persisted nodes. + // AI-built nodes are plain drafts too (no separate pending/approval state). unsaved?: boolean } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index fdba95ca7f..343965b709 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -88,6 +88,11 @@ import { type GlobalToolHelpers } from './global/core' import { isGlobalAiEnabled } from './global/gate' +import { + pipelineTools, + getPipelinePromptSection, + type PipelineAIChatHelpers +} from './pipeline/core' import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userScopedStorage' import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { AttachedFilesStore } from './files/attachedFiles.svelte' @@ -254,6 +259,7 @@ export class AIChatManager { skipResponsesApi = false mode = $state(AIMode.NAVIGATOR) + pipelineAiChatHelpers = $state(undefined) readonly isOpen = $derived(chatState.size > 0) savedSize = $state(0) instructions = $state('') @@ -649,18 +655,11 @@ export class AIChatManager { ) switch (result) { case 'ok': - await this.historyManager.saveChat( - this.displayMessages, - this.messages, - this.contextUsage - ) + await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) sendUserToast('Conversation compacted.') break case 'empty': - sendUserToast( - 'Compaction produced an empty summary — conversation left unchanged.', - true - ) + sendUserToast('Compaction produced an empty summary — conversation left unchanged.', true) break case 'error': sendUserToast('Failed to compact the conversation.', true) @@ -958,28 +957,7 @@ export class AIChatManager { this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] this.helpers = {} } else if (mode === AIMode.GLOBAL) { - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(mode), { - previewTools: this.isSessionChat, - skills: this.globalSkills - }) - this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) - this.helpers = { - ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), - testActiveFlow: async (args?: Record) => - this.flowAiChatHelpers?.testFlow(args), - attachedFiles: this.attachedFiles, - getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', - setUserInstructions: (instructions: string) => { - const prompts = getUserCustomPrompts() - if (instructions.trim()) { - prompts[AIMode.GLOBAL] = instructions - } else { - delete prompts[AIMode.GLOBAL] - } - setUserCustomPrompts(prompts) - this.rebuildGlobalSystemMessage() - } - } satisfies GlobalToolHelpers + this.configureGlobalMode() void this.refreshGlobalSkills() } else if (mode === AIMode.APP) { const customPrompt = getCombinedCustomPrompt(mode) @@ -992,6 +970,43 @@ export class AIChatManager { // Fetch the workspace's AI skills and, if GLOBAL mode is still active, rebuild // the system message so the next chat-loop iteration advertises them. Ignore // stale resolves so workspace changes cannot overwrite newer skills. + // Build the global-mode system message, tools, and helpers, layering on the + // pipeline surface when a /pipeline editor has registered helpers. Centralized + // so changeMode, refreshGlobalSkills, and setPipelineHelpers stay consistent — + // each rebuild would otherwise drop the pipeline augmentation the others added. + private configureGlobalMode = () => { + const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { + previewTools: this.isSessionChat, + skills: this.globalSkills + }) + const baseHelpers: GlobalToolHelpers = { + ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), + testActiveFlow: async (args?: Record) => this.flowAiChatHelpers?.testFlow(args), + attachedFiles: this.attachedFiles, + getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', + setUserInstructions: (instructions: string) => { + const prompts = getUserCustomPrompts() + if (instructions.trim()) { + prompts[AIMode.GLOBAL] = instructions + } else { + delete prompts[AIMode.GLOBAL] + } + setUserCustomPrompts(prompts) + this.rebuildGlobalSystemMessage() + } + } + const pipeline = this.pipelineAiChatHelpers + if (pipeline) { + systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) + this.tools = [...globalToolsFor({ sessionPreview: this.isSessionChat }), ...pipelineTools] + this.helpers = { ...baseHelpers, pipeline } + } else { + this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) + this.helpers = baseHelpers + } + this.systemMessage = systemMessage + } + refreshGlobalSkills = async (workspace = get(workspaceStore) ?? '') => { const refreshId = ++this.globalSkillsRefreshId const skills = await loadWorkspaceSkills(workspace) @@ -1000,10 +1015,7 @@ export class AIChatManager { } this.globalSkills = skills if (this.mode === AIMode.GLOBAL) { - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { - previewTools: this.isSessionChat, - skills - }) + this.configureGlobalMode() } } @@ -1014,10 +1026,18 @@ export class AIChatManager { if (this.mode !== AIMode.GLOBAL) { return } - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { + const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { previewTools: this.isSessionChat, skills: this.globalSkills }) + // Preserve the active pipeline-editor augmentation that configureGlobalMode + // adds — otherwise update_user_instructions (which calls this) would drop the + // /pipeline/ context + direct-draft/materialize guidance mid-session. + const pipeline = this.pipelineAiChatHelpers + if (pipeline) { + systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) + } + this.systemMessage = systemMessage } private expandGlobalSkillCommand = (instructions: string): string => { @@ -2132,7 +2152,7 @@ export class AIChatManager { moduleState && !moduleState.previewSuccess ? getStringError(moduleState.previewResult) : undefined, - getCode: () => module.value.type === 'rawscript' ? module.value.content : '', + getCode: () => (module.value.type === 'rawscript' ? module.value.content : ''), lang: module.value.language, path: module.id, ...editorRelated @@ -2176,6 +2196,28 @@ export class AIChatManager { } } + // Registered by the /pipeline editor while it is mounted. Rebuilds the global + // tool set so the pipeline tools appear (and disappear on unregister). Pipeline + // AI edits apply directly as drafts, so there is nothing to auto-accept. + // Returns a cleanup that tears the registration back down. + setPipelineHelpers = (pipelineHelpers: PipelineAIChatHelpers) => { + this.pipelineAiChatHelpers = pipelineHelpers + untrack(() => { + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + }) + + return () => { + this.pipelineAiChatHelpers = undefined + untrack(() => { + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + }) + } + } + /** * Refresh cached datatables from the app helpers (async) * Creates one context element per table (not per datatable) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b85e271632..90dcb655f4 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -56,6 +56,7 @@ import { createInlineScriptSession } from '../flow/inlineScriptsUtils' import { getDatatableSdkReference, getFlowPrompt, + getPipelinePrompt, getRawAppPrompt, getResourcePrompt, getScriptPrompt @@ -138,7 +139,9 @@ const INSTRUCTION_SUBJECTS = [ ] as const satisfies readonly WorkspaceItemType[] // `datatable` is not a workspace item type, but the model can request the // datatable SDK reference (the wmill.datatable() runnable API) the same way. -const INSTRUCTION_SUBJECTS_EXTRA = ['datatable'] as const +// `pipeline` likewise isn't an item type — a data pipeline is a set of +// annotated scripts in a folder, so it gets authoring guidance, not a CRUD type. +const INSTRUCTION_SUBJECTS_EXTRA = ['datatable', 'pipeline'] as const const ALL_INSTRUCTION_SUBJECTS = [...INSTRUCTION_SUBJECTS, ...INSTRUCTION_SUBJECTS_EXTRA] as const const MAX_LIST_LIMIT = 100 type ActiveGlobalEditorType = Extract @@ -171,7 +174,7 @@ const scriptLangSchema = z.enum($ScriptLang.enum) const getInstructionsSchema = z.object({ subject: instructionSubjectSchema.describe( - 'What to get authoring instructions for: a workspace item type (script, flow, resource, app) or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' + 'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' ), language: scriptLangSchema .optional() @@ -206,7 +209,9 @@ const updateUserInstructionsSchema = z.object({ .string() .min(1) .optional() - .describe("Required when operation is 'append': the instruction to add. Ignored for 'replace'."), + .describe( + "Required when operation is 'append': the instruction to add. Ignored for 'replace'." + ), old_string: z .string() .min(1) @@ -678,11 +683,13 @@ const deleteAppRunnableSchema = z.object({ const openPreviewSchema = z.object({ kind: z - .enum(['script', 'flow', 'raw_app']) + .enum(['script', 'flow', 'raw_app', 'pipeline']) .describe( - 'Item kind to preview. Use "raw_app" for code-based apps (created via init_app). The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' + 'Item kind to preview. Use "raw_app" for code-based apps (created via init_app). Use "pipeline" to show the data-pipeline graph for a folder — here `path` is the folder name, not an item path. The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' ), - path: z.string().describe('Workspace path of the item to preview.') + path: z + .string() + .describe('Workspace path of the item to preview, or the folder name when kind is "pipeline".') }) const getPreviewStatusSchema = z.object({}) @@ -825,6 +832,7 @@ Rules: - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource. - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. +- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow. - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. @@ -833,6 +841,7 @@ Rules: previewTools ? ` - After writing or substantially editing a script / flow / app draft, show it via open_preview(kind, path) so the user sees the editor and live preview right next to the chat. First check whether it is already shown: if unsure, call get_preview_status. Only call open_preview (or offer to) when no preview is open or it is showing a different item — don't re-open a preview already showing the item you just edited. +- Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node — this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node — use ONLY those to add or change pipeline nodes, never write_script for a pipeline node — they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). - get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected.` : '' @@ -1608,6 +1617,10 @@ Datatables are workspace-scoped managed PostgreSQL databases. In chat, explore a ${getDatatableSdkReference(lang)}` } +function getPipelineInstructions(): string { + return getPipelinePrompt() +} + function getInstructions(subject: InstructionSubject, language?: ScriptLang): string { switch (subject) { case 'script': @@ -1620,6 +1633,8 @@ function getInstructions(subject: InstructionSubject, language?: ScriptLang): st return getAppInstructions() case 'datatable': return getDatatableInstructions(language) + case 'pipeline': + return getPipelineInstructions() } } @@ -1672,7 +1687,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( getInstructionsSchema, 'get_instructions', - 'Get authoring guidance for scripts, flows, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' + 'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' ), fn: async ({ args, toolId, toolCallbacks }) => { const parsed = getInstructionsSchema.parse(args) @@ -2490,7 +2505,7 @@ function activeFlowTestFromCtx( export type OpenPreviewHandler = (req: { sessionId: string | undefined - kind: 'script' | 'flow' | 'raw_app' + kind: 'script' | 'flow' | 'raw_app' | 'pipeline' path: string }) => string @@ -2501,7 +2516,7 @@ export function setOpenPreviewHandler(handler: OpenPreviewHandler | undefined): } function openSessionPreview( - args: { kind: 'script' | 'flow' | 'raw_app'; path: string }, + args: { kind: 'script' | 'flow' | 'raw_app' | 'pipeline'; path: string }, sessionId: string | undefined ) { if (!openPreviewHandler) { diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts new file mode 100644 index 0000000000..4f3e31986d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect, vi } from 'vitest' + +// `../shared` transitively pulls in the monaco editor (and its CSS), which the +// node test environment can't load — mirror the sibling chat tests' stub. +vi.mock('monaco-editor', () => ({ editor: {} })) + +import { + pipelineTools, + getPipelinePromptSection, + type PipelineAIChatHelpers, + type PipelineContext +} from './core' +import type { ToolCallbacks } from '../shared' + +function toolByName(name: string) { + const tool = pipelineTools.find((t) => t.def.function.name === name) + if (!tool) throw new Error(`tool ${name} not found`) + return tool +} + +function noopCallbacks(): ToolCallbacks { + return { setToolStatus: () => {}, removeToolStatus: () => {} } +} + +const sampleContext: PipelineContext = { + folder: 'analytics', + mode: 'edit', + nodes: [ + { + path: 'f/analytics/orders', + language: 'bun', + unsaved: true, + writes: ['ducklake://main/orders'], + reads: [], + triggers: ['schedule'] + } + ], + assets: ['ducklake://main/orders'] +} + +function makeHelpers(overrides: Partial = {}): { + helpers: { pipeline: PipelineAIChatHelpers } + calls: Record +} { + const calls: Record = {} + const record = + (name: string, ret?: any) => + (...args: any[]) => { + ;(calls[name] ??= []).push(args) + return ret + } + const pipeline: PipelineAIChatHelpers = { + getPipelineContext: () => sampleContext, + getNodeBody: async (path: string) => { + calls.getNodeBody = [...(calls.getNodeBody ?? []), [path]] + return { language: 'bun', content: 'export async function main() { return 1 }' } + }, + proposeNode: async (input) => { + calls.proposeNode = [...(calls.proposeNode ?? []), [input]] + return { path: input.path } + }, + editNode: async (path, content) => { + calls.editNode = [...(calls.editNode ?? []), [path, content]] + }, + removeProposedNode: record('removeProposedNode'), + testNode: async () => 'job-123', + ...overrides + } + return { helpers: { pipeline }, calls } +} + +describe('pipeline tools', () => { + it('exposes the expected tool surface', () => { + expect(pipelineTools.map((t) => t.def.function.name).sort()).toEqual([ + 'build_pipeline_node', + 'edit_pipeline_node', + 'get_pipeline_graph', + 'read_pipeline_node', + 'remove_pipeline_node', + 'test_pipeline_node' + ]) + }) + + it('get_pipeline_graph returns the live context as JSON', async () => { + const { helpers } = makeHelpers() + const out = await toolByName('get_pipeline_graph').fn({ + args: {}, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(JSON.parse(out)).toMatchObject({ folder: 'analytics' }) + }) + + it('build_pipeline_node forwards to proposeNode and does not deploy', async () => { + const { helpers, calls } = makeHelpers() + const out = await toolByName('build_pipeline_node').fn({ + args: { + path: 'f/analytics/clean', + language: 'bun', + content: '// pipeline\nexport async function main() {}', + output_kind: 'ducklake' + }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(calls.proposeNode?.[0]?.[0]).toMatchObject({ + path: 'f/analytics/clean', + language: 'bun', + outputKind: 'ducklake' + }) + expect(out).toContain('not deployed') + }) + + it('edit_pipeline_node reads then applies an exact find/replace', async () => { + const { helpers, calls } = makeHelpers({ + getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\nconst y = 2\n' }) + }) + await toolByName('edit_pipeline_node').fn({ + args: { path: 'f/analytics/orders', old_string: 'const x = 1', new_string: 'const x = 42' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(calls.editNode?.[0]?.[1]).toContain('const x = 42') + }) + + it('edit_pipeline_node surfaces a clear error when old_string is absent', async () => { + const { helpers } = makeHelpers({ + getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\n' }) + }) + await expect( + toolByName('edit_pipeline_node').fn({ + args: { path: 'f/analytics/orders', old_string: 'NOT THERE', new_string: 'x' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + ).rejects.toThrow(/was not found/) + }) + + it('mutation tools fail clearly when no pipeline editor is registered', async () => { + await expect( + toolByName('build_pipeline_node').fn({ + args: { path: 'f/a/b', language: 'bun', content: 'x' }, + workspace: 'w', + helpers: {}, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + ).rejects.toThrow(/No pipeline editor is open/) + }) + + it('test_pipeline_node requires confirmation', () => { + expect(toolByName('test_pipeline_node').requiresConfirmation).toBe(true) + }) +}) + +describe('getPipelinePromptSection', () => { + it('names the active folder and the direct-draft workflow', () => { + const section = getPipelinePromptSection(sampleContext) + expect(section).toContain('/pipeline/analytics') + expect(section).toContain('build_pipeline_node') + expect(section).toContain('directly as unsaved drafts') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts new file mode 100644 index 0000000000..f99bd0f210 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -0,0 +1,328 @@ +import { z } from 'zod' +import { $ScriptLang } from '$lib/gen/schemas.gen' +import type { ScriptLang } from '$lib/gen' +import { createToolDef, executeTestRun, findAndReplace, type Tool } from '../shared' +import type { PipelineOutputKind } from '$lib/components/assets/AssetGraph/pipelineTemplates' + +// ============================================================================ +// Pipeline AI chat tools. +// +// These tools extend the GLOBAL chat mode when the user is on a /pipeline/ +// editor (the page registers `PipelineAIChatHelpers` on the AIChatManager). They +// let the model read the live pipeline graph and BUILD/EDIT pipeline nodes +// (scripts annotated with `// pipeline`). Mutations don't deploy: they apply +// directly as an unsaved DRAFT on the canvas — the same way the flow/script +// editor applies AI edits — which the user then deploys. There is no separate +// approve/reject step: the draft IS the change. +// +// The pipeline tools are added on top of the full global tool set, so docs +// search, datatable SQL, and workspace-item tools are already available +// alongside them — this file only carries the pipeline-graph-specific surface. +// ============================================================================ + +/** Compact, model-facing summary of one node in the pipeline graph. */ +export type PipelineNodeSummary = { + path: string + language?: ScriptLang + /** Has an unsaved local edit (draft) not yet deployed. */ + unsaved: boolean + summary?: string + /** Asset URIs this node writes (its outputs). */ + writes: string[] + /** Asset URIs this node reads (its inputs). */ + reads: string[] + /** Declared `// on ` execution-DAG bindings (asset URIs or native kinds). */ + triggers: string[] +} + +/** Compact, model-facing snapshot of the whole pipeline graph. */ +export type PipelineContext = { + folder: string + mode: 'view' | 'edit' + nodes: PipelineNodeSummary[] + /** All storage assets referenced by the graph, as URIs. */ + assets: string[] +} + +/** + * Bridge the pipeline page registers on the AIChatManager. Reads expose the live + * graph; writes apply directly as unsaved drafts (never deploy). Kept intentionally + * small — the page owns the draft Map and canvas rendering. + */ +export interface PipelineAIChatHelpers { + getPipelineContext: () => PipelineContext + /** Read a node's source (the in-flight draft body if one exists, else deployed). */ + getNodeBody: (path: string) => Promise<{ language: ScriptLang; content: string } | undefined> + /** Create a brand-new pipeline node as an unsaved draft on the canvas. */ + proposeNode: (input: { + path: string + language: ScriptLang + content: string + outputKind?: PipelineOutputKind + }) => Promise<{ path: string }> + /** Replace an existing node's body, applied as an unsaved draft. */ + editNode: (path: string, content: string) => Promise + /** Discard the unsaved draft at a path (undo a build_pipeline_node). */ + removeProposedNode: (path: string) => Promise + /** Preview-run a node (draft body preferred). Returns the started job id. */ + testNode: (path: string, args?: Record) => Promise +} + +/** Helper bag the pipeline tools receive from the manager in global mode. */ +export type PipelineToolHelpers = { pipeline?: PipelineAIChatHelpers } + +function requirePipeline(helpers: PipelineToolHelpers): PipelineAIChatHelpers { + if (!helpers?.pipeline) { + throw new Error( + 'No pipeline editor is open. Pipeline tools only work on a /pipeline/ page in edit mode.' + ) + } + return helpers.pipeline +} + +const scriptLangSchema = z.enum($ScriptLang.enum) + +const outputKindSchema = z + .enum(['none', 'datatable', 'ducklake', 'materialize', 's3_parquet', 's3_object']) + .describe( + 'Kind of output asset this node materializes, used to seed the output edge on the canvas before the body is parsed: "materialize"/"ducklake" → a DuckLake table, "datatable" → a Postgres data table, "s3_parquet"/"s3_object" → an S3 file, "none" → side-effect only. Defaults to none.' + ) + +// ---------------------------------------------------------------------------- +// Read tools +// ---------------------------------------------------------------------------- + +const getPipelineGraphSchema = z.object({}) + +const getPipelineGraphToolDef = createToolDef( + getPipelineGraphSchema, + 'get_pipeline_graph', + "Read the live pipeline graph for the open /pipeline/ editor: its nodes (scripts), each node's language, asset reads/writes, declared triggers, and whether it has an unsaved draft edit. Call this before building or editing nodes so you reuse existing assets/paths and understand the current DAG." +) + +const readPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the pipeline node (script) to read.') +}) + +const readPipelineNodeToolDef = createToolDef( + readPipelineNodeSchema, + 'read_pipeline_node', + 'Read the full source of one pipeline node (its in-flight draft body if it has unsaved edits, otherwise the deployed body). Use before edit_pipeline_node so edits target the exact current text.' +) + +// ---------------------------------------------------------------------------- +// Mutation tools (apply directly as unsaved drafts; never deploy) +// ---------------------------------------------------------------------------- + +const buildPipelineNodeSchema = z.object({ + path: z + .string() + .describe( + "Workspace path for the new node, e.g. f//. Use the open pipeline's folder. Must not collide with an existing node." + ), + language: scriptLangSchema.describe( + 'Script language. SQL-shaped data work uses duckdb (DuckLake/S3) or postgresql (data tables); bun/python3 for general transforms.' + ), + content: z + .string() + .describe( + "Full script source. Start it with the `pipeline` annotation as a top-of-file comment in the LANGUAGE'S comment syntax — `-- pipeline` for SQL (duckdb/postgresql), `# pipeline` for python3/bash, `// pipeline` for bun/TS — to mark it a pipeline member; declare inputs the same way (e.g. `-- on `), and write outputs via the wmill SDK / SQL so the lineage edges are inferred. A `// pipeline` line in a SQL node is a syntax error. Read existing node bodies first to match conventions." + ), + output_kind: outputKindSchema.optional() +}) + +const buildPipelineNodeToolDef = createToolDef( + buildPipelineNodeSchema, + 'build_pipeline_node', + 'Build a NEW pipeline node. It is applied directly as an unsaved draft on the canvas (a dashed node wired by its parsed asset reads/writes) — it does NOT deploy; the user deploys it. Prefer this over editing for new scripts.', + { strict: false } +) + +const editPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node to edit.'), + old_string: z.string().min(1).describe("Exact text to find in the node's current source."), + new_string: z.string().describe('Replacement text.'), + replace_all: z + .boolean() + .optional() + .default(false) + .describe( + 'When true, replace every exact match. When false, old_string must match exactly once.' + ) +}) + +const editPipelineNodeToolDef = createToolDef( + editPipelineNodeSchema, + 'edit_pipeline_node', + 'Edit an existing pipeline node by exact find/replace. The result is applied directly as an unsaved draft (does NOT deploy). Call read_pipeline_node first to get the exact current text.', + { strict: false } +) + +const removePipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node whose unsaved draft should be discarded.') +}) + +const removePipelineNodeToolDef = createToolDef( + removePipelineNodeSchema, + 'remove_pipeline_node', + 'Discard the unsaved draft at a path (undo a build_pipeline_node). Only affects the in-flight draft — to delete a deployed node, ask the user to do it on the canvas.' +) + +const testPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node to preview-run.'), + args: z + .record(z.string(), z.any()) + .nullable() + .optional() + .describe('Arguments to pass to the script. Omit or pass null when none are needed.') +}) + +const testPipelineNodeToolDef = createToolDef( + testPipelineNodeSchema, + 'test_pipeline_node', + 'Preview-run one pipeline node (using its draft body when unsaved) and return the result/logs, without deploying. Requires user confirmation before it runs.', + { strict: false } +) + +// ---------------------------------------------------------------------------- +// Tool set +// ---------------------------------------------------------------------------- + +export const pipelineTools: Tool[] = [ + { + def: getPipelineGraphToolDef, + fn: async ({ helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + toolCallbacks.setToolStatus(toolId, { content: 'Reading pipeline graph...' }) + const ctx = pipeline.getPipelineContext() + toolCallbacks.setToolStatus(toolId, { + content: `Read pipeline graph (${ctx.nodes.length} node${ctx.nodes.length === 1 ? '' : 's'})`, + result: 'Success' + }) + return JSON.stringify(ctx, null, 2) + } + }, + { + def: readPipelineNodeToolDef, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path } = readPipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Reading node '${path}'...` }) + const node = await pipeline.getNodeBody(path) + if (!node) { + return `No pipeline node found at '${path}'. Call get_pipeline_graph to list the available nodes.` + } + toolCallbacks.setToolStatus(toolId, { content: `Read node '${path}'`, result: 'Success' }) + return JSON.stringify({ path, language: node.language, content: node.content }) + } + }, + { + def: buildPipelineNodeToolDef, + streamArguments: true, + showDetails: true, + showFade: true, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, language, content, output_kind } = buildPipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Building node '${path}'...` }) + await pipeline.proposeNode({ + path, + language: language as ScriptLang, + content, + outputKind: output_kind as PipelineOutputKind | undefined + }) + toolCallbacks.setToolStatus(toolId, { + content: `Added draft node '${path}'`, + result: 'Success' + }) + return `Pipeline node '${path}' added as an unsaved draft on the canvas. It is not deployed — the user deploys it.` + } + }, + { + def: editPipelineNodeToolDef, + streamArguments: true, + showDetails: true, + showFade: true, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, old_string, new_string, replace_all } = editPipelineNodeSchema.parse(args) + const node = await pipeline.getNodeBody(path) + if (!node) { + return `No pipeline node found at '${path}'. Call get_pipeline_graph to list the available nodes.` + } + toolCallbacks.setToolStatus(toolId, { content: `Editing node '${path}'...` }) + const updated = findAndReplace( + node.content, + old_string, + new_string, + replace_all ?? false, + 'node source' + ) + await pipeline.editNode(path, updated) + toolCallbacks.setToolStatus(toolId, { + content: `Edited draft '${path}'`, + result: 'Success' + }) + return `Pipeline node '${path}' updated as an unsaved draft on the canvas (not deployed).` + } + }, + { + def: removePipelineNodeToolDef, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path } = removePipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Discarding draft '${path}'...` }) + await pipeline.removeProposedNode(path) + toolCallbacks.setToolStatus(toolId, { + content: `Discarded draft '${path}'`, + result: 'Success' + }) + return `Discarded the unsaved draft at '${path}'.` + } + }, + { + def: testPipelineNodeToolDef, + requiresConfirmation: true, + confirmationMessage: 'Run pipeline node', + showDetails: true, + autoCollapseDetails: false, + fn: async ({ args, workspace, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, args: runArgs } = testPipelineNodeSchema.parse(args) + return executeTestRun({ + jobStarter: async () => { + const jobId = await pipeline.testNode(path, runArgs ?? undefined) + if (!jobId) { + throw new Error(`Could not start a run for node '${path}'.`) + } + return jobId + }, + workspace, + toolCallbacks, + toolId, + startMessage: `Starting run of '${path}'...`, + contextName: 'script' + }) + } + } +] + +/** + * Pipeline-specific guidance appended to the global system prompt when a + * /pipeline editor is open. Describes the annotation model and the direct-draft + * workflow so the model uses the pipeline tools rather than the generic + * write_script draft tools. + */ +export function getPipelinePromptSection(ctx: PipelineContext): string { + return ` + +Data Pipeline editor (ACTIVE): +- The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. +- Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. +- Build new nodes with build_pipeline_node and edit existing ones with edit_pipeline_node. These apply directly as unsaved drafts on the canvas (like the flow/script editor applies AI edits) — they DO NOT deploy. There is no separate Accept/Reject step. Prefer these over the generic write_script/edit_script draft tools while a pipeline is open. +- Reuse existing asset paths from the graph when wiring a downstream node to an upstream one (read the upstream's write asset, then \`// on\` that same URI). +- Only deploy when the user explicitly asks; the user deploys drafts from the canvas.` +} diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index b6ae577d4b..086760ac6e 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -13,6 +13,7 @@ type ListableRawApp } from '$lib/gen' import { resource } from 'runed' + import { getDraftItems } from '$lib/workspaceDrafts.svelte' import { userStore, workspaceStore } from '$lib/stores' import type uFuzzy from '@leeoniya/ufuzzy' import { @@ -73,23 +74,44 @@ type TableApp = TableItem type TableRawApp = TableItem - // Folders with ≥1 pipeline script (auto_kind='pipeline'). Used by - // TreeView to surface a "Pipeline" entry inside those folders. Cheap - // thanks to the partial index on script.auto_kind. + // Folders that are data pipelines, surfaced as their own "Pipeline" entry + // (the member scripts are folded into it, not listed individually). Two + // sources: deployed pipelines (folders with ≥1 `auto_kind='pipeline'` script, + // cheap via the partial index) AND bundle-phase pipelines that only exist as a + // `data_pipeline` draft so far — so a pipeline shows up the moment its first + // node is drafted, before anything is deployed. let pipelineFoldersRes = resource( () => $workspaceStore, async (ws) => { if (!ws) return new Set() + const folders = new Set() try { - const rows = await AssetService.listPipelineFolders({ workspace: ws }) - return new Set(rows.map((r) => r.folder)) + for (const r of await AssetService.listPipelineFolders({ workspace: ws })) + folders.add(r.folder) } catch { - // Decorative tree entry — degrade to "no pipelines" on failure. - return new Set() + // Decorative entry — degrade gracefully on failure. } + try { + for (const d of await getDraftItems(ws)) { + if (d.kind !== 'data_pipeline') continue + const m = d.path.match(/^f\/([^/]+)\/data_pipeline$/) + if (m) folders.add(m[1]) + } + } catch { + // Drafts unavailable — show deployed pipelines only. + } + return folders } ) - let pipelineFolders = $derived(pipelineFoldersRes.current ?? new Set()) + // Folders of pipeline-member scripts present in the current listing (captured + // in loadScripts before they're filtered out). Unioned in so a folder whose + // only pipeline node is a never-deployed `// pipeline` script draft — not in + // listPipelineFolders (deployed-only) nor a `data_pipeline` bundle — still gets + // a pipeline entry instead of vanishing. + let pipelineMemberFolders = $state(new Set()) + let pipelineFolders = $derived( + new Set([...(pipelineFoldersRes.current ?? []), ...pipelineMemberFolders]) + ) let scripts: TableScript[] | undefined = $state() let flows: TableFlow[] | undefined = $state() @@ -115,12 +137,26 @@ withoutDescription: true }) - scripts = loadedScripts.map((script: Script) => { - return { - canWrite: canWrite(script.path, script.extra_perms, $userStore) && !$userStore?.operator, - ...script - } - }) + // Pipeline-member scripts (`auto_kind='pipeline'`) are represented by their + // pipeline's entry, not listed individually — but capture their folders so + // the pipeline entry still surfaces (incl. a members-only / draft-only folder). + const memberFolders = new Set() + scripts = loadedScripts + .filter((script: Script) => { + if (script.auto_kind === 'pipeline') { + const m = script.path.match(/^f\/([^/]+)\//) + if (m) memberFolders.add(m[1]) + return false + } + return true + }) + .map((script: Script) => { + return { + canWrite: canWrite(script.path, script.extra_perms, $userStore) && !$userStore?.operator, + ...script + } + }) + pipelineMemberFolders = memberFolders loading = false } @@ -246,6 +282,25 @@ : undefined ) let filterUserFolders = $state(getLocalSetting(FILTER_USER_FOLDER_SETTING_NAME) == 'true') + + // Pipeline entries are rendered independently of the item list, so apply the + // same gates the items get — otherwise a pipeline would still show under the + // Flows/Apps tabs, in the archived view, under a label filter, or outside a + // selected owner. Pipelines are script-based units always at `f/`, so + // kind=script and the user-folder toggle always include them; kind=flow/app, + // archived, a label filter (pipelines carry no labels), and a non-matching + // owner exclude them. + let visiblePipelineFolders = $derived.by(() => { + if (archived) return new Set() + if (itemKind !== 'all' && itemKind !== 'script') return new Set() + if (labelFilter != undefined) return new Set() + if (ownerFilter == undefined) return pipelineFolders + return new Set( + [...pipelineFolders].filter( + (f) => `f/${f}` === ownerFilter || `f/${f}`.startsWith(ownerFilter + '/') + ) + ) + }) let includeWithoutMain = $state( getLocalSetting(INCLUDE_WITHOUT_MAIN_SETTING_NAME) ? getLocalSetting(INCLUDE_WITHOUT_MAIN_SETTING_NAME) == 'true' @@ -826,14 +881,17 @@ {#each new Array(6) as _} {/each} - {:else if filteredItems.length === 0} + {:else if filteredItems.length === 0 && (filter !== '' || visiblePipelineFolders.size === 0)} + {:else if treeView} loadScripts(includeWithoutMain)} on:flowChanged={loadFlows} @@ -850,7 +908,7 @@ {:else}
{#if filter === ''} - {#each [...pipelineFolders].sort() as folder (folder)} + {#each [...visiblePipelineFolders].sort() as folder (folder)} i && 'folderName' in i + // Hidden while searching: pipelines aren't part of the text filter (the list + // view hides their rows on a query too), so a folder matching the search + // shouldn't surface an unrelated Pipeline row. let hasPipeline = $derived( - depth === 0 && isFolderItem(item) && (pipelineFolders?.has(item.folderName) ?? false) + depth === 0 && + !isSearching && + isFolderItem(item) && + (pipelineFolders?.has(item.folderName) ?? false) ) const isFolder = isFolderItem diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index 59adf1aaa9..ef5cf25d58 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -24,7 +24,42 @@ let groupedItems: ReturnType | 'loading' = $state('loading') $effect(() => { items - untrack(() => (groupedItems = groupItems(items))) + pipelineFolders + isSearching + untrack(() => { + const grouped = groupItems(items) + // Ensure every pipeline folder is present at the top level so its + // "Pipeline" entry shows even when it has no listed items — a bundle-phase + // pipeline (only a draft so far) or a folder whose only scripts are + // pipeline members (folded into the pipeline, hidden from the list). + // Skip while searching: pipelines aren't part of the text filter (list view + // hides them on `filter !== ''`), so injecting them would surface unrelated + // folders in the results. + if (!isSearching) { + const present = new Set( + grouped + .filter((g) => 'folderName' in g) + .map((g) => (g as { folderName: string }).folderName) + ) + // Insert each missing pipeline folder among the existing folders in name + // order — `groupItems` already sorts user groups first then folders + // alphabetically, so inserting before the first greater-named folder + // keeps that ordering (rather than prepending out of order). + for (const folderName of [...(pipelineFolders ?? [])] + .filter((f) => !present.has(f)) + .sort()) { + const item = { folderName, items: [] } + const idx = grouped.findIndex( + (g) => + 'folderName' in g && + (g as { folderName: string }).folderName.localeCompare(folderName) > 0 + ) + if (idx < 0) grouped.push(item) + else grouped.splice(idx, 0, item) + } + } + groupedItems = grouped + }) }) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 19e71530e6..3e631a55e9 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -505,12 +505,16 @@ time: new Date(x.edited_at).getTime(), search_id: x.path })), - ...scripts.map((x) => ({ - ...x, - type: 'script' as 'script', - time: new Date(x.created_at).getTime(), - search_id: x.path - })), + // Pipeline-member scripts (`auto_kind='pipeline'`) are reached through + // their pipeline, not searched individually. + ...scripts + .filter((x) => x.auto_kind !== 'pipeline') + .map((x) => ({ + ...x, + type: 'script' as 'script', + time: new Date(x.created_at).getTime(), + search_id: x.path + })), ...apps.map((x) => ({ ...x, type: 'app' as 'app', diff --git a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte index 0716398131..0b529b2056 100644 --- a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte @@ -1,7 +1,7 @@ + +
+
+ + f/{path} + · data pipeline +
+
+ + {#if graphRes.loading && !graphRes.current && pe.drafts.size === 0} +
+ + Loading pipeline… +
+ {:else if graphRes.error && pe.drafts.size === 0} +
+ Failed to load pipeline: {graphRes.error.message} +
+ {:else} + + runNode(path, args)} + canRunByPath + onTestStateChange={(running) => { + const openPath = pe.openScriptPath + if (running && openPath) { + activeRunnable = { kind: 'script', path: openPath } + activeRunnables.arm(`script:${openPath}`) + activeRunnableJobId = undefined + } else if (!running && activeRunnable?.path === openPath) { + // Only clear the hint for the script the pane just finished — a + // canvas per-node run of a different script keeps its own hint. + activeRunnable = undefined + activeRunnableJobId = undefined + } + }} + onRunCompleted={() => { + activeRunnable = undefined + activeRunnableJobId = undefined + }} + onSelect={handleCanvasSelect} + onDraftSaved={afterSaved} + onPersistedSaved={afterSaved} + onScriptRemoved={async (removedPath) => { + pe.forgetPath(removedPath) + await graphRes.refetch() + }} + onScriptRenamed={async (oldPath, newPath) => { + // Repoint the selection so the canvas follows the renamed node instead + // of staying on the now-gone old path until an unrelated refetch. + if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) { + pe.selection = { ...pe.selection, path: newPath } + } + await graphRes.refetch() + }} + onDiscard={() => { + if (pe.activeDraftPath) pe.discardDraft(pe.activeDraftPath) + }} + onClose={() => { + pe.selection = undefined + pe.activeDraftPath = undefined + pe.clearLiveOverlays() + }} + /> + {/if} +
+
+ + + graphRes.refetch()} +/> diff --git a/frontend/src/lib/components/sessions/SessionItemNotFound.svelte b/frontend/src/lib/components/sessions/SessionItemNotFound.svelte index dc1a088c53..77c5173813 100644 --- a/frontend/src/lib/components/sessions/SessionItemNotFound.svelte +++ b/frontend/src/lib/components/sessions/SessionItemNotFound.svelte @@ -3,7 +3,11 @@ import type { WorkspaceItem, WorkspaceItemKind } from '$lib/components/workspacePicker' import type { SessionTarget } from './sessionState.svelte' - const KIND_NOT_FOUND_LABEL: Record = { + // `pipeline` targets never hit this component (they aren't slot-loaded, so they + // can't 404 through SessionEditorTarget) — exclude it from the kinds here. + type NotFoundKind = Exclude + + const KIND_NOT_FOUND_LABEL: Record = { flow: 'Flow', script: 'Script', raw_app: 'Raw app' @@ -14,7 +18,7 @@ path, onNavigate }: { - kind: SessionTarget['kind'] + kind: NotFoundKind path: string onNavigate?: (item: WorkspaceItem) => void } = $props() diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index 027089d4ae..f62aa07926 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -27,6 +27,7 @@ import FlowEditorView from './FlowEditorView.svelte' import ScriptEditorView from './ScriptEditorView.svelte' import RawAppEditorView from './RawAppEditorView.svelte' + import PipelineEditorView from './PipelineEditorView.svelte' import SessionWorkspaceBar from './SessionWorkspaceBar.svelte' import SessionForkBar from './SessionForkBar.svelte' import SessionDraftBar from './SessionDraftBar.svelte' @@ -261,7 +262,8 @@ {@const hasTarget = session.target?.kind === 'flow' || session.target?.kind === 'script' || - session.target?.kind === 'raw_app'} + session.target?.kind === 'raw_app' || + session.target?.kind === 'pipeline'} {@const hasEditor = mountEditor && hasTarget && editorVisible} {#snippet inputPreface()} @@ -469,6 +471,13 @@ onNavigate={pickEditorTarget} isActiveSession={sessionState.currentSessionId === sessionId} /> + {:else if session.target.kind === 'pipeline'} + {/if}
diff --git a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte index c4099edd91..e6b4e5fd2f 100644 --- a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte @@ -15,6 +15,11 @@ /** Summary supplied by the data source. Preferred over the one derived * from the loaded diff value, and shown before that value loads. */ summary?: string + /** Explicit unique row identity, overriding the default `kind/path`. For a + * row whose `kind/path` isn't unique on its own (a pipeline-bundle node + * shares `script/` with a standalone script draft at the same path) + * while `path` must stay the real edit/display target. */ + key?: string } @@ -110,6 +115,7 @@ // e.g. a runnable rendered as `script` at `/runnables/foo` vs a real // script literally at that path. Prefix synthetic items so the {#each} key, // load cache, row id and nav identity never collide with a real DiffRow. + if ('key' in d && d.key) return d.key return ('appPath' in d ? 'rawapp:' : '') + `${d.kind}/${d.path}` } diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index d7d9f77630..f8a9e1045d 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -1,6 +1,7 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity' import { get } from 'svelte/store' import { AIChatManager, AIMode } from '$lib/components/copilot/chat/AIChatManager.svelte' +import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' import { initFlow } from '$lib/components/flows/flowStore.svelte' import { AppService, @@ -79,6 +80,9 @@ export type SessionTargetKind = 'flow' | 'script' | 'raw_app' export interface SessionRuntime { readonly sessionId: string readonly manager: AIChatManager + // Pipeline target state — persists across editor hide/show (the pane unmounts + // on hide, so this can't be component-local) and across session switches. + readonly pipelineEditorState: PipelineEditorState // Kind-agnostic accessor over the per-kind load slots, for consumers (the // editor-target gate) that only need load state and not the typed store. slot(kind: SessionTargetKind): LoadSlot @@ -307,6 +311,12 @@ function createRuntime(session: Session): SessionRuntime { const rawApp: { val: SessionRuntime['rawApp']['val'] } = $state({ val: undefined }) const savedRawApp: { val: SessionRuntime['savedRawApp']['val'] } = $state({ val: undefined }) const rawAppSlot: LoadSlot = $state({ loadedPath: undefined, loading: false, notFound: false }) + + // Pipeline target state lives on the runtime (not the PipelineEditorView + // component) so the in-session drafts survive hide/show of the editor pane — + // the pane unmounts on hide, and a component-local store would be discarded. + const pipelineEditorState = new PipelineEditorState() + let runtimeLogRequester: RawAppRuntimeLogRequester | undefined = undefined let appRunsProvider: RawAppRunsProvider | undefined = undefined @@ -344,6 +354,7 @@ function createRuntime(session: Session): SessionRuntime { slot(kind: SessionTargetKind): LoadSlot { return kind === 'flow' ? flowSlot : kind === 'script' ? scriptSlot : rawAppSlot }, + pipelineEditorState, flowStore, flowStateStore, savedFlow, diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index e3d9e620f6..4e12d8622f 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -31,14 +31,18 @@ import type HistoryManager from '$lib/components/copilot/chat/HistoryManager.sve import { onUserChange } from '$lib/userScopedStorage' // Kinds the in-session editor pane can host. Legacy drag-and-drop apps are -// intentionally not previewable — only code-based 'raw_app' apps are. -export type SessionTarget = { kind: 'flow' | 'script' | 'raw_app'; path: string } +// intentionally not previewable — only code-based 'raw_app' apps are. A +// 'pipeline' target's `path` is the folder name (not a workspace item path): +// it hosts the data-pipeline graph editor for that folder, which uses its own +// fetch/draft model rather than the single-item load slots the other kinds share. +export type SessionTarget = { kind: 'flow' | 'script' | 'raw_app' | 'pipeline'; path: string } // Useful for filtering dropdowns / pickers to "items the side panel can open". export const EDITOR_TARGET_KINDS: ReadonlySet = new Set([ 'flow', 'script', - 'raw_app' + 'raw_app', + 'pipeline' ]) // Lifecycle status for a fork session. Git-parallel: diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 3056245e51..7933ef6d5e 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -4,7 +4,7 @@ import { page } from '$app/state' import Button from '$lib/components/common/button/Button.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' - import AssetGraphCanvas from '$lib/components/assets/AssetGraph/AssetGraphCanvas.svelte' + import PipelineGraphEditor from '$lib/components/assets/AssetGraph/PipelineGraphEditor.svelte' import { useActiveRunnableIds, isActiveEvent @@ -15,9 +15,7 @@ RunStatus } from '$lib/components/assets/AssetGraph/activeRunnables.svelte' import { usePipelineHistory } from '$lib/components/assets/AssetGraph/pipelineHistory.svelte' - import PipelineEventLog from '$lib/components/assets/AssetGraph/PipelineEventLog.svelte' import PipelineActivityPanel from '$lib/components/assets/AssetGraph/PipelineActivityPanel.svelte' - import AssetGraphDetailsPane from '$lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte' import PipelinePickerModal from '$lib/components/assets/AssetGraph/PipelinePickerModal.svelte' import { extractWrites, @@ -31,11 +29,7 @@ PipelineMode } from '$lib/components/assets/AssetGraph/types' import PipelineModeToggle from '$lib/components/assets/AssetGraph/PipelineModeToggle.svelte' - import { - parsePipelineAnnotations, - type ColumnLineage, - type PipelineAnnotations - } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' import { buildColumnGraph, type ColumnLineageGraph @@ -68,9 +62,11 @@ type PipelineOutputKind, type DraftTriggerSource } from '$lib/components/assets/AssetGraph/pipelineTemplates' - import { decodeState, encodeState } from '$lib/utils' - import { DraftService } from '$lib/gen' - import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { + createPipelineAiHelpers, + type PipelineDraft + } from '$lib/components/assets/AssetGraph/pipelineAiHelpers' + import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' import AutosaveIndicator from '$lib/components/AutosaveIndicator.svelte' import { onMount, tick, untrack } from 'svelte' import { aiChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte' @@ -98,14 +94,12 @@ type ScriptLang } from '$lib/gen' import { resource } from 'runed' - import { Pane, Splitpanes } from 'svelte-splitpanes' import { emptySchema, sendUserToast } from '$lib/utils' import type { Schema } from '$lib/common' import { beforeNavigate, goto } from '$app/navigation' import { fade } from 'svelte/transition' import { twMerge } from 'tailwind-merge' import Popover from '$lib/components/meltComponents/Popover.svelte' - import HideButton from '$lib/components/apps/editor/settingsPanel/HideButton.svelte' import { inferArgs, inferAssets } from '$lib/infer' import PipelineTriggerEditors from '$lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte' @@ -115,7 +109,33 @@ const DATA_KINDS = ['s3object', 'ducklake', 'datatable', 'volume'] let folder = $derived(page.params.folder as string) - let selection = $state(undefined) + + // Externalized editor state (drafts, live overlays, selection), shared with + // the in-session pipeline preview via PipelineEditorState. Referenced as + // `pe.*` throughout; the persistence / graph / run logic below stays here. + const pe = new PipelineEditorState() + + // The in-app folder switcher navigates same-route (`/pipeline/`), which + // reuses this page component — nothing remounts. Mirror the session preview's + // retarget guard: reset the editor state on a folder change so folder A's drafts + // don't display under B (and aren't autosaved into B's bundle), and so + // hydratedFromDb flips back to false and B's draft bundle re-hydrates. + $effect(() => { + const f = folder + untrack(() => { + if (pe.folder !== f) { + if (pe.folder !== undefined) { + pe.reset() + // Re-scope the Global chat's pipeline prompt to the new folder. The + // helper methods already read the reactive folder, but the system + // message string was built for the old one and is only rebuilt when + // Global mode is reconfigured — so rebuild it here. + aiChatManager.rebuildGlobalSystemMessage() + } + pe.folder = f + } + }) + }) // Page mode, URL-addressable via `?mode=`. No param = view (the // default): deployed-only graph focused on past/live executions. @@ -129,10 +149,10 @@ // selection / open draft / hidden-pane state so view opens on Activity // rather than a stale details pane (mirrors toggleActivity's show path). if (m === 'view' && mode === 'edit') { - selection = undefined - activeDraftPath = undefined + pe.selection = undefined + pe.activeDraftPath = undefined panelHidden = false - liveAnnotations = EMPTY_LIVE_ANNOTATIONS + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS } const url = new URL(page.url) if (m === 'view') url.searchParams.delete('mode') @@ -166,346 +186,44 @@ // output asset, and they all render on the graph simultaneously. // Saving removes a draft from the map; closing the pane keeps it so the // user can come back to it. - // Counter-based id source — sufficient for "stable across renames in - // this session"; doesn't need to survive a reload. (We have crypto. - // randomUUID() too but a short numeric id keeps localStorage tidy.) - let nextDraftLocalIdCounter = 0 - function newDraftLocalId(): string { - nextDraftLocalIdCounter += 1 - return `d${nextDraftLocalIdCounter}-${Date.now()}` - } + // The draft shape + draft Map + activeDraftPath now live in the shared + // PipelineEditorState (`pe`). `Draft` aliases the store's type so existing + // annotations keep working. + type Draft = PipelineDraft - type Draft = { - // Stable per-draft identifier, generated on first create and - // preserved across renames. Used to track concurrent deploys (a - // fast double-rename otherwise fires two saves that each leave a - // persisted script behind — the latest deploy archives the prior - // one keyed on this id). - localId: string - script: Script - // Undefined when the user picked `outputKind === 'none'` — the draft - // has no auto-generated output asset, so the graph overlay skips - // synthesizing a write edge for it. - outputAsset?: { kind: AssetKind; path: string } - // Inferred body writes from the last time this draft was open in - // the details pane. Captured on transition (selection change / - // pane close) so the canvas keeps showing the user's renamed - // outputs after they've clicked away. Falls back to outputAsset - // when undefined (initial state or parser miss). - outputAssets?: Array<{ kind: AssetKind; path: string }> - } - let drafts = $state>(new Map()) - - // Which draft (if any) is currently open in the details pane. When - // undefined and `selection` is set, the pane shows the persisted - // selection's script. Never both at once. - let activeDraftPath = $state(undefined) - - // Splitpanes sizes: bound so user-resized widths persist when the - // details pane is hidden + re-shown, or when switching between draft - // and persisted selections (previously the panes were sized inline, - // which made the right pane jump width when activeDraft was set - // since the left was hardcoded to 100%). When the right pane - // unmounts, svelte-splitpanes does NOT auto-stretch the remaining - // pane — the bound `leftPaneSize` stays at its last value and the - // other 40% renders as blank space. We explicitly set leftPaneSize - // to 100 in that state via a $derived effect below, and restore the - // stored split when the pane comes back. - let leftPaneSize = $state(60) - let rightPaneSize = $state(40) - let storedRightPaneSize = $state(40) + // Splitpane sizing + details-pane-open derivation live inside PipelineGraphEditor. // Explicit hide flag — keeps `selection` / `activeDraftPath` intact // so re-opening the pane re-uses them. Mirrors AppEditor's // hideRightPanel/showRightPanel pattern. let panelHidden = $state(false) - // Edit mode opens the pane only for a selection/draft; view mode - // keeps it open permanently — it shows the activity panel when nothing - // is selected and swaps to the details pane on node select. - let detailsPaneOpen = $derived( - mode === 'edit' - ? (selection != undefined || activeDraftPath != undefined) && !panelHidden - : !panelHidden - ) // View mode's idle pane is the activity feed; once a node is selected // (or the pane hidden) the only ways back were the pane's X and the // floating hide toggle — neither is named. The top-bar Activity toggle // is the explicit affordance: shows the feed from any state, hides the // pane when the feed is already showing. let activityShowing = $derived( - mode === 'view' && !panelHidden && selection == undefined && activeDraftPath == undefined + mode === 'view' && !panelHidden && pe.selection == undefined && pe.activeDraftPath == undefined ) function toggleActivity() { if (activityShowing) { panelHidden = true } else { - selection = undefined - activeDraftPath = undefined + pe.selection = undefined + pe.activeDraftPath = undefined panelHidden = false // Same reset as the pane's close button — clears the live // annotation overlay of whichever script was open. - liveAnnotations = EMPTY_LIVE_ANNOTATIONS + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS } } - $effect(() => { - if (detailsPaneOpen) { - // Pane visible: reapply the stored split. We don't depend on - // rightPaneSize here (only stored) so user resizes via the - // splitter handle aren't immediately overridden. - const restore = storedRightPaneSize - untrack(() => { - rightPaneSize = restore - leftPaneSize = 100 - restore - }) - } else { - // About to hide. Stash the current right size so the next show - // restores it, then expand the left pane to fill — splitpanes - // won't do this automatically when a Pane unmounts. - untrack(() => { - if (rightPaneSize > 0) storedRightPaneSize = rightPaneSize - leftPaneSize = 100 - }) - } - }) - - // All of this folder's in-flight drafts live in ONE per-user DB draft - // (typ `data_pipeline`) keyed at the folder, so they sync across devices - // and surface in the global drafts list — replacing the prior - // browser-only blob. The `f//...` path drives the backend's - // folder-write access check. localStorage is kept as a synchronous crash - // mirror (no size cap, survives a hard close inside the debounce window) - // but is only ever READ for the one-time import below; the DB is the - // source of truth on load. - const PIPELINE_DRAFT_KIND = 'data_pipeline' as const + // Draft autosave (the data_pipeline DraftService bundle) lives inside + // PipelineGraphEditor now; the route just supplies its path to the indicator. let pipelineDraftPath = $derived(`f/${folder}/data_pipeline`) - let storageKey = $derived(`pipeline-${folder}`) - type PipelineDraftBundle = { drafts: Array<[string, Draft]>; activeDraftPath?: string } - // Gate the persist effect until the initial DB load resolves so empty - // pre-hydration state can't clobber the server copy. `lastPersistedBundle` - // holds the last value we pushed so an unchanged re-render (and the - // just-loaded value itself) isn't re-POSTed. - let draftsHydrated = $state(false) - let lastPersistedBundle: string | undefined = undefined - // True once a bundle was restored from the DB on load — drives the - // AutosaveIndicator's one-shot "Loaded from draft" hint. - let loadedFromDbDraft = $state(false) - - function restoreBundle(bundle: PipelineDraftBundle) { - if (Array.isArray(bundle.drafts)) { - const loaded = new Map() - for (const entry of bundle.drafts) { - if (entry && typeof entry[0] === 'string' && entry[1]?.script) { - const d = entry[1] as Draft - // Backfill localId for state persisted by older builds. - if (typeof d.localId !== 'string' || d.localId === '') { - d.localId = newDraftLocalId() - } - loaded.set(entry[0], d) - } - } - if (loaded.size > 0) drafts = loaded - } - if (typeof bundle.activeDraftPath === 'string') { - activeDraftPath = bundle.activeDraftPath - } - } - - // The pre-DB localStorage blob, for the one-time migration when the user - // has no DB draft yet. - function readLocalBundle(): PipelineDraftBundle | undefined { - if (typeof localStorage === 'undefined') return undefined - const raw = localStorage.getItem(`pipeline-${folder}`) - if (!raw) return undefined - try { - const state = decodeState(raw) - if (state && (Array.isArray(state.drafts) || typeof state.activeDraftPath === 'string')) { - return { - drafts: Array.isArray(state.drafts) ? state.drafts : [], - activeDraftPath: - typeof state.activeDraftPath === 'string' ? state.activeDraftPath : undefined - } - } - } catch (e) { - console.warn('failed to read local pipeline state', e) - } - return undefined - } - - onMount(() => { - void hydrateDrafts() - }) - - async function hydrateDrafts() { - const ws = $workspaceStore - const path = pipelineDraftPath - try { - let bundle: PipelineDraftBundle | undefined - let serverSavedAt: string | undefined - if (ws) { - const row = await DraftService.getOwnDraft({ - workspace: ws, - kind: PIPELINE_DRAFT_KIND, - path - }) - if (row?.value) { - bundle = row.value as PipelineDraftBundle - serverSavedAt = row.created_at - loadedFromDbDraft = true - } - } - // One-time migration: no DB draft yet, but an older build left a - // localStorage blob — adopt it and let the persist effect push it up. - let migratedFromLocal = false - if (!bundle) { - const local = readLocalBundle() - if (local) { - bundle = local - migratedFromLocal = true - } - } - if (bundle) restoreBundle(bundle) - // Seed the conflict baseline: server timestamp when loaded from the - // DB, none otherwise (first save omits last_sync → backend first-push - // branch). A local migration counts as "nothing server-side yet". - UserDraftDbSyncer.recordRemoteSync( - { workspace: ws ?? '', itemKind: PIPELINE_DRAFT_KIND, path }, - migratedFromLocal ? undefined : serverSavedAt - ) - // Record what we loaded so the first persist run is a no-op — UNLESS - // we migrated from localStorage, which must push to the DB once. - if (!migratedFromLocal) { - lastPersistedBundle = bundle ? JSON.stringify(bundle) : undefined - } - } catch (e) { - console.warn('failed to load pipeline drafts', e) - } finally { - draftsHydrated = true - } - } - - // Persist on change: debounced DB sync (UserDraftDbSyncer handles the - // debounce + optimistic-concurrency) plus a synchronous localStorage - // mirror for crash recovery before the first DB confirm. - $effect(() => { - // Track deps explicitly so Svelte 5 re-runs on mutation. - // For the active draft, also snapshot the latest live body writes - // at serialize time. Without this, edits made since the last - // pane-transition (the cleanup that calls onDraftPersist) are - // lost on reload — the draft restores with stale `outputAssets` - // and the graph drops the corresponding edges until the user - // re-opens the draft and types something new. - const liveWritesSnapshot = - liveBodyAssets.scriptPath != undefined && drafts.has(liveBodyAssets.scriptPath) - ? extractWrites(liveBodyAssets.assets) - : undefined - const liveWritesPath = liveBodyAssets.scriptPath - // Live editor buffer for the open draft. `onDraftPersist` only commits - // content into the Map on pane teardown, so without this overlay the - // autosave (and a crash reload) would lag a whole editing session behind. - const liveContentPath = - liveContent.scriptPath != undefined && drafts.has(liveContent.scriptPath) - ? liveContent.scriptPath - : undefined - const liveContentValue = liveContent.content - const serialized = Array.from(drafts.entries()).map(([p, d]) => { - const outputAssets = - liveWritesSnapshot != undefined && liveWritesPath === p - ? liveWritesSnapshot.length > 0 - ? liveWritesSnapshot - : undefined - : d.outputAssets - const script = - liveContentPath === p && d.script.content !== liveContentValue - ? { ...d.script, content: liveContentValue } - : d.script - if (script === d.script && outputAssets === d.outputAssets) { - return [p, d] as [string, Draft] - } - return [p, { ...d, script, outputAssets }] as [string, Draft] - }) - const activePath = activeDraftPath - const key = storageKey - const ws = $workspaceStore - const path = pipelineDraftPath - const hydrated = draftsHydrated - untrack(() => { - // Don't touch storage until the initial load settled. - if (!hydrated) return - const isEmpty = serialized.length === 0 && !activePath - const bundle: PipelineDraftBundle | undefined = isEmpty - ? undefined - : { drafts: serialized, activeDraftPath: activePath } - // localStorage crash mirror — synchronous, no debounce, no size cap. - try { - if (typeof localStorage !== 'undefined') { - if (isEmpty) localStorage.removeItem(key) - else - localStorage.setItem( - key, - encodeState({ drafts: serialized, activeDraftPath: activePath }) - ) - } - } catch (e) { - console.warn('failed to mirror pipeline state', e) - } - const serializedBundle = bundle ? JSON.stringify(bundle) : undefined - if (serializedBundle === lastPersistedBundle) return - lastPersistedBundle = serializedBundle - if (!ws) return - void UserDraftDbSyncer.save({ - workspace: ws, - itemKind: PIPELINE_DRAFT_KIND, - path, - // `null` deletes the bundle once the last draft is gone. - value: bundle ?? null, - auto: true - }) - }) - }) - - // Live-parsed annotations from whatever script is currently open in the - // details pane (draft or existing). Refreshed on every keystroke via - // `onAnnotationsChange`. Used to overlay unsaved schedule / trigger-asset - // edges onto the graph so the editor buffer and the graph stay in sync. - let liveAnnotations = $state<{ - scriptPath: string | undefined - annotations: PipelineAnnotations - }>({ - scriptPath: undefined, - annotations: { - inPipeline: false, - triggerAssets: [], - nativeTriggers: [], - dataTests: [], - columnLineage: [] - } - }) - - // Live-inferred body assets (read/write usages parsed by inferAssets - // — e.g. CREATE TABLE in SQL, loadS3File / writeS3File in TS/Python). - // Refreshed via onAssetsChange. We use the write subset as the - // authoritative output node set for drafts whose body has been edited - // past the seeded template; without this, renaming a CREATE TABLE - // target leaves the stale auto-output node on the graph. - let liveBodyAssets = $state<{ - scriptPath: string | undefined - assets: AssetWithAltAccessType[] - columnLineage?: ColumnLineage[] - }>({ scriptPath: undefined, assets: [] }) - - // The open draft's live editor buffer, emitted by the pane on every - // keystroke (`onContentChange`). The persist effect overlays it onto the - // Map's copy so autosave reflects in-progress edits — `onDraftPersist` - // only commits content into the Map on pane teardown, which would leave - // autosave a full editing session behind. - let liveContent = $state<{ scriptPath: string | undefined; content: string }>({ - scriptPath: undefined, - content: '' - }) - - // Canonical "empty" overlay literals, reused both as reset values for the - // live-* state above and as the no-overlay inputs to the deployed graph. + // The live editor overlays (annotations / body assets / content for the open + // script) now live in `pe`. The canonical "empty" literals stay here — they + // also seed the no-overlay inputs to the deployed graph below. const EMPTY_LIVE_ASSETS = { scriptPath: undefined, assets: [] } const EMPTY_LIVE_ANNOTATIONS = { scriptPath: undefined, @@ -518,16 +236,6 @@ } } - // Reset every live editor overlay (annotations / body assets / content) - // back to empty, unconditionally. Used by the leave-edit path so a stale - // buffer for the previously-open script can't leak into the view graphs. - // (forgetPath resets these per-path instead — see there.) - function clearLiveOverlays() { - liveAnnotations = EMPTY_LIVE_ANNOTATIONS - liveBodyAssets = EMPTY_LIVE_ASSETS - liveContent = { scriptPath: undefined, content: '' } - } - // Only-add cache of (script_path → body content) populated lazily by // `bodyFetchEffect` for every script in the current folder. We never // remove entries: stale keys (renamed-away, deleted) are simply ignored @@ -559,7 +267,9 @@ const g = graphRes.current if (!g) return { writes, reads } const liveAssetsForPath = (path: string) => - liveBodyAssets.scriptPath === path ? liveBodyAssets.assets : inferredAssetsByPath.get(path) + pe.liveBodyAssets.scriptPath === path + ? pe.liveBodyAssets.assets + : inferredAssetsByPath.get(path) for (const r of g.runnables) { if (r.usage_kind !== 'script') continue const assets = liveAssetsForPath(r.path) @@ -580,12 +290,12 @@ const out = new Map>() const g = graphRes.current if (!g) return out - const livePath = liveAnnotations.scriptPath + const livePath = pe.liveAnnotations.scriptPath for (const r of g.runnables) { if (r.usage_kind !== 'script') continue let kinds: Set if (r.path === livePath) { - kinds = new Set(liveAnnotations.annotations.nativeTriggers.map((n) => n.kind)) + kinds = new Set(pe.liveAnnotations.annotations.nativeTriggers.map((n) => n.kind)) } else { const body = bodiesByPath.get(r.path) if (!body) continue @@ -663,13 +373,17 @@ const script = buildDraft(language, scriptPath, triggers, outputKind, out, input) // Write the new draft into the map (structural update so Svelte // re-derives graphWithDraft) and focus it in the details pane. When - // the user picked `none`, `outputAsset` is undefined and the graph - // overlay skips synthesizing a write edge. - const next = new Map(drafts) - next.set(scriptPath, { localId: newDraftLocalId(), script, outputAsset: out }) - drafts = next - activeDraftPath = scriptPath - selection = undefined + // the user picked `none`, `out` is undefined and the graph overlay + // skips synthesizing a write edge. + const next = new Map(pe.drafts) + next.set(scriptPath, { + localId: pe.newDraftLocalId(), + script, + outputAssets: out ? [out] : undefined + }) + pe.drafts = next + pe.activeDraftPath = scriptPath + pe.selection = undefined // Follow the new node with a smooth pan. The id matches the runnable // node the canvas builds for a draft script (`script:`). @@ -713,6 +427,43 @@ aiChatManager.sendRequest({ instructions }) } + // ===================== AI chat pipeline integration ===================== + // The global AI chat (dev-gated) gains pipeline-building tools while this + // editor is mounted, via the helpers registered below. AI mutations don't + // deploy — they apply directly as unsaved drafts on the canvas (the same way + // the flow/script editor applies AI edits), which the user then deploys. The + // build/edit logic is shared verbatim with the in-session preview + // (PipelineEditorView) via createPipelineAiHelpers. + + const pipelineAiHelpers = createPipelineAiHelpers({ + getFolder: () => folder, + getWorkspace: () => $workspaceStore, + getResolvedGraph: () => graphWithDraft, + getDrafts: () => pe.drafts, + setDrafts: (next) => (pe.drafts = next), + newDraftLocalId: pe.newDraftLocalId, + onForgetPath: (path) => forgetPath(path), + onShowDrafts: () => (includeDrafts = true), + onProposeNode: (path) => focusPipelineNode(`script:${path}`), + ensureEditable: () => { + // Auto-enter edit so AI changes are visible/actionable, unless the user + // is an operator (no edit permission) — then refuse with a clear error. + if (isOperator) { + throw new Error('This pipeline is read-only for your role; AI edits are disabled.') + } + if (mode !== 'edit') setMode('edit') + }, + onRunStarted: (jobId, path) => { + activeRunnables.arm(`script:${path}`) + runsPendingJobId = jobId + runsRefreshKey++ + activeRunnable = { kind: 'script', path } + activeRunnableJobId = jobId + } + }) + + onMount(() => aiChatManager.setPipelineHelpers(pipelineAiHelpers)) + // Navigation guard state. `pendingNavigationUrl` holds the URL the user // tried to leave to so we can complete the navigation after they pick // "Save all" or "Discard all"; `bypassNavigationGuard` is the standard @@ -728,7 +479,7 @@ bypassNavigationGuard = false return } - if (drafts.size === 0) return + if (pe.drafts.size === 0) return // `leave` covers tab close / hard reload / cross-origin nav. SvelteKit // turns a cancelled leave into a browser-native "Leave site?" prompt, // which we explicitly don't want — match the rest of the editors and @@ -762,8 +513,8 @@ // Wipe every draft and the active selection so saved drafts and // stale active path don't bleed into the next page. localStorage // is overwritten by the persist effect on the next tick. - drafts = new Map() - activeDraftPath = undefined + pe.drafts = new Map() + pe.activeDraftPath = undefined saveErrors = new Map() const target = pendingNavigationUrl leaveModalOpen = false @@ -782,7 +533,7 @@ // page so they can deal with the failures via the bar's error // popover. Otherwise resume the navigation that triggered the // guard. - if (drafts.size === 0) { + if (pe.drafts.size === 0) { const target = pendingNavigationUrl leaveModalOpen = false pendingNavigationUrl = undefined @@ -871,10 +622,10 @@ } async function saveAllDrafts() { - if (!$workspaceStore || drafts.size === 0 || savingAll) return + if (!$workspaceStore || pe.drafts.size === 0 || savingAll) return savingAll = true const ws = $workspaceStore - const entries = [...drafts.entries()] + const entries = [...pe.drafts.entries()] // Snapshot what the preview promises for every draft before anything // deploys — used to verify the persisted graph below. const predicted = predictCascadeFacts(entries.map(([p]) => p)) @@ -905,22 +656,22 @@ // entries to keep insertion order stable. if (savedPaths.length > 0) { const next = new Map() - for (const [k, v] of drafts) { + for (const [k, v] of pe.drafts) { if (!savedPaths.includes(k)) next.set(k, v) } - drafts = next + pe.drafts = next // If the open draft just got deployed, transfer the focus to // its now-persisted runnable so the pane stays on the same // script the user was editing — otherwise the pane closes, // the canvas re-fits, and the user has to re-find their // script after every save. - if (activeDraftPath && savedPaths.includes(activeDraftPath)) { - selection = { + if (pe.activeDraftPath && savedPaths.includes(pe.activeDraftPath)) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', - path: activeDraftPath + path: pe.activeDraftPath } - activeDraftPath = undefined + pe.activeDraftPath = undefined } await graphRes.refetch() // Verify only what actually deployed — failed drafts would @@ -939,10 +690,10 @@ } function discardDraft(path: string) { - if (!drafts.has(path)) return - const next = new Map(drafts) + if (!pe.drafts.has(path)) return + const next = new Map(pe.drafts) next.delete(path) - drafts = next + pe.drafts = next forgetPath(path) } @@ -961,18 +712,18 @@ // selection + per-path save errors. `bodiesByPath` keeps its entry // (only-add cache, harmless if stale). function forgetPath(path: string) { - if (activeDraftPath === path) activeDraftPath = undefined - if (selection?.kind === 'runnable' && selection.path === path) { - selection = undefined + if (pe.activeDraftPath === path) pe.activeDraftPath = undefined + if (pe.selection?.kind === 'runnable' && pe.selection.path === path) { + pe.selection = undefined } - if (liveAnnotations.scriptPath === path) { - liveAnnotations = EMPTY_LIVE_ANNOTATIONS + if (pe.liveAnnotations.scriptPath === path) { + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS } - if (liveBodyAssets.scriptPath === path) { - liveBodyAssets = EMPTY_LIVE_ASSETS + if (pe.liveBodyAssets.scriptPath === path) { + pe.liveBodyAssets = EMPTY_LIVE_ASSETS } - if (liveContent.scriptPath === path) { - liveContent = { scriptPath: undefined, content: '' } + if (pe.liveContent.scriptPath === path) { + pe.liveContent = { scriptPath: undefined, content: '' } } clearSaveError(path) } @@ -983,13 +734,13 @@ // open and surface the conflict inline. function renameDraft(oldPath: string, newPath: string): boolean | string { if (oldPath === newPath) return true - const draft = drafts.get(oldPath) + const draft = pe.drafts.get(oldPath) if (!draft) return 'Draft not found' - if (drafts.has(newPath)) return 'Another draft already uses this path' + if (pe.drafts.has(newPath)) return 'Another draft already uses this path' const next = new Map() // Preserve insertion order: replace the entry at its original // position so the canvas / lists don't reshuffle on rename. - for (const [k, v] of drafts) { + for (const [k, v] of pe.drafts) { if (k === oldPath) { const updatedScript = { ...v.script, path: newPath } next.set(newPath, { ...v, script: updatedScript }) @@ -997,8 +748,8 @@ next.set(k, v) } } - drafts = next - if (activeDraftPath === oldPath) activeDraftPath = newPath + pe.drafts = next + if (pe.activeDraftPath === oldPath) pe.activeDraftPath = newPath // Path-keyed live overlays: re-key for the renamed draft so the // graph stays consistent between the moment we mutate `drafts` // here and the next editor event that re-emits annotations / @@ -1007,14 +758,14 @@ // re-applying live overlays against the same OLD path — leaving // phantom edges that displace the + node off the top of the // graph and shuffle the layout. - if (liveAnnotations.scriptPath === oldPath) { - liveAnnotations = { ...liveAnnotations, scriptPath: newPath } + if (pe.liveAnnotations.scriptPath === oldPath) { + pe.liveAnnotations = { ...pe.liveAnnotations, scriptPath: newPath } } - if (liveBodyAssets.scriptPath === oldPath) { - liveBodyAssets = { ...liveBodyAssets, scriptPath: newPath } + if (pe.liveBodyAssets.scriptPath === oldPath) { + pe.liveBodyAssets = { ...pe.liveBodyAssets, scriptPath: newPath } } - if (liveContent.scriptPath === oldPath) { - liveContent = { ...liveContent, scriptPath: newPath } + if (pe.liveContent.scriptPath === oldPath) { + pe.liveContent = { ...pe.liveContent, scriptPath: newPath } } // `inferredWritesByPath` / `inferredReadsByPath` / // `annotatedNativeKindsByPath` are derived from `g.runnables` × @@ -1055,7 +806,7 @@ >() function deployRenamedDraft(path: string) { - const draft = drafts.get(path) + const draft = pe.drafts.get(path) if (!draft) return const localId = draft.localId let state = deployQueue.get(localId) @@ -1078,7 +829,7 @@ try { while (true) { if (!$workspaceStore) break - const draft = drafts.get(path) + const draft = pe.drafts.get(path) if (!draft) break try { await saveDraft(path, draft, $workspaceStore) @@ -1111,12 +862,12 @@ // queued path is waiting, the next loop iteration will // pick it up and we keep the draft live. if (!state.queuedPath) { - const nextDrafts = new Map(drafts) + const nextDrafts = new Map(pe.drafts) nextDrafts.delete(path) - drafts = nextDrafts - if (activeDraftPath === path) { - selection = { kind: 'runnable', runnable_kind: 'script', path } - activeDraftPath = undefined + pe.drafts = nextDrafts + if (pe.activeDraftPath === path) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', path } + pe.activeDraftPath = undefined } if (saveErrors.has(path)) { const nextErrors = new Map(saveErrors) @@ -1142,22 +893,22 @@ state.inflight = false // On the rare path where the draft is also gone (deployed + // no queued path), drop the slot to keep the map bounded. - if (!drafts.has(path) && !state.queuedPath) { + if (!pe.drafts.has(path) && !state.queuedPath) { deployQueue.delete(localId) } } } // Currently-open draft shape (if any) — fed into the details pane. - let activeDraft = $derived(activeDraftPath ? drafts.get(activeDraftPath) : undefined) + let activeDraft = $derived(pe.activeDraftPath ? pe.drafts.get(pe.activeDraftPath) : undefined) // Path of the script currently open in the details pane (draft or // persisted selection), used wherever run-routing / overlay logic needs // "the one script the user is editing right now". let openScriptPath = $derived( - activeDraftPath ?? - (selection?.kind === 'runnable' && selection.runnable_kind === 'script' - ? selection.path + pe.activeDraftPath ?? + (pe.selection?.kind === 'runnable' && pe.selection.runnable_kind === 'script' + ? pe.selection.path : undefined) ) @@ -1168,12 +919,12 @@ $effect(() => { if (mode !== 'edit') return if ( - selection?.kind === 'runnable' && - selection.runnable_kind === 'script' && - drafts.has(selection.path) + pe.selection?.kind === 'runnable' && + pe.selection.runnable_kind === 'script' && + pe.drafts.has(pe.selection.path) ) { - activeDraftPath = selection.path - selection = undefined + pe.activeDraftPath = pe.selection.path + pe.selection = undefined } }) // Symmetric demotion: view mode shows the deployed truth, so an open @@ -1185,95 +936,13 @@ if (mode !== 'view') return const d = activeDraft if (d && d.script.hash) { - selection = { kind: 'runnable', runnable_kind: 'script', path: d.script.path } - activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: d.script.path } + pe.activeDraftPath = undefined } }) - // Named handlers for the details pane's live callbacks. Inline arrows - // would be rebuilt on every parent re-render, and the pane's $effects - // track those refs as deps — combined with the drafts mutation in - // `handleDraftContentChange`, that creates a parent ↔ child feedback - // loop ("effect_update_depth_exceeded"). Named functions keep the - // prop reference stable so the $effects only re-fire on real - // content changes (e.g. handleContentChange below mutating drafts). - function handleAnnotationsChange( - scriptPath: string | undefined, - annotations: PipelineAnnotations - ) { - liveAnnotations = { scriptPath, annotations } - } - function handleAssetsChange( - scriptPath: string | undefined, - assets: AssetWithAltAccessType[], - columnLineage?: ColumnLineage[] - ) { - // Single update site for the live overlay. `inferredWritesByPath` - // / `inferredReadsByPath` are now derived from `liveBodyAssets` - // (for the open script) + `inferredAssetsByPath` (prefetched - // snapshot for every other script), so we don't have to write - // into those caches here — the derive picks up our update on the - // next reactive tick. - liveBodyAssets = { scriptPath, assets, columnLineage } - } - function handleContentChange(scriptPath: string | undefined, content: string) { - liveContent = { scriptPath, content } - } - function handleDraftPersist( - p: string, - snapshot: { - content: string - writes: { kind: AssetKind; path: string }[] - script?: Script - } - ) { - // Persist body edits + inferred outputs back into the drafts Map so - // they survive switching to another node and back (the details pane - // clones draftScript locally on every prop change, and `outputAsset` - // would otherwise stay frozen at the value seeded when the draft - // was opened — leaving a stale write edge on the canvas after the - // user has renamed a CREATE TABLE / writeS3File target). - // - // Deferred a microtask: this is called from the pane's $effect - // teardown, which observes the *previous* batch values — after a - // discard, `drafts` still appears to contain the discarded entry, - // and writing a map cloned from that stale read would resurrect it - // (the "discard needs two clicks" bug). One microtask later the - // batch has committed and the reads are fresh. - queueMicrotask(() => { - const d = drafts.get(p) - if (!d) { - // Unsaved edits to a *deployed* script (the pane only emits - // these when the buffer differs from the deployed content): - // promote to a draft so the work survives mode switches / - // selection changes, shows in the drafts chip, and deploys - // via Save all. The script snapshot carries the deployed - // hash, so saving chains a new version off it. - if (!snapshot.script) return - const next = new Map(drafts) - next.set(p, { - localId: newDraftLocalId(), - script: snapshot.script, - outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined - }) - drafts = next - return - } - const writesEqual = - d.outputAssets?.length === snapshot.writes.length && - (d.outputAssets ?? []).every( - (a, i) => a.kind === snapshot.writes[i]?.kind && a.path === snapshot.writes[i]?.path - ) - if (d.script.content === snapshot.content && writesEqual) return - const next = new Map(drafts) - next.set(p, { - ...d, - script: { ...d.script, content: snapshot.content }, - outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined - }) - drafts = next - }) - } + // The details pane's live callbacks (annotations / assets / content / persist) + // now live on `pe` as stable arrow fields — pass `pe.handleX` straight through. // Canvas callbacks, named so the prop refs stay stable across re-renders // (same rationale as the live-callback handlers above) and so the @@ -1297,13 +966,13 @@ s && s.kind === 'runnable' && s.runnable_kind === 'script' && - drafts.has(s.path) + pe.drafts.has(s.path) ) { - activeDraftPath = s.path - selection = undefined + pe.activeDraftPath = s.path + pe.selection = undefined } else { - activeDraftPath = undefined - selection = s + pe.activeDraftPath = undefined + pe.selection = s } } function handleAddScriptForAsset( @@ -1349,8 +1018,8 @@ return } if (info.runnable_kind !== 'script') return - activeDraftPath = undefined - selection = { kind: 'runnable', runnable_kind: 'script', path: info.path } + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: info.path } requestRemoveSignal++ } async function handleRunProducer(producer: { @@ -1386,7 +1055,7 @@ if (cascade) { const hasDraftInChain = producer.unsaved === true || - computeDownstreamClosure(graphWithDraft, producer.path).nodes.some((p) => drafts.has(p)) + computeDownstreamClosure(graphWithDraft, producer.path).nodes.some((p) => pe.drafts.has(p)) if (hasDraftInChain) { return await runDraftAwareCascade(producer.path) } @@ -1403,7 +1072,7 @@ const skipArg = cascade ? {} : { _wmill_skip_asset_dispatch: true } let jobId: string | undefined if (producer.unsaved) { - const draft = drafts.get(producer.path) + const draft = pe.drafts.get(producer.path) if (!draft?.script.content || !draft.script.language) return undefined jobId = await JobService.runScriptPreview({ workspace: $workspaceStore, @@ -1483,9 +1152,9 @@ let graphWithDraft = $derived.by(() => resolveGraph({ base: graphRes.current ?? EMPTY_GRAPH, - drafts, - liveBodyAssets, - liveAnnotations, + drafts: pe.drafts, + liveBodyAssets: pe.liveBodyAssets, + liveAnnotations: pe.liveAnnotations, inferredWritesByPath, inferredReadsByPath, annotatedNativeKindsByPath @@ -1523,11 +1192,11 @@ // Guard before reassigning so an already-empty overlay doesn't // needlessly invalidate the graph derives every mode toggle. if ( - liveAnnotations.scriptPath != undefined || - liveBodyAssets.scriptPath != undefined || - liveContent.scriptPath != undefined + pe.liveAnnotations.scriptPath != undefined || + pe.liveBodyAssets.scriptPath != undefined || + pe.liveContent.scriptPath != undefined ) { - clearLiveOverlays() + pe.clearLiveOverlays() } }) }) @@ -1536,20 +1205,11 @@ // preserved, so toggling back to edit keeps them). Tracks activeDraftPath // too, covering the onMount localStorage restore landing after a view load. $effect(() => { - if (mode === 'view' && !includeDrafts && activeDraftPath != undefined) { - activeDraftPath = undefined + if (mode === 'view' && !includeDrafts && pe.activeDraftPath != undefined) { + pe.activeDraftPath = undefined } }) - // Selection highlights the active draft (if any) or the user's picked - // node. Non-active drafts render without selection highlight but are - // still clickable to re-enter their edit pane. - let effectiveSelection = $derived( - activeDraftPath - ? { kind: 'runnable', runnable_kind: 'script', path: activeDraftPath } - : selection - ) - // Bumped after every successful run dispatch so AssetRunsPanel re-fetches // the listing immediately — the new (preview or script) job appears in // the history popover without waiting on its 3 s poll tick. @@ -1696,7 +1356,7 @@ // drafts (same condition as `displayGraph`). Otherwise — View mode with // drafts hidden — a bounded run must execute the *deployed* scripts the // user is looking at, not preview jobs from hidden local drafts. - const draft = mode === 'edit' || includeDrafts ? drafts.get(path) : undefined + const draft = mode === 'edit' || includeDrafts ? pe.drafts.get(path) : undefined if (draft) { if (!draft.script.content || !draft.script.language) { throw new Error(`draft ${path} has no content/language`) @@ -1961,7 +1621,7 @@ // running a draft via runScriptPreview creates a `preview`-kind job at // the same path, which the panel's listing query picks up. let selectionProducers = $derived.by(() => { - const sel = selection + const sel = pe.selection if (!sel || sel.kind !== 'asset') return [] return graphWithDraft.edges .filter((e) => { @@ -1991,7 +1651,7 @@ // ducklake-asset selection so it isn't rebuilt on every editor keystroke when // the trace UI isn't even shown. let columnGraph = $derived( - selection?.kind === 'asset' && selection.asset_kind === 'ducklake' + pe.selection?.kind === 'asset' && pe.selection.asset_kind === 'ducklake' ? buildColumnGraph(displayGraph) : EMPTY_COLUMN_GRAPH ) @@ -2008,7 +1668,7 @@ // without it) is treated as unknown → evolvable, so captured history is never // hidden behind a stale "fixed" verdict. let schemaCanEvolve = $derived.by(() => { - const sel = selection + const sel = pe.selection if (!sel || sel.kind !== 'asset' || sel.asset_kind !== 'ducklake') return true const producerPaths = new Set(selectionProducers.map((p) => p.path)) const producers = graphWithDraft.runnables.filter((r) => producerPaths.has(r.path)) @@ -2068,7 +1728,7 @@ // flow. Drafts have no deployed endpoint yet, so nudge the user to save // first (mirrors openMissingTriggerDrawer). function openWebhookDrawer(scriptPath: string) { - if (drafts.has(scriptPath)) { + if (pe.drafts.has(scriptPath)) { sendUserToast( `Save the script "${scriptPath}" first — webhooks only trigger the deployed version.`, true @@ -2090,12 +1750,12 @@ // Same mode gate as handleCanvasSelect: view mode targets the // deployed script (its run form runs the deployed version), even // when unsaved edits were promoted to a draft. - if (mode === 'edit' && drafts.has(scriptPath)) { - activeDraftPath = scriptPath - selection = undefined + if (mode === 'edit' && pe.drafts.has(scriptPath)) { + pe.activeDraftPath = scriptPath + pe.selection = undefined } else { - activeDraftPath = undefined - selection = { kind: 'runnable', runnable_kind: 'script', path: scriptPath } + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: scriptPath } } // Bump after a tick so the pane has reacted to the new selection/draft // and begun mounting the run form before it hunts for the S3 input. @@ -2110,7 +1770,7 @@ // time or silently bind to nothing. Surface that as a toast and // keep the drawer closed; the user needs to save the script first // (which also creates it under the new path if they renamed it). - if (drafts.has(scriptPath)) { + if (pe.drafts.has(scriptPath)) { sendUserToast( `Save the script "${scriptPath}" first — triggers can only be attached to deployed scripts.`, true @@ -2239,7 +1899,7 @@ g.runnables .filter((r) => r.usage_kind === 'script') .map((r) => r.path) - .filter((p) => !drafts.has(p) && !bodiesByPath.has(p)) + .filter((p) => !pe.drafts.has(p) && !bodiesByPath.has(p)) ) if (targets.length === 0) return let i = 0 @@ -2351,8 +2011,8 @@ anchored between the two flex-1 side groups so it stays centered, with breathing room on both sides. -->
- setMode(m)} /> - {#if mode === 'view' && drafts.size > 0} + setMode(m)} /> + {#if mode === 'view' && pe.drafts.size > 0} {/if}
@@ -2410,7 +2070,7 @@ {/snippet} {/if} - {#if mode === 'edit' && drafts.size > 0} + {#if mode === 'edit' && pe.drafts.size > 0} @@ -2420,7 +2080,7 @@ itemKind="data_pipeline" path={pipelineDraftPath} draftOnly - loadedFromDraft={loadedFromDbDraft} + loadedFromDraft={pe.loadedFromDbDraft} /> {/if} {/if} {#if mode === 'view'} @@ -2470,257 +2130,152 @@ Failed to load pipeline: {graphRes.error.message} {:else} - - -
- - {#if boundPick} - -
- -
- - {boundPickEnds.size === 0 - ? 'Click end node(s) to bound the run' - : `${boundScripts.length} script${boundScripts.length === 1 ? '' : 's'} up to ${boundPickEnds.size} end${boundPickEnds.size === 1 ? '' : 's'}`} - - - from {boundPickStart ? shortPath(boundPickStart) : ''} - -
- - + (panelHidden = !panelHidden)} + {prefetchingAssets} + hoveredPaths={activityHoverPaths} + selectedRunPaths={activitySelectPaths} + {activeRunnable} + activeRunnableIds={activeRunnables.ids} + runStates={mergedRunStates} + eventLogEvents={activeRunnables.events} + {runsRefreshKey} + {runsPendingJobId} + {boundPick} + validStartPaths={isOperator ? undefined : validStartPaths} + onStartBoundedRun={isOperator ? undefined : startBoundedRun} + onPickEnd={pickBoundEnd} + {panToNodeId} + onCreateMissingTrigger={mode === 'edit' ? openMissingTriggerDrawer : undefined} + onEditTrigger={mode === 'edit' ? openEditTriggerDrawer : undefined} + onDeleteTrigger={mode === 'edit' ? deleteAttachedTrigger : undefined} + onOpenWebhook={openWebhookDrawer} + onOpenDataUpload={openDataUploadRun} + onSelect={handleCanvasSelect} + onAddScriptForAsset={mode === 'edit' ? handleAddScriptForAsset : undefined} + onAddPipelineScript={mode === 'edit' ? handleAddPipelineScript : undefined} + onRunnableMenuRemove={mode === 'edit' ? handleRunnableMenuRemove : undefined} + onRunProducer={mode === 'edit' ? handleRunProducer : undefined} + onRequestEdit={isOperator ? undefined : () => setMode('edit')} + canRunByPath={openScriptHasDataUpload} + onRunByPath={runByPathLegit} + {selectionProducers} + selectionColumnGraph={pe.activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} + {schemaCanEvolve} + downstreamSubscribers={editedScriptDownstreamCount} + onStartBoundedRunForOpen={startBoundedRun} + canBoundedRunOpenScript={!!openScriptPath && + validStartPaths.has(openScriptPath) && + lineageDownstreamPaths.has(openScriptPath)} + onRunCompleted={() => { + activeRunnable = undefined + activeRunnableJobId = undefined + }} + onTestStateChange={(running) => { + const openPath = openScriptPath + if (running && openPath) { + activeRunnable = { kind: 'script', path: openPath } + activeRunnables.arm(`script:${openPath}`) + activeRunnableJobId = undefined + } else if (!running && activeRunnable?.path === openPath) { + activeRunnable = undefined + activeRunnableJobId = undefined + } + }} + {requestRemoveSignal} + {requestRunSignal} + {requestRunCascadeSignal} + focusUploadSignal={focusDataUploadSignal} + onDraftPathChange={renameDraft} + onClose={() => { + pe.selection = undefined + pe.activeDraftPath = undefined + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS + }} + onDiscard={() => { + if (pe.activeDraftPath) discardDraft(pe.activeDraftPath) + }} + onDraftSaved={async (savedPath) => { + const predicted = predictCascadeFacts([savedPath]) + const nextDrafts = new Map(pe.drafts) + nextDrafts.delete(savedPath) + pe.drafts = nextDrafts + if (pe.activeDraftPath === savedPath) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: savedPath } + pe.activeDraftPath = undefined + } + clearSaveError(savedPath) + await graphRes.refetch() + reportDeployDrift(predicted) + }} + onPersistedSaved={async (savedPath) => { + const predicted = predictCascadeFacts([savedPath]) + await graphRes.refetch() + reportDeployDrift(predicted) + }} + onScriptRenamed={async (oldPath, newPath) => { + if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) { + pe.selection = { ...pe.selection, path: newPath } + } + await graphRes.refetch() + }} + onScriptRemoved={async (removedPath) => { + forgetPath(removedPath) + await graphRes.refetch() + }} + > + {#snippet boundBar()} + {#if boundPick} +
+ +
+ + {boundPickEnds.size === 0 + ? 'Click end node(s) to bound the run' + : `${boundScripts.length} script${boundScripts.length === 1 ? '' : 's'} up to ${boundPickEnds.size} end${boundPickEnds.size === 1 ? '' : 's'}`} + + + from {boundPickStart ? shortPath(boundPickStart) : ''} +
- {/if} - {#if mode === 'edit'} - - - {/if} - {#if prefetchingAssets} -
Cancel +
- {/if} - {#if mode !== 'edit' || selection != undefined || activeDraftPath != undefined} - -
-
- {/if} -
- {#if detailsPaneOpen && $workspaceStore} - - {#if mode !== 'edit' && selection == undefined && activeDraftPath == undefined} - - (activityHoverPaths = p ?? [])} - onSelectRun={(p) => (activitySelectPaths = p ?? [])} - /> - {:else} - setMode('edit')} - canRunByPath={openScriptHasDataUpload} - onRunByPath={runByPathLegit} - selection={activeDraft ? undefined : selection} - selectionProducers={activeDraft ? [] : selectionProducers} - selectionColumnGraph={activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} - {schemaCanEvolve} - {runsRefreshKey} - {runsPendingJobId} - {activeRunnable} - downstreamSubscribers={editedScriptDownstreamCount} - onStartBoundedRun={openScriptPath && - validStartPaths.has(openScriptPath) && - lineageDownstreamPaths.has(openScriptPath) - ? () => startBoundedRun(openScriptPath!) - : undefined} - onRunCompleted={() => { - activeRunnable = undefined - activeRunnableJobId = undefined - }} - onTestStateChange={(running) => { - // Bridge: ScriptEditor's Test button triggers the - // same canvas-level "is running" hint as the - // per-node Run button. The currently-edited script - // is whichever path is open in the pane (active - // draft, or the persisted-script selection). - const openPath = openScriptPath - if (running && openPath) { - activeRunnable = { kind: 'script', path: openPath } - // Mark the tested runnable as launched-from-here so the - // folder poll's catch-up pulse won't re-flash its edge a - // poll-interval after a fast job already finished (the - // edge is animated zero-latency by `activeRunnable`, and - // the test loader clears that the instant it completes). - // Also upgrades to the fast poll so the badge lands sooner. - activeRunnables.arm(`script:${openPath}`) - // Editor Test path clears via its own callbacks, not - // the job-id effect — drop any stale tracked id so a - // prior canvas run's completion can't clear this hint. - activeRunnableJobId = undefined - } else if (!running && activeRunnable?.path === openPath) { - activeRunnable = undefined - activeRunnableJobId = undefined - } - }} - {requestRemoveSignal} - {requestRunSignal} - {requestRunCascadeSignal} - focusUploadSignal={focusDataUploadSignal} - draftScript={activeDraft?.script} - {pathPrefix} - onDraftPathChange={renameDraft} - workspace={$workspaceStore} - onAnnotationsChange={handleAnnotationsChange} - onAssetsChange={handleAssetsChange} - onContentChange={handleContentChange} - onDraftPersist={handleDraftPersist} - onclose={() => { - // Close dismisses the pane but preserves drafts so - // the user can come back to them. Discarding is - // via the explicit "Discard" button in the pane. - selection = undefined - activeDraftPath = undefined - liveAnnotations = EMPTY_LIVE_ANNOTATIONS - }} - onHide={() => (panelHidden = true)} - onDiscard={() => { - if (activeDraftPath) discardDraft(activeDraftPath) - }} - onDraftSaved={async (savedPath) => { - // Snapshot the preview's promise while the draft - // overlay still exists (dropped from `drafts` below). - const predicted = predictCascadeFacts([savedPath]) - // Drop the now-deployed draft and hand focus to its - // persisted runnable so the pane stays open on the - // same script. `discardDraft` would clear - // activeDraftPath without setting selection — the - // canvas would deselect and the view reset on the - // next refetch. - const nextDrafts = new Map(drafts) - nextDrafts.delete(savedPath) - drafts = nextDrafts - if (activeDraftPath === savedPath) { - selection = { - kind: 'runnable', - runnable_kind: 'script', - path: savedPath - } - activeDraftPath = undefined - } - clearSaveError(savedPath) - await graphRes.refetch() - reportDeployDrift(predicted) - }} - onPersistedSaved={async (savedPath) => { - // Snapshot before the refetch replaces the base graph - // — the live editor overlay is the prediction here. - const predicted = predictCascadeFacts([savedPath]) - // Refresh the asset graph so the rows the deploy - // just inserted (from the body-asset write list we - // pass at save time) make it into base.edges. The - // in-memory `inferredWritesByPath` overlay - // dedupes against base, so the edge stays put - // instead of flickering when the ScriptEditor - // remounts on the new hash. - await graphRes.refetch() - reportDeployDrift(predicted) - }} - onScriptRenamed={async (oldPath, newPath) => { - // Repoint the selection at the new path before the - // graph refetches so the pane stays focused on the - // same script. Order matters: update selection - // first, then refetch — otherwise the resource - // driving the pane would briefly resolve to nothing. - if (selection?.kind === 'runnable' && selection.path === oldPath) { - selection = { ...selection, path: newPath } - } - await graphRes.refetch() - }} - onScriptRemoved={async (removedPath) => { - // Drop every path-keyed overlay / cache entry - // pointing at the now-archived runnable so - // resolveGraph doesn't keep emitting lineage - // edges or missing-trigger placeholders against - // a script that no longer exists. Without this - // the inferred writes / annotation maps would - // keep dragging phantom nodes onto the canvas - // until the next folder change. - forgetPath(removedPath) - await graphRes.refetch() - }} - /> - {/if} - - {/if} - + Run selection + +
+ {/if} + {/snippet} + {#snippet idlePane()} + (activityHoverPaths = p ?? [])} + onSelectRun={(p) => (activitySelectPaths = p ?? [])} + /> + {/snippet} + {/if}
@@ -2761,18 +2316,18 @@

- {drafts.size === 1 ? 'Unsaved draft' : `${drafts.size} unsaved drafts`} + {pe.drafts.size === 1 ? 'Unsaved draft' : `${pe.drafts.size} unsaved drafts`}

- You have {drafts.size === 1 + You have {pe.drafts.size === 1 ? 'a draft pipeline script' - : `${drafts.size} draft pipeline scripts`} that {drafts.size === 1 + : `${pe.drafts.size} draft pipeline scripts`} that {pe.drafts.size === 1 ? 'has' : 'have'} not been deployed yet. What would you like to do?

    - {#each [...drafts.keys()] as p} + {#each [...pe.drafts.keys()] as p}
  • {p}
    • {/each}
@@ -2787,7 +2342,9 @@ unifiedSize="sm" startIcon={{ icon: leaveSaving ? Loader2 : Save }} > - {leaveSaving ? 'Saving…' : `Save all (${drafts.size})`} + {leaveSaving ? 'Saving…' : `Save all (${pe.drafts.size})`}
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. + +\`// materialize \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- \`// materialize append\` → INSERT-append rows (incremental); +- \`// materialize key=\` → merge/upsert on \`\`. + +\`// materialize manual \` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. + +\`materialize\` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. +2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`), language chosen for the work: \`duckdb\` or \`postgresql\` for SQL-shaped data work, \`bun\`/\`python3\` for general transforms. SQL-heavy lakehouse steps usually use \`duckdb\`. +3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the \`/pipeline/\` editor open, prefer the dedicated \`build_pipeline_node\` / \`edit_pipeline_node\` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node \`f/sales/orders_ingest\` (runs on a schedule, materializes a DuckLake table): + +\`\`\`sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +\`\`\` + +Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup): + +\`\`\`sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +\`\`\` +`; + export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide ## Scope diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md new file mode 100644 index 0000000000..a0f2e2492a --- /dev/null +++ b/system_prompts/base/pipeline-base.md @@ -0,0 +1,60 @@ +# Data pipeline authoring + +A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. + +## What makes a script a pipeline node + +A script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** — `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: + +- `// on ` — declares an execution-DAG **input** (what triggers/feeds this node). `` is either: + - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `s3://`, `$res:f/folder/my_resource`, `volume://name/path`. + - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. +- **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. + +## Materialize (the managed output) + +> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. + +`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- `// materialize append` → INSERT-append rows (incremental); +- `// materialize key=` → merge/upsert on ``. + +`// materialize manual ` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time. + +`materialize` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: `f//`. The folder is the pipeline. +2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`. +3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table): + +```sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +``` + +Node `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup): + +```sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 6ee1971f43..98c03c4367 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -2369,6 +2369,7 @@ def main(): flow_base = read_markdown_file(base_dir / "flow-base.md") resources_base = read_markdown_file(base_dir / "resources.md") raw_app_base = read_markdown_file(base_dir / "raw-app.md") + pipeline_base = read_markdown_file(base_dir / "pipeline-base.md") workflow_as_code_base = read_markdown_file(base_dir / "workflow-as-code.md") flow_cli = read_markdown_file(base_dir / "flow-cli.md") flow_chat_special_modules = read_markdown_file(base_dir / "flow-chat-special-modules.md") @@ -2435,6 +2436,7 @@ def main(): 'FLOW_BASE': flow_base, 'RESOURCES_BASE': resources_base, 'RAW_APP_BASE': raw_app_base, + 'PIPELINE_BASE': pipeline_base, 'WORKFLOW_AS_CODE_BASE': workflow_as_code_base, 'FLOW_CHAT_SPECIAL_MODULES': flow_chat_special_modules, @@ -2534,6 +2536,11 @@ export function getRawAppPrompt(): string { return prompts.RAW_APP_BASE; } +// Helper for data pipeline authoring (chat consumers) +export function getPipelinePrompt(): string { + return prompts.PIPELINE_BASE; +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { @@ -2586,6 +2593,7 @@ export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(): string; +export declare function getPipelinePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """ From 9b65161c643bf3f120d2ebd82f786c17233a971b Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 29 Jun 2026 22:37:39 +0200 Subject: [PATCH 124/273] fix(gcp): require token verification for authenticated push delivery (#9834) * fix(gcp): require token verification for authenticated push delivery Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 38e87caeca6a1dce9e4f3fa029ac36dffb30f1b2 This commit updates the EE repository reference after PR #636 was merged in windmill-ee-private. Previous ee-repo-ref: 8c63d487c486002baf09c77ab937fd77a91765eb New ee-repo-ref: 38e87caeca6a1dce9e4f3fa029ac36dffb30f1b2 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ce3db8ebcb..4dd9981afc 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -95352c13c4c82247d8cfd80936f9203aeb079802 +38e87caeca6a1dce9e4f3fa029ac36dffb30f1b2 From a9ffdb996b418ad51a84973ff01dcd3f2c820f16 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 29 Jun 2026 22:41:38 +0200 Subject: [PATCH 125/273] chore(main): release 1.743.0 (#9837) * chore(main): release 1.743.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 13 ++ backend/Cargo.lock | 172 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 139 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 361b6fbd91..77316bdff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.743.0](https://github.com/windmill-labs/windmill/compare/v1.742.0...v1.743.0) (2026-06-29) + + +### Features + +* **home:** redesign create-new popover and home header ([#9827](https://github.com/windmill-labs/windmill/issues/9827)) ([2493eaf](https://github.com/windmill-labs/windmill/commit/2493eaf031f30072637a297398674e761f039005)) +* **pipeline:** AI-chat data-pipeline editor (route + in-session) + home surfacing ([#9805](https://github.com/windmill-labs/windmill/issues/9805)) ([c910278](https://github.com/windmill-labs/windmill/commit/c91027824be1f1f49cdd14148baf6aad092a1dd0)) + + +### Bug Fixes + +* **gcp:** require token verification for authenticated push delivery ([#9834](https://github.com/windmill-labs/windmill/issues/9834)) ([9b65161](https://github.com/windmill-labs/windmill/commit/9b65161c643bf3f120d2ebd82f786c17233a971b)) + ## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7072389c21..c154084c51 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -2601,9 +2601,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.89+curl-8.20.0" +version = "0.4.90+curl-8.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22" +checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" dependencies = [ "cc", "libc", @@ -6085,9 +6085,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -12922,9 +12922,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" dependencies = [ "serde", "stable_deref_trait", @@ -13734,7 +13734,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-nats", @@ -13816,7 +13816,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.742.0" +version = "1.743.0" dependencies = [ "async-stream", "async-trait", @@ -13849,7 +13849,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13862,7 +13862,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "argon2", @@ -14000,7 +14000,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14023,7 +14023,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14038,7 +14038,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14064,7 +14064,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.742.0" +version = "1.743.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14074,7 +14074,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14091,7 +14091,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14113,7 +14113,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14136,7 +14136,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14152,7 +14152,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14173,7 +14173,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14194,7 +14194,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14208,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-nats", @@ -14243,7 +14243,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14268,7 +14268,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14286,7 +14286,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14308,7 +14308,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14328,7 +14328,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14365,7 +14365,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14393,7 +14393,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.742.0" +version = "1.743.0" dependencies = [ "lazy_static", "serde", @@ -14405,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.742.0" +version = "1.743.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14430,7 +14430,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14444,7 +14444,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14477,7 +14477,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.742.0" +version = "1.743.0" dependencies = [ "chrono", "lazy_static", @@ -14491,7 +14491,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14510,7 +14510,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.743.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14612,7 +14612,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.742.0" +version = "1.743.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14631,7 +14631,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.742.0" +version = "1.743.0" dependencies = [ "regex", "serde", @@ -14646,7 +14646,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14670,7 +14670,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "futures", @@ -14687,7 +14687,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.743.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14703,7 +14703,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -14724,7 +14724,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -14755,7 +14755,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "arc-swap", @@ -14780,7 +14780,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-stream", @@ -14814,7 +14814,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "futures", @@ -14832,7 +14832,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.743.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14841,7 +14841,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -14853,7 +14853,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -14865,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "gosyn", @@ -14877,7 +14877,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -14889,7 +14889,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -14901,7 +14901,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "nu-parser", @@ -14912,7 +14912,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14923,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14935,7 +14935,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14946,7 +14946,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -14968,7 +14968,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -14980,7 +14980,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -14994,7 +14994,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15011,7 +15011,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -15024,7 +15024,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -15036,7 +15036,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -15054,7 +15054,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15070,7 +15070,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15086,7 +15086,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "const_format", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.742.0" +version = "1.743.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15186,7 +15186,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -15220,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15244,7 +15244,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15277,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15310,7 +15310,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15330,7 +15330,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15364,7 +15364,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15423,7 +15423,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15447,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-nats", @@ -15471,7 +15471,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15506,7 +15506,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15534,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15559,7 +15559,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15578,7 +15578,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-once-cell", @@ -15688,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.742.0" +version = "1.743.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 45682215cf..725f460df1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.742.0" +version = "1.743.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.742.0" +version = "1.743.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 096d93163b..31e3ee26b6 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.743.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.743.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.743.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index e2afd2483b..62faf03ec3 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.742.0" +version = "1.743.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f9a15ef72d..18b1914826 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.742.0 + version: 1.743.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f8cc748bdb..38995ac0e9 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.742.0"; +export const VERSION = "v1.743.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 42d8109c49..7be2d4df35 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.742.0"; +export const VERSION = "1.743.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4a3fcf0696..0d66dfed67 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.742.0", + "version": "1.743.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.742.0", + "version": "1.743.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index e9ec640b02..0c122ffac7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.742.0", + "version": "1.743.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 7f9d00903a..1dc2caa039 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.742.0" +wmill = ">=1.743.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index f4342930e9..b2b394c836 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.742.0 + version: 1.743.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index e5b553e6f3..500f453c30 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.742.0' + ModuleVersion = '1.743.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 5310b72896..508f2376c1 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.742.0" +version = "1.743.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7ae2023a7c..650ccf5be0 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.742.0", + "version": "1.743.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 8edc248c45..0a22c432d7 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.742.0", + "version": "1.743.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index bee795f5f0..9926885433 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.742.0 +1.743.0 From a27e814a03c615259381eaf684aa90d56569b0af Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 30 Jun 2026 10:10:31 +0200 Subject: [PATCH 126/273] feat: add copy-to-clipboard button to rendered Mermaid diagrams in AI chat (#9838) MermaidDisplay only showed the rendered SVG, hiding the raw source once rendering succeeded. Add a copy button in the showSvg branch mirroring the pattern in HighlightCode.svelte so the diagram source can be extracted. Fixes WIN-2109 Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/chat/script/MermaidDisplay.svelte | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte index 06bcb63d98..4adf86e76d 100644 --- a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte @@ -1,6 +1,9 @@ {#if showSvg} -
- - {@html svg} +
+
{:else} From a37a144e81cf6b3de935688a617e9d0e1756004a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 30 Jun 2026 17:49:57 +0200 Subject: [PATCH 127/273] fix(ai-chat): replay anthropic turns verbatim to keep thinking valid (#9843) Co-authored-by: Claude Opus 4.8 (1M context) --- .../components/copilot/chat/anthropic.test.ts | 164 ++++++++++++++++++ .../lib/components/copilot/chat/anthropic.ts | 94 +++++++--- 2 files changed, 230 insertions(+), 28 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/anthropic.test.ts diff --git a/frontend/src/lib/components/copilot/chat/anthropic.test.ts b/frontend/src/lib/components/copilot/chat/anthropic.test.ts new file mode 100644 index 0000000000..9c9a4d0bb0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/anthropic.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import { convertOpenAIToAnthropicMessages } from './anthropic' + +// anthropic.ts pulls in the chat client/registry layer at import time; the +// converter under test is pure, so stub those side-effecting modules away. +vi.mock('../lib', () => ({ + getProviderAndCompletionConfig: vi.fn(), + workspaceAIClients: {} +})) + +vi.mock('../reasoningRegistry', () => ({ + applyReasoningToConfig: vi.fn() +})) + +vi.mock('./shared', () => ({ + processToolCall: vi.fn() +})) + +describe('convertOpenAIToAnthropicMessages', () => { + it('replays a captured assistant turn verbatim, skips the standalone text copy, and leaves the turn untouched', () => { + const anthropicContent = [ + { type: 'thinking', thinking: 'first', signature: 'sig-1' }, + { + type: 'server_tool_use', + id: 'srv_1', + name: 'web_search', + input: { query: 'nist password length' } + }, + { + type: 'web_search_tool_result', + tool_use_id: 'srv_1', + content: [{ type: 'web_search_result', title: 'NIST', url: 'https://nist.gov' }] + }, + { type: 'thinking', thinking: 'second', signature: 'sig-2' }, + { type: 'tool_use', id: 'tool_1', name: 'list_resources', input: {} } + ] + // Snapshot to assert the stored content is never mutated by the converter. + const anthropicContentSnapshot = JSON.parse(JSON.stringify(anthropicContent)) + + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'find the nist length then list resources' }, + // Standalone text the streamer emits before the tool-call message. + { role: 'assistant', content: 'Let me search the web.' }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool_1', + type: 'function', + function: { name: 'list_resources', arguments: '{}' } + } + ], + _anthropicContent: anthropicContent + } as any, + { role: 'tool', tool_call_id: 'tool_1', content: 'resource A, resource B' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + // user + the verbatim assistant turn + the tool result; the standalone text is dropped. + expect(out).toHaveLength(3) + expect(out[0]).toEqual({ role: 'user', content: 'find the nist length then list resources' }) + + // Assistant turn replayed in original order, no reordering or dropped blocks. + expect(out[1].role).toBe('assistant') + expect((out[1].content as any[]).map((b) => b.type)).toEqual([ + 'thinking', + 'server_tool_use', + 'web_search_tool_result', + 'thinking', + 'tool_use' + ]) + // The verbatim turn must stay byte-identical — no cache_control injected into it, + // or a thinking-block signature would no longer validate. + expect(out[1].content).toEqual(anthropicContentSnapshot) + expect((out[1].content as any[]).some((b) => 'cache_control' in b)).toBe(false) + expect(anthropicContent).toEqual(anthropicContentSnapshot) + + // The cache breakpoint lands on the trailing tool result, not the assistant turn. + expect(out[2]).toEqual({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tool_1', + content: 'resource A, resource B', + cache_control: { type: 'ephemeral' } + } + ] + }) + }) + + it('converts a plain text assistant turn (no tools) and caches the trailing text block', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: 'hi there' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(2) + expect(out[0]).toEqual({ role: 'user', content: 'hello' }) + expect(out[1].role).toBe('assistant') + expect(out[1].content).toEqual([ + { type: 'text', text: 'hi there', cache_control: { type: 'ephemeral' } } + ]) + }) + + it('falls back to _anthropicThinkingBlocks for turns persisted before _anthropicContent', () => { + const thinkingBlocks = [{ type: 'thinking', thinking: 'reasoning', signature: 'sig-old' }] + + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'do a thing' }, + // The standalone text persisted alongside an old-style turn must NOT be skipped + // (the fallback reconstruction relies on it for the assistant text). + { role: 'assistant', content: 'working on it' }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool_old', + type: 'function', + function: { name: 'list_resources', arguments: '{}' } + } + ], + _anthropicThinkingBlocks: thinkingBlocks + } as any + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(3) + expect(out[1]).toEqual({ role: 'assistant', content: 'working on it' }) + const content = out[2].content as any[] + // Thinking block re-injected first, then the tool_use. + expect(content.map((b) => b.type)).toEqual(['thinking', 'tool_use']) + expect(content[0]).toEqual(thinkingBlocks[0]) + expect(content[1]).toMatchObject({ type: 'tool_use', id: 'tool_old', name: 'list_resources' }) + }) + + it('caches a trailing tool result even when the prior turn used no captured content', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'q' }, + { + role: 'assistant', + tool_calls: [ + { id: 't1', type: 'function', function: { name: 'list_resources', arguments: '{}' } } + ] + } as any, + { role: 'tool', tool_call_id: 't1', content: 'done' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + const last = out[out.length - 1] + expect(last.role).toBe('user') + expect((last.content as any[])[0]).toMatchObject({ + type: 'tool_result', + tool_use_id: 't1', + cache_control: { type: 'ephemeral' } + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 48b068039f..b6e7e57a17 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -286,15 +286,15 @@ export async function parseAnthropicCompletion( role: 'assistant', tool_calls: toolCallsToProcess } - // Preserve thinking blocks (with signatures) so the next request keeps the - // reasoning chain — Anthropic requires this when thinking is combined with tool - // use. They are re-injected by convertOpenAIToAnthropicMessages. - const thinkingBlocks = finalMessage.content.filter( - (b) => b.type === 'thinking' || b.type === 'redacted_thinking' - ) - if (thinkingBlocks.length > 0) { - ;(assistantWithTools as any)._anthropicThinkingBlocks = thinkingBlocks - } + // Preserve the assistant turn verbatim (thinking/redacted_thinking with their + // signatures, server_tool_use + web_search_tool_result, text and tool_use) in + // original order. Anthropic binds each thinking block's signature to the blocks + // that precede it in the latest assistant message, so when this turn is replayed + // to continue past its tool call it must be byte-identical: reordering thinking to + // the front or dropping the web-search blocks invalidates a later block's + // signature and the request 400s with "thinking blocks ... cannot be modified". + // convertOpenAIToAnthropicMessages replays this content as-is. + ;(assistantWithTools as any)._anthropicContent = finalMessage.content messages.push(assistantWithTools) addedMessages.push(assistantWithTools) @@ -323,7 +323,33 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage let system: TextBlockParam[] | undefined const anthropicMessages: MessageParam[] = [] - for (const message of messages) { + // A streamed assistant turn that ends in tool calls is persisted as one or more + // standalone text messages followed by the tool-call message carrying + // _anthropicContent. That text is already inside _anthropicContent (replayed verbatim + // below), so drop the standalone copies — otherwise the text is duplicated and + // emitted ahead of the turn's thinking blocks. The scan stops at the preceding + // user/tool message, so only the current turn's own text is skipped. + const skipStandaloneText = new Set() + for (let i = 0; i < messages.length; i++) { + if (!(messages[i] as any)._anthropicContent) continue + for (let j = i - 1; j >= 0; j--) { + const m = messages[j] + if ( + m.role === 'assistant' && + typeof m.content === 'string' && + !m.tool_calls && + !(m as any)._anthropicContent + ) { + skipStandaloneText.add(j) + } else { + break + } + } + } + + for (let i = 0; i < messages.length; i++) { + const message = messages[i] + if (skipStandaloneText.has(i)) continue if (message.role === 'system') { const systemText = typeof message.content === 'string' ? message.content : JSON.stringify(message.content) @@ -345,10 +371,19 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage typeof message.content === 'string' ? message.content : JSON.stringify(message.content) }) } else if (message.role === 'assistant') { + // Replay a captured assistant turn verbatim so its thinking-block signatures + // stay valid (see the _anthropicContent note where the streamed turn is stored). + const anthropicContent = (message as any)._anthropicContent + if (Array.isArray(anthropicContent) && anthropicContent.length > 0) { + anthropicMessages.push({ role: 'assistant', content: anthropicContent as any }) + continue + } + const content: any[] = [] - // Re-inject preserved thinking blocks first (Anthropic requires thinking to - // precede tool_use in the same assistant turn when thinking is enabled). + // Fallback for sessions persisted before _anthropicContent existed: re-inject + // the preserved thinking blocks first (Anthropic requires thinking to precede + // tool_use in the same assistant turn when thinking is enabled). const thinkingBlocks = (message as any)._anthropicThinkingBlocks if (Array.isArray(thinkingBlocks) && thinkingBlocks.length > 0) { content.push(...thinkingBlocks) @@ -404,26 +439,29 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage } } - // Add cache_control to the last message content blocks + // Cache the conversation prefix: put an ephemeral breakpoint on the last content + // block of the last message. Each continuation only appends a tool result plus the + // next turn, so everything up to here is read from cache — which is what keeps + // replaying assistant turns verbatim (web-search results included) affordable. + // cache_control is valid on text/tool_use/tool_result blocks, but a thinking or + // redacted_thinking block must never be modified, so skip the breakpoint there. if (anthropicMessages.length > 0) { const lastMessage = anthropicMessages[anthropicMessages.length - 1] - if (Array.isArray(lastMessage.content)) { - // Add cache_control to the last content block - if (lastMessage.content.length > 0) { - const lastBlock = lastMessage.content[lastMessage.content.length - 1] - if (lastBlock.type === 'text') { - lastBlock.cache_control = { type: 'ephemeral' } - } - } - } else if (typeof lastMessage.content === 'string') { - // Convert string content to array format with cache_control + if (typeof lastMessage.content === 'string') { lastMessage.content = [ - { - type: 'text', - text: lastMessage.content, - cache_control: { type: 'ephemeral' } - } + { type: 'text', text: lastMessage.content, cache_control: { type: 'ephemeral' } } ] + } else if (Array.isArray(lastMessage.content) && lastMessage.content.length > 0) { + const lastIndex = lastMessage.content.length - 1 + const lastBlock = lastMessage.content[lastIndex] as any + if (lastBlock.type !== 'thinking' && lastBlock.type !== 'redacted_thinking') { + // Clone the block instead of mutating in place: the array may be a verbatim + // _anthropicContent turn that must stay unaltered for later requests. + lastMessage.content = [ + ...lastMessage.content.slice(0, lastIndex), + { ...lastBlock, cache_control: { type: 'ephemeral' } } + ] + } } } From 83ed011e264f20ffa66a7bf933f2fe3615cf6b67 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 30 Jun 2026 17:57:26 +0200 Subject: [PATCH 128/273] feat(object-store): make GCS service account key optional for Workload Identity (#9842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_gcs_client always called `.with_service_account_key(...)`, so an absent key (the settings UI stores "no key" as the empty JSON object `{}`) was handed to the builder and failed to parse instead of falling through to the object_store crate's InstanceCredentialProvider. Skip the call when the key is blank so GCS uses the instance's ambient credentials (GKE Workload Identity / the GCP metadata server). "Blank" (empty/whitespace/`{}`/`null`) is centralized in a shared `gcs_service_account_key_is_blank` predicate so the build path and the non-super-admin connectivity-test SSRF guard (`validate_object_storage_test`) agree on what counts as "no key" — otherwise a blank key would bypass the guard yet still trigger the ambient-credential fallback, letting an untrusted caller probe arbitrary buckets with the server's instance role. Also clarify the settings UI hint that the key may be left empty for ambient credentials, and add regression tests for the blank-key build path and the guard. Fixes WIN-2110 Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-api-settings/src/lib.rs | 26 ++++++++- backend/windmill-object-store/src/lib.rs | 58 ++++++++++++++++++- .../ObjectStoreConfigSettings.svelte | 5 +- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a7e1322ad9..8339df7b76 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -406,7 +406,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul ) } ObjectSettings::Gcs(gcs) => { - if gcs.service_account_key.is_empty() { + // Mirror `build_gcs_client`'s blank-key check (shared predicate): a blank/`{}` key falls + // back to the instance's ambient credentials there, so it must be rejected here too — + // otherwise an untrusted caller could probe with the server's identity (the very + // SSRF/credential-exfil this function guards against). + if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { return Err(error::Error::NotAuthorized( "Testing GCS storage without a service account key requires a super admin" .to_string(), @@ -2187,6 +2191,26 @@ mod object_storage_test_hardening { ); } + #[tokio::test] + async fn rejects_gcs_blank_service_account_key() { + // A blank key makes build_gcs_client fall back to the instance's ambient credentials, so an + // untrusted caller must not be allowed to test with it. The `serviceAccountKey` field is + // serialized via serde's `as_string` (`to_string` of the JSON value), so the settings UI's + // "no key" empty object arrives as `"{}"` and a null as `"null"` — both must be rejected. + for key in [serde_json::json!({}), serde_json::json!(null)] { + let settings: ObjectSettings = serde_json::from_value(serde_json::json!({ + "type": "Gcs", + "bucket": "b", + "serviceAccountKey": key + })) + .unwrap(); + assert!( + validate_object_storage_test(&settings).await.is_err(), + "blank key {key:?} should be rejected" + ); + } + } + fn ip(s: &str) -> IpAddr { s.parse().unwrap() } diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cf30efe116..553eb69917 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -494,6 +494,23 @@ fn build_azure_blob_client( return Ok(Arc::new(store)); } +/// Whether a GCS `service_account_key` carries no static credentials, in which case the client +/// should fall back to the instance's ambient credentials (GKE Workload Identity / metadata server) +/// instead of being handed an unparseable key. Besides an empty/whitespace string, the settings UI +/// stores "no key" as an empty JSON object `{}` (and `serde_json` may yield `null`), so treat those +/// as absent too. Shared with the connectivity-test SSRF guard so both agree on what "no key" means. +pub fn gcs_service_account_key_is_blank(service_account_key: &str) -> bool { + let trimmed = service_account_key.trim(); + if trimmed.is_empty() { + return true; + } + match serde_json::from_str::(trimmed) { + Ok(serde_json::Value::Null) => true, + Ok(serde_json::Value::Object(map)) => map.is_empty(), + _ => false, + } +} + #[cfg(feature = "parquet")] async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result> { let gcs_resource = gcs_resource_ref.clone(); @@ -509,7 +526,12 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result
@@ -1129,6 +1436,27 @@ + + { + const it = createConfirm + createConfirm = undefined + if (it) createOnRemote(it) + }} + onCanceled={() => (createConfirm = undefined)} + > +

+ This copies the current value of {createConfirm?.path} + (including any secret value) from + {createConfirm?.onCurrent ? currentWorkspaceId : parentWorkspaceId} + into {createConfirm?.onCurrent ? parentWorkspaceId : currentWorkspaceId}. It stays + workspace-specific afterward, so later promotes won't overwrite it. If it already exists + there, it's left untouched and just marked workspace-specific. +

+ {:else}
No comparison data available
diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte new file mode 100644 index 0000000000..bdba23c57f --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -0,0 +1,191 @@ + + +{#if isDev && parentId} +
+

+ This is a dev workspace paired with root workspace {parentId}. Promote changes + from the home page banner or the Compare & Deploy page. +

+
+ +
+
+{:else if pairedDev} +
+

+ This workspace's dev workspace is {pairedDev.name} ({pairedDev.id}). Edits to this + workspace are redirected there. +

+
+ {#if pairedDev.isMember} + + {/if} + +
+
+{:else if parentId} +

+ Dev workspace pairing is only available for root workspaces. This workspace is a fork of + {parentId}. +

+{:else} +
+

+ Pair this workspace with a dev workspace: the same code with a different environment (resource + and variable values). Edits are made in the dev workspace and promoted here. +

+
+ Attach an existing workspace as dev + Date: Thu, 2 Jul 2026 11:14:19 +0200 Subject: [PATCH 159/273] feat(frontend): pipelines index page and sql editor hint (#9881) * feat(frontend): surface pipelines in sidebar nav, index page and sql editor hint Co-Authored-By: Claude Fable 5 * feat(frontend): remove pipelines sidebar nav item Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../src/lib/components/ScriptBuilder.svelte | 55 +++++++++ .../AssetGraph/PipelineFolderList.svelte | 111 ++++++++++++++++++ .../AssetGraph/PipelinePickerModal.svelte | 93 +-------------- .../(root)/(logged)/pipeline/+page.svelte | 63 +++++----- 4 files changed, 204 insertions(+), 118 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineFolderList.svelte diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 36ac67559b..fbfa4f1c1b 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -56,6 +56,7 @@ Code, DiffIcon, EllipsisVertical, + Network, Plus, Rocket, Settings, @@ -63,6 +64,9 @@ Tag, X } from 'lucide-svelte' + import { base } from '$lib/base' + import { useLocalStorageValue } from '$lib/svelte5Utils.svelte' + import { parsePipelineAnnotations } from './assets/AssetGraph/parsePipelineAnnotations' import DropdownV2 from './DropdownV2.svelte' import { type Item } from '$lib/utils' import { sendUserToast } from '$lib/toast' @@ -314,6 +318,21 @@ const enterpriseLangs = ['bigquery', 'snowflake', 'mssql', 'oracledb'] + // Languages the pipeline editor treats as warehouse/dataset transforms — + // the ones where a `-- pipeline` annotation is a natural next step. + const pipelineHintLangs = ['duckdb', 'postgresql', 'bigquery', 'snowflake', 'mysql', 'mssql'] + const pipelineHintDismissed = useLocalStorageValue( + 'pipelineScriptHintDismissed', + false, + 'boolean' + ) + let showPipelineHint = $derived( + !pipelineHintDismissed.val && + (script.kind === 'script' || script.kind === undefined) && + pipelineHintLangs.includes(script.language ?? '') && + !parsePipelineAnnotations(script.content ?? '').inPipeline + ) + export function setCode(code: string): void { editor?.setCode(code) } @@ -1993,6 +2012,42 @@
+ {#if showPipelineHint} +
+ + + This script can become a data pipeline step: annotate it with + -- pipeline + and + + -- materialize + + to materialize its result, or build it in the + pipeline editor. + + Learn more + + +
+
+
+ {/if} + + import { userStore, workspaceStore } from '$lib/stores' + import { base } from '$lib/base' + import { goto } from '$app/navigation' + import Button from '$lib/components/common/button/Button.svelte' + import FolderPicker from '$lib/components/FolderPicker.svelte' + import { AssetService, type ListPipelineFoldersResponse } from '$lib/gen' + import { resource } from 'runed' + import { ArrowRight, Loader2 } from 'lucide-svelte' + + interface Props { + // When provided, the current folder is dropped from the existing list + // so users don't "switch" to the folder they're already on. + currentFolder?: string | undefined + // Carried over to the target URL so switching folders from the + // editor keeps the user in edit mode; the default view mode needs + // no param. + mode?: 'view' | 'edit' + // Called right before navigating — lets a wrapping modal close itself. + onOpen?: () => void + } + let { currentFolder = undefined, mode = 'view', onOpen = undefined }: Props = $props() + + let pipelines = resource( + () => $workspaceStore, + async (ws) => { + if (!ws) return [] as ListPipelineFoldersResponse + return await AssetService.listPipelineFolders({ workspace: ws }) + } + ) + + let pickedFolder = $state('') + + let visiblePipelines = $derived( + (pipelines.current ?? []).filter((p) => p.folder !== currentFolder) + ) + + async function openExistingPipeline(folder: string) { + onOpen?.() + const modeQuery = mode !== 'view' ? `?mode=${mode}` : '' + await goto(`${base}/pipeline/${encodeURIComponent(folder)}${modeQuery}`) + } + + async function openPicked() { + const name = pickedFolder.trim() + if (!name) return + await openExistingPipeline(name) + } + + +
+
+

+ Existing pipelines +

+ {#if pipelines.loading && !pipelines.current} +
+ + Loading… +
+ {:else if pipelines.error} +
Failed: {pipelines.error.message}
+ {:else if visiblePipelines.length === 0} +
+ {currentFolder + ? 'No other pipelines in this workspace.' + : 'No pipelines yet. A pipeline is any folder whose scripts carry pipeline annotations.'} +
+ {:else} +
+ {#each visiblePipelines as p (p.folder)} + + {/each} +
+ {/if} +
+ + {#if !$userStore?.operator} + +
+

+ Pick or create a folder +

+
+
+ +
+ +
+
+ {/if} +
diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte index ded1013e25..b2183eec77 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte @@ -1,13 +1,6 @@ -
- {#if visiblePipelines.length > 0 || pipelines.loading} -
-

- Existing pipelines -

- {#if pipelines.loading && !pipelines.current} -
- - Loading… -
- {:else if pipelines.error} -
Failed: {pipelines.error.message}
- {:else} -
- {#each visiblePipelines as p} - - {/each} -
- {/if} -
- {/if} - - {#if !$userStore?.operator} - -
-

- Pick or create a folder -

-
-
- -
- -
-
- {/if} -
+ (open = false)} /> diff --git a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte index 98f598165e..ab36b4c1f7 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte @@ -1,24 +1,20 @@ - Pipeline editor — Windmill + Pipelines — Windmill
@@ -45,28 +40,42 @@ >
-

- {$userStore?.operator ? 'Pipelines' : 'Pipeline editor'} -

- · no folder selected +

Pipelines

+ + Alpha +
-
-
- - Pick a folder to open its pipeline. - +
+
+
+

Data pipelines

+

+ Chain ingestion, transformation and materialization steps into an asset-aware graph. + {#if !$userStore?.operator} + Each pipeline lives in a folder: open one below, or pick a folder to start a new + pipeline. + {:else} + Open a pipeline below to view its graph and runs. + {/if} +

+ + + Pipelines documentation + +
+ +
- From b92a86b8b3a60b877540c3a7f0ffefe36ccbb053 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 2 Jul 2026 11:48:11 +0200 Subject: [PATCH 160/273] fix: polish pipeline graph view (layout, viewport, minimap, lineage, timestamps) (#9883) * fix(frontend): keep pipeline graph layered when lineage has cycles Co-Authored-By: Claude Fable 5 * fix(frontend): fit pipeline graph to visible canvas on initial load Co-Authored-By: Claude Fable 5 * fix(frontend): style pipeline minimap so it reads as a minimap Co-Authored-By: Claude Fable 5 * fix(parser): don't infer s3 reads from bare string-literal mentions in sql Co-Authored-By: Claude Fable 5 * fix(duckdb): render temporal values as ISO strings in job results Co-Authored-By: Claude Fable 5 * fix(frontend): key pipeline viewport fit on the loaded graph's folder Co-Authored-By: Claude Fable 5 * fix(parser): treat list/named read-fn arguments as definitive s3 reads Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../src/asset_parser.rs | 220 ++++++++++++++++-- .../windmill-duckdb-ffi-internal/src/lib.rs | 114 ++++++++- .../assets/AssetGraph/AssetGraphCanvas.svelte | 50 +++- .../assets/AssetGraph/InitialFitView.svelte | 30 +++ .../AssetGraph/PipelineGraphEditor.svelte | 9 + .../AssetGraph/assetGraphLayout.test.ts | 40 +++- .../assets/AssetGraph/assetGraphLayout.ts | 37 ++- .../sessions/PipelineEditorView.svelte | 10 + .../(logged)/pipeline/[folder]/+page.svelte | 11 + 9 files changed, 482 insertions(+), 39 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index f7c578a274..c4f93efdb7 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -83,18 +83,56 @@ pub fn parse_assets(input: &str) -> anyhow::Result { // Body-inferred column lineage, with `// column` annotations taking // precedence per output column (explicit declaration overrides inference). pipeline.column_lineage = merge_column_lineage(inferred, pipeline.column_lineage); - Ok(ParseAssetsOutput::new( - merge_assets(collector.assets), - Vec::new(), - pipeline, - )) + // A bare string literal in query position is only weak read evidence: a + // summary `SELECT 's3:///out.csv' AS target` after `COPY … TO + // 's3:///out.csv'` must not turn the write into rw (which draws a + // spurious read edge and an asset⇄script cycle in the pipeline graph). + // Surface weak reads only for assets with no other recorded usage, so a + // path that is *merely* mentioned still shows up linked to the script. + let mut assets = merge_assets(collector.assets); + for weak in merge_assets(collector.weak_string_reads) { + if !assets + .iter() + .any(|a| a.kind == weak.kind && a.path == weak.path) + { + assets.push(weak); + } + } + assets.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(ParseAssetsOutput::new(assets, Vec::new(), pipeline)) +} + +/// Provenance of the innermost access context. The access type alone can't +/// tell a definitive read apart from a mere mention: a bare string literal in +/// generic query position (`QueryRead`) is only *weak* read evidence — e.g. a +/// summary `SELECT 's3:///out.csv' AS target` echoing a path — while the same +/// literal as a read-function argument or a `COPY` target is definitive. +#[derive(Clone, Copy, PartialEq, Eq)] +enum AccessCtx { + QueryRead, + ReadFn, + CopyWrite, +} + +impl AccessCtx { + fn access_type(self) -> AssetUsageAccessType { + match self { + AccessCtx::QueryRead | AccessCtx::ReadFn => R, + AccessCtx::CopyWrite => W, + } + } } /// Visitor that collects S3 asset literals from SQL statements struct AssetCollector { assets: Vec, - // e.g set to Read when we are inside a SELECT ... FROM ... statement - current_access_type_stack: Vec, + // Bare string literals seen in generic query position — weak read + // evidence, surfaced by `parse_assets` only when the script has no other + // recorded usage of the same asset (a real write must not gain a spurious + // read edge from a mention). + weak_string_reads: Vec, + // e.g set to QueryRead when we are inside a SELECT ... FROM ... statement + current_access_type_stack: Vec, // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") } var_identifiers: BTreeMap, // e.g USE dl; @@ -119,6 +157,7 @@ impl AssetCollector { fn new() -> Self { Self { assets: Vec::new(), + weak_string_reads: Vec::new(), current_access_type_stack: Vec::with_capacity(8), var_identifiers: BTreeMap::new(), currently_used_asset: None, @@ -162,7 +201,11 @@ impl AssetCollector { name: &ObjectName, access_type: Option, ) -> Option { - let access_type = access_type.or_else(|| self.current_access_type_stack.last().copied()); + let access_type = access_type.or_else(|| { + self.current_access_type_stack + .last() + .map(|c| c.access_type()) + }); if let Some((kind, path)) = &self.currently_used_asset { // We don't want to infer that any simple identifier refers to an asset if // we are not in a known R/W context @@ -301,12 +344,18 @@ impl AssetCollector { // Check if the string matches our asset syntax patterns if let Some((kind, path)) = parse_asset_syntax(s, false) { if kind == AssetKind::S3Object { - self.assets.push(ParseAssetsResult { + let ctx = self.current_access_type_stack.last().copied(); + let result = ParseAssetsResult { kind, path: path.to_string(), - access_type: self.current_access_type_stack.last().copied(), + access_type: ctx.map(AccessCtx::access_type), columns: None, - }); + }; + if ctx == Some(AccessCtx::QueryRead) { + self.weak_string_reads.push(result); + } else { + self.assets.push(result); + } } } } @@ -314,7 +363,7 @@ impl AssetCollector { fn handle_obj_name_pre(&mut self, name: &ObjectName) { if let Some(fname) = get_trivial_obj_name(name) { if is_read_fn(fname) { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::ReadFn); } } if let Some(str_lit) = get_str_lit_from_obj_name(name) { @@ -691,9 +740,20 @@ impl Visitor for AssetCollector { match table_factor { TableFactor::Table { name, args, .. } => { if args.is_none() { - // Avoid Table Functions - self.handle_obj_name_pre(name); + // FROM 's3:///…' is a definitive read — record it directly + // so it isn't demoted to a weak in-query mention. + if let Some(asset) = self.get_s3_asset_from_str_literal_table(table_factor) { + self.assets.push(asset); + } } + // For a read-function table factor this pushes ReadFn, making + // every literal inside its arguments a definitive read — the + // direct form (read_csv('s3:///…')) but also list and named + // arguments (read_parquet(['s3:///…'])). Must run for BOTH the + // plain-table and table-function branches: post_visit_table_factor + // pops via handle_obj_name_post unconditionally, so skipping the + // push here would unbalance the stack. + self.handle_obj_name_pre(name); } _ => {} } @@ -719,6 +779,13 @@ impl Visitor for AssetCollector { Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => { self.handle_string_literal(s); } + // Read-function call in expression position: its argument literals + // are definitive reads. Balances the pop in `post_visit_expr`. + Expr::Function(func) => { + if get_trivial_obj_name(&func.name).is_some_and(is_read_fn) { + self.current_access_type_stack.push(AccessCtx::ReadFn); + } + } _ => {} } std::ops::ControlFlow::Continue(()) @@ -946,7 +1013,7 @@ impl Visitor for AssetCollector { } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { - self.current_access_type_stack.push(W); + self.current_access_type_stack.push(AccessCtx::CopyWrite); self.handle_string_literal(filename); self.current_access_type_stack.pop(); } @@ -1013,7 +1080,7 @@ impl Visitor for AssetCollector { &mut self, query: &sqlparser::ast::Query, ) -> std::ops::ControlFlow { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::QueryRead); self.cte_name_stack.push(collect_cte_names(query)); std::ops::ControlFlow::Continue(()) } @@ -1098,6 +1165,127 @@ mod tests { ); } + #[test] + fn test_copy_target_echoed_in_select_stays_write_only() { + // The trailing summary SELECT merely mentions the COPY target — it + // must not add a read (rw would draw an asset⇄script cycle). + let input = r#" + COPY (SELECT 1 AS x) TO 's3:///out.csv'; + SELECT 's3:///out.csv' AS target, 42 AS rows_written; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/out.csv".to_string(), + access_type: Some(W), + columns: None + }]) + ); + } + + #[test] + fn test_bare_string_mention_without_other_usage_is_a_read() { + let input = r#" + SELECT 's3:///referenced.csv' AS path; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/referenced.csv".to_string(), + access_type: Some(R), + columns: None + }]) + ); + } + + #[test] + fn test_self_refresh_read_fn_plus_copy_stays_rw() { + // A definitive read (read_csv) of the same file the script rewrites + // is a real rw — only *bare-literal* mentions are demoted. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_csv('s3:///data.csv'); + COPY (SELECT * FROM tmp) TO 's3:///data.csv'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.csv".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_plus_copy_stays_rw() { + // read_parquet's list form is as definitive as the direct literal — + // it must not be demoted to a weak mention when the file is rewritten. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_parquet(['s3:///data.parquet']); + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_multiple_files_are_reads() { + let input = r#" + SELECT * FROM read_parquet(['s3:///a.parquet', 's3:///b.parquet']); + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/a.parquet".to_string(), + access_type: Some(R), + columns: None + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/b.parquet".to_string(), + access_type: Some(R), + columns: None + } + ]) + ); + } + + #[test] + fn test_from_string_literal_of_written_file_stays_rw() { + // FROM-position string literal is likewise a definitive read. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM 's3:///data.parquet'; + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + #[test] fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() { let input = r#" diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index f3ee59cab7..054f892220 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -635,7 +635,9 @@ fn duckdb_value_to_json_value( .ok_or_else(|| "Could not convert to f64".to_string())?, ), duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), - duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Timestamp(unit, ts) => { + serde_json::Value::String(duckdb_timestamp_to_iso(unit, ts)) + } duckdb::types::Value::Text(s) if type_alias.as_deref().unwrap_or_default() == "JSON" => { serde_json::from_str(&s) .map_err(|e| format!("Error parsing JSON text: {}", e.to_string()))? @@ -646,8 +648,15 @@ fn duckdb_value_to_json_value( .map(|byte| serde_json::Value::Number(byte.into())) .collect(), ), - duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), - duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Date32(d) => { + match chrono::DateTime::from_timestamp(i64::from(d) * 86_400, 0) { + Some(dt) => serde_json::Value::String(dt.date_naive().to_string()), + None => serde_json::Value::Number(d.into()), + } + } + duckdb::types::Value::Time64(unit, t) => { + serde_json::Value::String(duckdb_time_to_iso(unit, t).unwrap_or_else(|| t.to_string())) + } duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ "months": months, "days": days, @@ -688,6 +697,105 @@ fn duckdb_value_to_json_value( Ok(json_value) } +// DuckDB surfaces TIMESTAMP[_S/_MS/_NS] values as a raw count since the epoch; +// stringifying that count leaks values like "1782974022218435" into job +// results. Render ISO-8601 instead (the Postgres executor's +// "2024-01-15T10:30:00" shape); a count outside chrono's representable range +// falls back to the raw number. +fn duckdb_timestamp_to_iso(unit: duckdb::types::TimeUnit, ts: i64) -> String { + let dt = match unit { + duckdb::types::TimeUnit::Second => chrono::DateTime::from_timestamp(ts, 0), + duckdb::types::TimeUnit::Millisecond => chrono::DateTime::from_timestamp_millis(ts), + duckdb::types::TimeUnit::Microsecond => chrono::DateTime::from_timestamp_micros(ts), + duckdb::types::TimeUnit::Nanosecond => Some(chrono::DateTime::from_timestamp_nanos(ts)), + }; + dt.map(|dt| dt.naive_utc().format("%Y-%m-%dT%H:%M:%S%.f").to_string()) + .unwrap_or_else(|| ts.to_string()) +} + +// Same story for TIME: a raw count since midnight. None when out of range +// (caller falls back to the raw number). +fn duckdb_time_to_iso(unit: duckdb::types::TimeUnit, t: i64) -> Option { + let (secs, nanos) = match unit { + duckdb::types::TimeUnit::Second => (t, 0), + duckdb::types::TimeUnit::Millisecond => (t / 1_000, (t % 1_000) * 1_000_000), + duckdb::types::TimeUnit::Microsecond => (t / 1_000_000, (t % 1_000_000) * 1_000), + duckdb::types::TimeUnit::Nanosecond => (t / 1_000_000_000, t % 1_000_000_000), + }; + chrono::NaiveTime::from_num_seconds_from_midnight_opt( + u32::try_from(secs).ok()?, + u32::try_from(nanos).ok()?, + ) + .map(|t| t.format("%H:%M:%S%.f").to_string()) +} + +#[cfg(test)] +mod temporal_json_tests { + use super::*; + use duckdb::types::TimeUnit; + + #[test] + fn timestamp_micros_renders_iso() { + // 2026-07-01 23:13:42.218435 UTC + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Microsecond, 1_782_947_622_218_435), + "2026-07-01T23:13:42.218435" + ); + } + + #[test] + fn timestamp_seconds_renders_iso_without_subseconds() { + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Second, 1_735_689_600), + "2025-01-01T00:00:00" + ); + } + + #[test] + fn out_of_range_timestamp_falls_back_to_raw() { + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Second, i64::MAX), + i64::MAX.to_string() + ); + } + + #[test] + fn time_micros_renders_iso() { + assert_eq!( + duckdb_time_to_iso(TimeUnit::Microsecond, 37_800_500_000).as_deref(), + Some("10:30:00.500") + ); + } + + #[test] + fn temporal_values_render_iso_through_real_query() { + let conn = duckdb::Connection::open_in_memory().unwrap(); + let mut stmt = conn + .prepare( + "SELECT TIMESTAMP '2026-07-01 23:13:42.218435' AS ts, + TIMESTAMPTZ '2026-07-01 23:13:42+00' AS tstz, + TIMESTAMP_NS '2026-07-01 23:13:42.218435678' AS ts_ns, + DATE '2026-07-01' AS d, + TIME '10:30:00' AS t", + ) + .unwrap(); + let mut rows = stmt.query([]).unwrap(); + let row = rows.next().unwrap().unwrap(); + let json_of = |i: usize| { + let v: duckdb::types::Value = row.get(i).unwrap(); + duckdb_value_to_json_value(v, &None).unwrap() + }; + assert_eq!(json_of(0), serde_json::json!("2026-07-01T23:13:42.218435")); + assert_eq!(json_of(1), serde_json::json!("2026-07-01T23:13:42")); + assert_eq!( + json_of(2), + serde_json::json!("2026-07-01T23:13:42.218435678") + ); + assert_eq!(json_of(3), serde_json::json!("2026-07-01")); + assert_eq!(json_of(4), serde_json::json!("10:30:00")); + } +} + fn json_value_to_duckdb_value( json_value: &serde_json::Value, arg_type: &str, diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 153d7c2965..6851fc843a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -16,6 +16,7 @@ import DataTestNode from './DataTestNode.svelte' import AssetGraphEdge from './AssetGraphEdge.svelte' import PanToNode from './PanToNode.svelte' + import InitialFitView from './InitialFitView.svelte' import { layoutAssetGraph } from './assetGraphLayout' import { buildDownstreamMap } from './graphTraversal' import { buildLineageDownstreamMap } from './boundedCascade' @@ -162,6 +163,10 @@ /** Hide the minimap when the canvas is too narrow for it to be worth the * space (e.g. stacked layout in a side panel). Defaults to shown. */ showMinimap?: boolean + /** Identity of the displayed graph (e.g. the pipeline folder). The + * initial viewport fit re-arms when it changes, so switching folders + * in-place gets a fresh fit. */ + viewportFitKey?: string } let { graph, @@ -188,7 +193,8 @@ onStartBoundedRun, boundPick, onPickEnd, - showMinimap = true + showMinimap = true, + viewportFitKey = '' }: Props = $props() // `${kind}:${path}` ids for the hovered / pinned runs (both script and flow @@ -677,11 +683,27 @@ // edges + paths → same layout. Renames *do* change the layout for // the renamed entry, since its id moves in the sort, but that's // expected: a rename is a path change, which is part of the input. + // A script with rw access to an asset yields both a write (script → asset) + // and a read/trigger (asset → script) edge — a 2-cycle. The layout resolves + // it producer-above-asset by omitting the backward direction from its + // input; the rendered edges are untouched (both arrows still drawn). + let writeEdgePairs = $derived( + new Set( + model.edges.filter((e) => e.kind === 'lineage-write').map((e) => `${e.source}\n${e.target}`) + ) + ) let layoutInput = $derived({ nodes: model.nodes .map((n) => ({ id: n.id, data: n.data })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), edges: model.edges + .filter( + (e) => + !( + (e.kind === 'lineage-read' || e.kind === 'trigger-asset') && + writeEdgePairs.has(`${e.target}\n${e.source}`) + ) + ) .map((e) => ({ source: e.source, target: e.target })) .sort((a, b) => a.source === b.source @@ -1023,21 +1045,35 @@ --background-color={false} >
+ {#if showMinimap} + n.type === 'asset' - ? 'rgb(96 165 250 / 0.5)' + ? 'rgb(59 130 246 / 0.3)' : n.type === 'trigger' - ? 'rgb(251 191 36 / 0.5)' - : 'rgb(52 211 153 / 0.5)'} - nodeStrokeColor="transparent" - maskColor="rgb(0 0 0 / 0.2)" + ? 'rgb(245 158 11 / 0.3)' + : 'rgb(148 163 184 / 0.15)'} + nodeStrokeColor={(n) => + n.type === 'asset' + ? 'rgb(59 130 246 / 0.8)' + : n.type === 'trigger' + ? 'rgb(245 158 11 / 0.8)' + : 'rgb(100 116 139 / 0.7)'} + maskColor="rgb(100 116 139 / 0.12)" + maskStrokeColor="rgb(59 130 246 / 0.5)" + maskStrokeWidth={4} /> {/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte b/frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte new file mode 100644 index 0000000000..eea1fb306d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte @@ -0,0 +1,30 @@ + diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 33d44eb7a4..8dca18bd8e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -31,6 +31,7 @@ mode, workspace, folder, + viewportFitKey = undefined, stackBelow = 680, persistDrafts = false, pathPrefix, @@ -95,6 +96,13 @@ workspace: string | undefined /** Folder the pipeline is scoped to — drives the autosave draft path. */ folder: string + /** Folder whose graph is actually *loaded* (not the route param). On an + * in-place folder switch the stale graph stays rendered while the new + * fetch is in flight, so keying the canvas's one-shot initial fit on + * `folder` would consume the new folder's fit on the old graph. Pages + * that stale-while-revalidate should pass the folder captured when the + * fetch resolved; defaults to `folder`. */ + viewportFitKey?: string /** Below this container width (px) the graph/details split stacks * vertically instead of side-by-side — for narrow side panels / AI * session previews. Defaults to 680. */ @@ -433,6 +441,7 @@ {onPickEnd} {panToNodeId} showMinimap={!stacked} + viewportFitKey={viewportFitKey ?? folder} /> {#if boundBar}{@render boundBar()}{/if} {#if mode === 'edit'} diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts index a5a92f61c2..24e7d4c2da 100644 --- a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts @@ -38,7 +38,14 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => { // stays strictly left of every node of the right branch. const pos = layoutAssetGraph({ nodes: [n('root'), n('a'), n('b'), n('a1'), n('a2'), n('a3'), n('b1')], - edges: [e('root', 'a'), e('root', 'b'), e('a', 'a1'), e('a', 'a2'), e('a', 'a3'), e('b', 'b1')] + edges: [ + e('root', 'a'), + e('root', 'b'), + e('a', 'a1'), + e('a', 'a2'), + e('a', 'a3'), + e('b', 'b1') + ] }) const leftMax = Math.max(...['a', 'a1', 'a2', 'a3'].map((id) => pos.get(id)!.x)) const rightMin = Math.min(...['b', 'b1'].map((id) => pos.get(id)!.x)) @@ -83,14 +90,37 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => { } }) - it('falls back to a grid on cyclic input', () => { + it('lays a 2-cycle out as a chain (feedback edge dropped, no grid)', () => { const pos = layoutAssetGraph({ nodes: [n('a'), n('b')], edges: [e('a', 'b'), e('b', 'a')] }) - expect(pos.size).toBe(2) - expect(pos.get('a')).toBeDefined() - expect(pos.get('b')).toBeDefined() + // First-in-input wins the top slot; the b→a feedback edge is ignored. + expect(pos.get('a')!.y).toBeLessThan(pos.get('b')!.y) + expect(pos.get('a')!.x).toBe(pos.get('b')!.x) + }) + + it('keeps the acyclic part of a graph layered when one cycle exists', () => { + // root → a ⇄ b → leaf: the a⇄b cycle must not degrade root/leaf layering. + const pos = layoutAssetGraph({ + nodes: [n('root'), n('a'), n('b'), n('leaf')], + edges: [e('root', 'a'), e('a', 'b'), e('b', 'a'), e('b', 'leaf')] + }) + expect(pos.get('root')!.y).toBeLessThan(pos.get('a')!.y) + expect(pos.get('a')!.y).toBeLessThan(pos.get('b')!.y) + expect(pos.get('b')!.y).toBeLessThan(pos.get('leaf')!.y) + // A linear chain stays in one column. + expect(new Set(['root', 'a', 'b', 'leaf'].map((id) => pos.get(id)!.x)).size).toBe(1) + }) + + it('handles a longer cycle without dropping nodes', () => { + const pos = layoutAssetGraph({ + nodes: [n('a'), n('b'), n('c')], + edges: [e('a', 'b'), e('b', 'c'), e('c', 'a')] + }) + expect(pos.size).toBe(3) + const ys = ['a', 'b', 'c'].map((id) => pos.get(id)!.y) + expect(new Set(ys).size).toBe(3) }) it('packs disjoint components side by side without overlap', () => { diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts index cf55db6e0a..c9d7d60355 100644 --- a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts @@ -53,8 +53,8 @@ interface Band { // // y comes from longest-path layering (same top-down orientation as before: // producers above, assets in the middle, consumers below). Returns positions -// (band centers) normalized so the component's min x,y = 0. Throws on cyclic -// input (caller falls back to a grid for the whole graph). +// (band centers) normalized so the component's min x,y = 0. Cyclic input is +// handled by dropping feedback edges (see the Kahn step below). function layoutComponent( nodes: GraphInput['nodes'], edges: GraphInput['edges'] @@ -76,21 +76,42 @@ function layoutComponent( if (!parents.get(e.target)!.includes(e.source)) parents.get(e.target)!.push(e.source) } - // Kahn topological order — also the cycle guard. + // Kahn topological order. Cycles don't abort the layout: when the queue + // drains with nodes left, the unplaced node with the fewest outstanding + // parents (first in input order on ties) is forced into the order and its + // not-yet-placed parent edges are dropped as feedback edges — layering and + // tree-building then operate on the resulting DAG while the rendered graph + // keeps every edge. (The caller already resolves write⇄read 2-cycles by + // omitting the read direction; this handles any longer cycle.) const indeg = new Map() for (const n of nodes) indeg.set(n.id, parents.get(n.id)!.length) const queue = nodes.filter((n) => indeg.get(n.id) === 0).map((n) => n.id) + const placed = new Set() const topo: string[] = [] - while (queue.length) { + while (topo.length < nodes.length) { + if (queue.length === 0) { + let pick: string | undefined + for (const n of nodes) { + if (placed.has(n.id)) continue + if (pick === undefined || indeg.get(n.id)! < indeg.get(pick)!) pick = n.id + } + parents.set( + pick!, + parents.get(pick!)!.filter((p) => placed.has(p)) + ) + queue.push(pick!) + } const cur = queue.shift()! + if (placed.has(cur)) continue + placed.add(cur) topo.push(cur) for (const c of children.get(cur)!) { + if (placed.has(c)) continue const d = indeg.get(c)! - 1 indeg.set(c, d) if (d === 0) queue.push(c) } } - if (topo.length !== nodes.length) throw new Error('cyclic asset graph') // Longest-path layering: a node sits one layer below its lowest parent. const layer = new Map() @@ -141,8 +162,7 @@ function layoutComponent( out.set(id, { x: left + w / 2, y: layer.get(id)! * LAYER_H }) const kids = treeChildren.get(id)! if (kids.length === 0) return - const kidsW = - kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1) + const kidsW = kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1) let cursor = left + (w - kidsW) / 2 for (const k of kids) { placeTree(k, cursor) @@ -221,7 +241,8 @@ function layoutComponent( // disjoint components, so it's excluded from component detection and instead // re-placed centered one layer above the whole packed graph. // -// Falls back to a stable grid if the component layout throws (cyclic inputs). +// Falls back to a stable grid if the component layout throws (defensive — +// cycles are already absorbed by feedback-edge dropping in layoutComponent). export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map { const byId = new Map() if (graph.nodes.length === 0) return byId diff --git a/frontend/src/lib/components/sessions/PipelineEditorView.svelte b/frontend/src/lib/components/sessions/PipelineEditorView.svelte index e0578ae139..0e3be64e36 100644 --- a/frontend/src/lib/components/sessions/PipelineEditorView.svelte +++ b/frontend/src/lib/components/sessions/PipelineEditorView.svelte @@ -75,6 +75,15 @@ workspace && folder ? await AssetService.getAssetsGraph({ workspace, folder }) : EMPTY_GRAPH ) + // Folder whose graph is actually rendered — `graphRes.current` is stale- + // while-revalidate on a folder retarget, so the canvas's one-shot initial + // fit is keyed on the folder captured when a graph lands, not on `path` + // (same rationale as the pipeline route page). + let viewportFitFolder = $state('') + $effect(() => { + if (graphRes.current) untrack(() => (viewportFitFolder = path)) + }) + // Deployed graph + the in-flight draft overlay (AI-built nodes render as plain // dashed unsaved drafts, same as manual drafts). The session // skips the route page's folder-wide asset prefetch (empty inferred maps); the @@ -304,6 +313,7 @@ { + if (graphRes.current) untrack(() => (viewportFitFolder = folder)) + }) + // Body / inferred-assets prefetch sweep. Watches `g.runnables`; for any // non-draft path we haven't fetched yet, fetches `getScriptByPath` and // `inferAssets`, and stores both in their respective only-add caches. @@ -2121,6 +2131,7 @@ Date: Thu, 2 Jul 2026 11:58:31 +0200 Subject: [PATCH 161/273] reserve more fixed height for unsaved-changes banner to avoid content shift (#9873) * fix(frontend): reserve fixed height for unsaved-changes banner to avoid content shift Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): remove border around reserved banner slot and shrink it Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): tighten top padding under the unsaved-changes banner Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): give unsaved-changes banner buttons minimal vertical breathing room Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(frontend): drop redundant Metadata section title in trigger and script editors Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): tuck schedule editor labels under summary to match convention Co-Authored-By: Claude Opus 4.8 (1M context) * revert(frontend): keep Metadata section title in ScriptBuilder for a separate PR Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): drop leftover header-content margin on headless Section Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): drop top padding above resource editor first field Co-Authored-By: Claude Opus 4.8 (1M context) * style(frontend): match variable editor bottom padding to resource Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): add Path label in new resource form to match edit Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): reserve half the banner height to halve the idle gap Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): reserve a third of the banner height when idle Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): don't reserve banner slot or tighten top for new entities Gate the reserved-height slot and the tight content top padding on the banner's baseline (bannerReserved) instead of merely on the banner snippet being present, so new-entity drawers keep normal top spacing and add no empty slot. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(frontend): trim banner comments to the 4-line invariant limit Co-Authored-By: Claude Opus 4.8 (1M context) * docs(frontend): describe partial-reserve banner behavior accurately Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/lib/components/AppConnectInner.svelte | 32 ++++--- .../lib/components/LocalDraftBanner.svelte | 91 +++++++++++-------- .../src/lib/components/ResourceEditor.svelte | 2 +- .../components/ResourceEditorDrawer.svelte | 2 + frontend/src/lib/components/Section.svelte | 2 +- .../src/lib/components/VariableEditor.svelte | 4 +- .../common/drawer/DrawerContent.svelte | 44 ++++++--- .../azure/AzureTriggerEditorInner.svelte | 2 + .../email/EmailTriggerEditorInner.svelte | 4 +- .../triggers/gcp/GcpTriggerEditorInner.svelte | 2 + .../triggers/http/RouteEditorInner.svelte | 4 +- .../kafka/KafkaTriggerEditorInner.svelte | 2 + .../mqtt/MqttTriggerEditorInner.svelte | 2 + .../native/NativeTriggerEditor.svelte | 2 +- .../nats/NatsTriggerEditorInner.svelte | 2 + .../PostgresTriggerEditorInner.svelte | 2 + .../schedules/ScheduleEditorInner.svelte | 6 +- .../triggers/sqs/SqsTriggerEditorInner.svelte | 2 + .../triggers/useTriggerDraftSync.svelte.ts | 12 +++ .../WebsocketTriggerEditorInner.svelte | 2 + 20 files changed, 148 insertions(+), 73 deletions(-) diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 9edd80dd54..ef632f5738 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -991,13 +991,15 @@
{:else if step == 2 && manual}
- + {#if deployTo}
`). + path: String, + // Inclusive calendar-day range (YYYY-MM-DD), local to the producer's + // partition tz. + from: chrono::NaiveDate, + to: chrono::NaiveDate, +} + +#[derive(Serialize)] +struct PartitionInRange { + partition: String, + // `missing` | `running` | `materialized` | `failed` — `missing` means no + // materialization was ever recorded for the slice. + status: &'static str, +} + +#[derive(Serialize)] +struct PartitionsInRangeResponse { + // The pipeline script that materializes the asset (managed `// materialize` + // target, or a partitioned writer using the SDK helpers) — the runnable a + // backfill launches (with an explicit `partition` arg per slice). + producer_path: String, + partition_kind: String, + partitions: Vec, +} + +// Backfill range preview: every partition the producer's `// partitioned` spec +// expects in `[from, to]`, joined with what `materialized_partition` records — +// the missing/failed subset is the backfill worklist. The logic is in the +// `backfill` module pair: EE resolves and enumerates, the OSS stub errors +// (single-partition runs stay available everywhere; fanning out over a range +// is enterprise). +async fn list_partitions_in_range( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let res = backfill::partitions_in_range(&mut tx, &w_id, &q).await?; + tx.commit().await?; + Ok(Json(res)) +} + // Per-asset captured output schema versions for a ducklake asset (gap #2a) — // the schema-evolution history persisted after each managed `// materialize`. // Newest version first; materialization targets are ducklake-only in v1, so the diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 56893a8737..ebbc98e83e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0fcbb9132e..e7b5badb8c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -21498,6 +21498,42 @@ paths: items: $ref: "#/components/schemas/MaterializedPartition" + /w/{workspace}/assets/partitions_in_range: + get: + summary: List expected partitions of a ducklake asset in a date range with their materialization status (enterprise) + operationId: listAssetPartitionsInRange + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/
`) + schema: + type: string + - name: from + in: query + required: true + description: Inclusive range start (YYYY-MM-DD), local to the producer's partition tz + schema: + type: string + format: date + - name: to + in: query + required: true + description: Inclusive range end (YYYY-MM-DD), local to the producer's partition tz + schema: + type: string + format: date + responses: + "200": + description: expected partitions in range with per-slice status — the missing/failed subset is the backfill worklist + content: + application/json: + schema: + $ref: "#/components/schemas/PartitionsInRange" + /w/{workspace}/assets/asset_schemas: get: summary: List captured output-schema versions for a ducklake asset @@ -29692,6 +29728,27 @@ components: error: type: string nullable: true + PartitionsInRange: + type: object + required: [producer_path, partition_kind, partitions] + properties: + producer_path: + type: string + description: the pipeline script that materializes the asset (managed `// materialize` target, or a partitioned writer using the SDK helpers) — the runnable a backfill launches + partition_kind: + type: string + enum: [daily, hourly, weekly, monthly, dynamic] + partitions: + type: array + items: + type: object + required: [partition, status] + properties: + partition: + type: string + status: + type: string + enum: [missing, running, materialized, failed] AssetSchemaVersion: type: object required: [version, columns, captured_at] diff --git a/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte b/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte index 1826ff6b41..e4390fe503 100644 --- a/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte @@ -2,78 +2,217 @@ import { Button } from '$lib/components/common' import Modal from '$lib/components/common/modal/Modal.svelte' import DateInput from '$lib/components/DateInput.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { Loader2 } from 'lucide-svelte' + import { resource } from 'runed' + import { AssetService } from '$lib/gen' import { enterpriseLicense } from '$lib/stores' + import type { BackfillSliceState } from './backfillRun' - // Backfill re-runs the CE materialization once per partition in [from, to]. - // It is an enterprise feature (orchestration over a range); the dialog is - // only reachable when licensed, but we guard here too so the action can - // never fire in CE. + // Range backfill of a partitioned ducklake asset (enterprise). The dialog + // previews the expected partitions in [from, to] against what + // `materialized_partition` records (the missing/failed set is the + // worklist), then hands the worklist to the parent, which runs the + // producer once per slice (see `backfillRun.ts`). Progress state lives in + // the parent so a run survives closing the dialog. interface Props { // Controlled by the parent. `$bindable()` without a default per the // AGENTS.md ban on `$bindable(default)` for optional props. open?: boolean assetPath: string - // Invoked with the inclusive ISO date range; the parent performs the - // actual fan-out (EE backfill endpoint). - onBackfill: (from: string, to: string) => Promise + workspace: string + // Live per-slice states of the in-flight (or last finished) backfill. + slices?: BackfillSliceState[] + running?: boolean + // Cancellation was requested but the in-flight slice hasn't finished yet. + cancelRequested?: boolean + onStart: (producerPath: string, partitions: string[]) => void + onCancel: () => void + // Clear the finished run so the range picker shows again. + onReset: () => void } - let { open = $bindable(), assetPath, onBackfill }: Props = $props() + let { + open = $bindable(), + assetPath, + workspace, + slices, + running, + cancelRequested, + onStart, + onCancel, + onReset + }: Props = $props() let fromDate = $state(undefined) let toDate = $state(undefined) - let loading = $state(false) - let error = $state(undefined) + let onlyMissing = $state(true) - let canSubmit = $derived(!!$enterpriseLicense && !!fromDate && !!toDate && !loading) - - async function submit() { - if (!fromDate || !toDate) return - loading = true - error = undefined - try { - await onBackfill(fromDate, toDate) - open = false - } catch (e) { - error = e instanceof Error ? e.message : String(e) - } finally { - loading = false + let preview = resource( + [() => workspace, () => assetPath, () => fromDate, () => toDate, () => open ?? false], + async ([ws, path, from, to, isOpen]) => { + if (!isOpen || !ws || !path || !from || !to) return undefined + return await AssetService.listAssetPartitionsInRange({ workspace: ws, path, from, to }) } + ) + + let worklist = $derived.by(() => { + const parts = preview.current?.partitions ?? [] + const picked = onlyMissing + ? parts.filter((p) => p.status === 'missing' || p.status === 'failed') + : parts + return picked.map((p) => p.partition) + }) + let counts = $derived.by(() => { + const c = { missing: 0, materialized: 0, failed: 0, running: 0 } + for (const p of preview.current?.partitions ?? []) c[p.status]++ + return c + }) + let canStart = $derived( + !!$enterpriseLicense && !running && !preview.loading && worklist.length > 0 + ) + + // Backend errors come back as a plain-text body (windmill_common::error). + function errText(e: unknown): string { + const err = e as { body?: unknown; message?: string } + if (typeof err?.body === 'string' && err.body) return err.body + return err?.message ?? String(e) + } + + const previewChipClass: Record = { + missing: 'bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300', + failed: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300', + materialized: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300', + running: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300' + } + const sliceChipClass: Record = { + pending: 'bg-surface-secondary text-tertiary', + running: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300', + success: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300', + failure: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300' } - open ?? false, (v) => (open = v)} title={`Backfill ${assetPath}`}> + open ?? false, (v) => (open = v)} + title={`Backfill ${assetPath}`} + cancelText="Close" +>
{#if !$enterpriseLicense}

Partition backfill is an enterprise feature. Materializing a single partition is available in the open-source edition; reprocessing a historical range requires an enterprise license.

+ {:else if slices?.length} + +

+ {#if running && cancelRequested} + Cancelling — waiting for the current partition to finish; the rest will not run. + {:else if running} + Materializing {slices.filter((s) => s.status === 'success' || s.status === 'failure') + .length}/{slices.length} partitions sequentially — each run gets its partition as an explicit + arg. + {:else} + Backfill finished: {slices.filter((s) => s.status === 'success').length} succeeded, + {slices.filter((s) => s.status === 'failure').length} failed, + {slices.filter((s) => s.status === 'pending').length} not run. + {/if} +

+
+ {#each slices as s (s.partition)} +
+ + {s.status} + + {s.partition} + {#if s.status === 'running'} + + {/if} + {#if s.error} + {s.error} + {/if} +
+ {/each} +
{:else}

- Re-runs the materialization for each partition in the range. Re-running a partition is - idempotent, so this is safe to repeat. + Re-runs the producing script once per partition in the range, each with an explicit + partition arg. Re-running a partition is idempotent, so this is + safe to repeat.

-
+
From - +
To - +
- {#if error} -

{error}

+ {#if preview.loading} +
+ Computing partitions in range… +
+ {:else if preview.error} +

{errText(preview.error)}

+ {:else if preview.current} +
+

+ {preview.current.partitions.length} + {preview.current.partition_kind} partitions in range — {counts.missing} missing, + {counts.failed} failed, {counts.materialized} materialized. Producer: + {preview.current.producer_path} +

+
+ {#each preview.current.partitions as p (p.partition)} + + {p.partition} + + {/each} +
+ +
{/if} {/if}
{#snippet actions()} - - + {#if running} + + {:else if slices?.length} + + + {:else} + + {/if} {/snippet} diff --git a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte index 4b7226e753..4be0bb8b6c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte @@ -1,10 +1,12 @@
Materialized partitions -
- +
+ {#if backfillRunning} + + {:else if backfillSlices?.length} + + {/if} +
+ +
@@ -122,4 +196,14 @@
- + (backfillSlices = undefined)} +/> diff --git a/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts b/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts new file mode 100644 index 0000000000..69fc6f9728 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest' +import { runBackfill, type BackfillSliceState } from './backfillRun' + +// Deterministic fake backend: launch resolves with `job:`, +// waitTerminal resolves per the `results` table (default success), recording +// launch order. +function fakeRunner(results: Record = {}) { + const launched: string[] = [] + return { + launched, + launch: async (partition: string) => { + launched.push(partition) + return `job:${partition}` + }, + waitTerminal: async (jobId: string) => results[jobId.slice(4)] ?? ('success' as const) + } +} + +describe('runBackfill', () => { + it('runs slices sequentially in order and reports ok', async () => { + const r = fakeRunner() + const res = await runBackfill({ + partitions: ['2026-06-26', '2026-06-27', '2026-06-29'], + launch: r.launch, + waitTerminal: r.waitTerminal + }) + expect(r.launched).toEqual(['2026-06-26', '2026-06-27', '2026-06-29']) + expect(res.ok).toBe(true) + expect(res.cancelled).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'success', 'success']) + expect(res.slices.map((s) => s.jobId)).toEqual([ + 'job:2026-06-26', + 'job:2026-06-27', + 'job:2026-06-29' + ]) + }) + + it('continues past a failed slice — each slice is independent', async () => { + const r = fakeRunner({ '2026-06-27': 'failure' }) + const res = await runBackfill({ + partitions: ['2026-06-26', '2026-06-27', '2026-06-29'], + launch: r.launch, + waitTerminal: r.waitTerminal + }) + expect(r.launched).toEqual(['2026-06-26', '2026-06-27', '2026-06-29']) + expect(res.ok).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'failure', 'success']) + }) + + it('records a launch error as slice failure and keeps going', async () => { + const r = fakeRunner() + const res = await runBackfill({ + partitions: ['a', 'b'], + launch: async (p) => { + if (p === 'a') throw new Error('boom') + return r.launch(p) + }, + waitTerminal: r.waitTerminal + }) + expect(res.slices[0]).toMatchObject({ status: 'failure', error: 'boom' }) + expect(res.slices[1].status).toBe('success') + }) + + it('stops before the next launch when cancelled, leaving the rest pending', async () => { + const r = fakeRunner() + let done = 0 + const res = await runBackfill({ + partitions: ['a', 'b', 'c'], + launch: r.launch, + waitTerminal: async (id) => { + done++ + return r.waitTerminal(id) + }, + isCancelled: () => done >= 1 + }) + expect(r.launched).toEqual(['a']) + expect(res.cancelled).toBe(true) + expect(res.ok).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'pending', 'pending']) + }) + + it('cancels a job whose launch raced the cancellation', async () => { + let cancelled = false + const cancelledJobs: string[] = [] + const res = await runBackfill({ + partitions: ['a', 'b'], + launch: async (p) => { + // The user clicks cancel while the launch request is in flight — + // there is no job id to cancel yet. + cancelled = true + return `job:${p}` + }, + waitTerminal: async () => 'failure', + isCancelled: () => cancelled, + cancelJob: async (id) => { + cancelledJobs.push(id) + } + }) + expect(cancelledJobs).toEqual(['job:a']) + expect(res.cancelled).toBe(true) + expect(res.slices.map((s) => s.status)).toEqual(['failure', 'pending']) + }) + + it('emits a snapshot per transition, never mutating earlier snapshots', async () => { + const r = fakeRunner() + const snapshots: BackfillSliceState[][] = [] + await runBackfill({ + partitions: ['a'], + launch: r.launch, + waitTerminal: r.waitTerminal, + onUpdate: (s) => snapshots.push(s) + }) + // initial pending, running, running+jobId, terminal + expect(snapshots.map((s) => s[0].status)).toEqual(['pending', 'running', 'running', 'success']) + expect(snapshots[1][0].jobId).toBeUndefined() + expect(snapshots[2][0].jobId).toBe('job:a') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts b/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts new file mode 100644 index 0000000000..51fbc9664b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts @@ -0,0 +1,83 @@ +// Client-side orchestration of a partition-range backfill (enterprise): one +// deployed run of the producing script per slice, launched with an explicit +// `partition` arg (the worker only resolves a partition when the arg is +// absent, so the caller-provided value wins). Slices run sequentially — +// concurrent materializations of the same ducklake table would contend on +// the catalog commit — and a failed slice does not stop the rest: each slice +// is independent, and the missing/failed set is simply the next worklist. +// +// Pure module (no Svelte runes) so the loop is unit-testable; reactive +// progress is delivered via `onUpdate` snapshots, mirroring +// `cascadeOrchestrator.ts`. + +export type BackfillSliceStatus = 'pending' | 'running' | 'success' | 'failure' + +export type BackfillSliceState = { + partition: string + status: BackfillSliceStatus + jobId?: string + error?: string +} + +export type BackfillRunOptions = { + /** Partition values to materialize, in run order. */ + partitions: string[] + /** Launch one run of the producer with the given partition arg; returns the job id. */ + launch: (partition: string) => Promise + /** Resolve once the job reaches a terminal state. */ + waitTerminal: (jobId: string) => Promise<'success' | 'failure'> + /** Snapshot of all slice states, emitted on every transition. */ + onUpdate?: (slices: BackfillSliceState[]) => void + /** Checked before each launch; a true stop leaves the remaining slices 'pending'. */ + isCancelled?: () => boolean + /** + * Cancel a job whose launch raced the cancellation — the cancel click had + * no job id to act on yet, so the loop cancels it as soon as the id + * arrives. Must not throw (the job may already be terminal). + */ + cancelJob?: (jobId: string) => Promise +} + +export type BackfillRunResult = { + /** True when every slice ran and succeeded. */ + ok: boolean + /** True when the loop stopped early on `isCancelled`. */ + cancelled: boolean + slices: BackfillSliceState[] +} + +export async function runBackfill(opts: BackfillRunOptions): Promise { + const { partitions, launch, waitTerminal, onUpdate, isCancelled, cancelJob } = opts + const slices: BackfillSliceState[] = partitions.map((partition) => ({ + partition, + status: 'pending' + })) + const emit = () => onUpdate?.(slices.map((s) => ({ ...s }))) + emit() + let cancelled = false + for (const slice of slices) { + if (isCancelled?.()) { + cancelled = true + break + } + slice.status = 'running' + emit() + try { + slice.jobId = await launch(slice.partition) + emit() + if (isCancelled?.() && cancelJob) { + await cancelJob(slice.jobId) + } + slice.status = await waitTerminal(slice.jobId) + } catch (e) { + slice.status = 'failure' + slice.error = e instanceof Error ? e.message : String(e) + } + emit() + } + return { + ok: !cancelled && slices.every((s) => s.status === 'success'), + cancelled, + slices + } +} From 5745dfc6ea69dcc15dbf70d548c706a8f2974d2a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 2 Jul 2026 12:51:17 +0200 Subject: [PATCH 164/273] smooth local pipeline dogfooding (#9888) --- AGENTS.md | 1 + backend/CLAUDE.md | 5 +++ cli/src/commands/pipeline/docs.ts | 25 ++++++++++---- cli/src/commands/pipeline/localGraph.ts | 38 +++++++++++++++++++++- cli/test/pipeline_local_graph_unit.test.ts | 23 ++++++++++++- 5 files changed, 83 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index aead237e59..ac00741c49 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ Open-source platform for internal tools, workflows, API integrations, background ## Dev Environment - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) +- **DuckDB local jobs**: before running DuckDB scripts locally, build the FFI shared library with `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. Re-run it after clean builds or when `backend/target/debug/libwindmill_duckdb_ffi_internal.*` is missing. - **Frontend**: `REMOTE=http://localhost:8000 npm run dev` from `frontend/` (port 3000+) - **DB**: `psql postgres://postgres:changeme@localhost:5432/windmill` - **Login**: `admin@windmill.dev` / `changeme` diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 9ff2c0be90..086e251962 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -6,3 +6,8 @@ - **DB schema**: `backend/summarized_schema.txt` - **API routes entry point**: `windmill-api/src/lib.rs` - **OpenAPI spec**: `windmill-api/openapi.yaml` +- **DuckDB local jobs**: build the dynamic FFI library before running DuckDB scripts locally: + ```bash + cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh + ``` + Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts index 57fb70618a..b360c8b802 100644 --- a/cli/src/commands/pipeline/docs.ts +++ b/cli/src/commands/pipeline/docs.ts @@ -157,8 +157,6 @@ export async function generatePipelineDocs( opts: GlobalOptions & { local?: boolean; defaultTs?: "bun" | "deno" }, folder: string, ) { - const workspace = await resolveWorkspace(opts); - await requireLogin(opts); const f = folder.replace(/^f\//, "").replace(/\/$/, ""); // `docs` WRITES PIPELINE.md / AGENTS.md / CLAUDE.md under `f/`; a `..` // segment would escape the folder and clobber files elsewhere in the tree. @@ -170,9 +168,12 @@ export async function generatePipelineDocs( // defaultTs (from wmill.yaml) drives .ts → bun/deno inference for the local graph. const merged = await mergeConfigWithConfigFile(opts); + const workspace = opts.local ? undefined : await resolveWorkspace(opts); + if (!opts.local) await requireLogin(opts); + const graph = opts.local ? (await buildLocalPipelineGraph({ root, folder: f, defaultTs: merged.defaultTs })).graph - : await fetchDeployedGraph(workspace.workspaceId, f); + : await fetchDeployedGraph(workspace!.workspaceId, f); if (graph.runnables.length === 0) { // The deployed graph is empty — but a user/agent in a working tree may have @@ -198,10 +199,20 @@ export async function generatePipelineDocs( } let datatableSchemas: any[] = []; - try { - datatableSchemas = await wmill.listDataTableSchemas({ workspace: workspace.workspaceId }); - } catch (err: any) { - log.warn(colors.yellow(`Could not fetch datatable schemas: ${err.message}`)); + const hasExplicitWorkspace = + !!opts.workspace || + (!!opts.baseUrl && !!opts.token) || + (!!process.env["WM_WORKSPACE"] && + !!process.env["WM_TOKEN"] && + !!(process.env["BASE_INTERNAL_URL"] ?? process.env["BASE_URL"])); + if (!opts.local || hasExplicitWorkspace) { + try { + const schemaWorkspace = workspace ?? await resolveWorkspace(opts); + if (opts.local) await requireLogin(opts); + datatableSchemas = await wmill.listDataTableSchemas({ workspace: schemaWorkspace.workspaceId }); + } catch (err: any) { + log.warn(colors.yellow(`Could not fetch datatable schemas: ${err.message}`)); + } } const md = generatePipelineMarkdown(f, graph, datatableSchemas, !!opts.local); diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index ce556d1d8e..9d40a0e4b7 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -241,6 +241,30 @@ function fallbackParse(content: string, language: string): ParseAssetsRaw { return out; } +function recoverHeaderNativeTriggers(content: string, language: string): string[] { + const raw = commentPrefix(language); + const p = raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const out: string[] = []; + const seen = new Set(); + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (trimmed === "") continue; + if (!trimmed.startsWith(raw)) break; + const marker = line.match(new RegExp(`^\\s*${p}\\s*on\\s+(\\S+)\\s*$`)); + if (!marker) continue; + const kind = marker[1]; + if (!NATIVE_KINDS.has(kind) || seen.has(kind)) continue; + seen.add(kind); + out.push(kind); + } + return out; +} + +function normalizeRetry(retry: ParseAssetsRaw["retry"]): ParseAssetsRaw["retry"] { + if (!retry?.delay) return retry; + return { ...retry, delay: retry.delay.replace(/^delay=/, "") }; +} + // Comment prefix for `volume:` annotations. Deliberately NOT `commentPrefix` // above (which returns `--` for SQL): volume annotations are only recognized for // the languages the backend/frontend recognize them for — mirrors @@ -398,6 +422,8 @@ export async function buildLocalPipelineGraph(args: { for (const s of all) { const out = await inferScriptAssets(s.content, s.language); if (!out.in_pipeline) continue; // not a pipeline member + const retry = normalizeRetry(out.retry); + const nativeTriggers = recoverHeaderNativeTriggers(s.content, s.language); // Carry the parsed `// tag` so previews route to the same worker the // deployed pipeline would (both `pipeline run --local` and `/pipeline_dev`). pipelineScripts.push(out.tag ? { ...s, tag: out.tag } : s); @@ -420,7 +446,7 @@ export async function buildLocalPipelineGraph(args: { ...(out.partition ? { partition_kind: out.partition.kind } : {}), ...(out.freshness ? { freshness: out.freshness.duration } : {}), ...(out.tag ? { tag: out.tag } : {}), - ...(out.retry ? { retry: out.retry } : {}), + ...(retry ? { retry } : {}), ...(out.data_tests && out.data_tests.length > 0 ? { data_tests: out.data_tests } : {}), ...(out.column_lineage && out.column_lineage.length > 0 ? { column_lineage: out.column_lineage } @@ -466,6 +492,7 @@ export async function buildLocalPipelineGraph(args: { }); } } + const existingNativeTriggers = new Set(); for (const t of out.triggers ?? []) { if (t.kind === "asset") { const at = t as { kind: "asset"; asset_kind: string; path: string }; @@ -481,6 +508,7 @@ export async function buildLocalPipelineGraph(args: { runnable_path: s.path, }); } else { + existingNativeTriggers.add(t.kind); triggers.push({ trigger_kind: t.kind, runnable_kind: "script", @@ -488,6 +516,14 @@ export async function buildLocalPipelineGraph(args: { }); } } + for (const kind of nativeTriggers) { + if (existingNativeTriggers.has(kind)) continue; + triggers.push({ + trigger_kind: kind, + runnable_kind: "script", + runnable_path: s.path, + }); + } } return { diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index 6b192cb317..1368e14504 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -67,11 +67,32 @@ test("native triggers surface as trigger rows (no deploy needed)", async () => { { "ingest.bun.ts": `// pipeline\n// on data_upload\nimport * as wmill from "windmill-client"\nexport async function main() {}\n`, + "upload_trigger.duckdb.sql": `-- pipeline\n-- on data_upload\nSELECT 1;\n`, }, async (root, folder) => { const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); const native = graph.triggers.filter((t) => t.trigger_kind !== "asset"); - expect(native.map((t) => t.trigger_kind)).toContain("data_upload"); + expect( + native + .filter((t) => t.trigger_kind === "data_upload") + .map((t) => t.runnable_path) + .sort(), + ).toEqual(["f/mypipe/ingest", "f/mypipe/upload_trigger"]); + }, + ); +}); + +test("retry delay metadata strips the optional `delay=` prefix", async () => { + await withFolder( + { + "retry.duckdb.sql": `-- pipeline\n-- retry 2 delay=10s\nSELECT 1;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.runnables.find((r) => r.path === "f/mypipe/retry")?.retry).toEqual({ + count: 2, + delay: "10s", + }); }, ); }); From 789450731b0a3c8dffa336f7bfc3f3de528c09fb Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 2 Jul 2026 12:51:36 +0200 Subject: [PATCH 165/273] fix(triggers): retry transient websocket connect failures before disabling (#9887) A websocket trigger is permanently disabled (with a critical alert) when a single connect attempt in get_consumer fails. Gateway endpoints fronted by edge proxies (e.g. Discord behind Cloudflare) sporadically answer the upgrade handshake with a transient 502/520, so a long-lived trigger that reconnects frequently eventually catches one and dies until a human re-enables it. Retry transient failures (HTTP 5xx/429 handshake responses and IO errors) up to 5 attempts with exponential backoff before surfacing the error, and report retry progress through the trigger's error status. Permanent-looking errors (bad URL, other 4xx, protocol/TLS mismatch) still disable immediately. Co-authored-by: Claude Fable 5 --- .../src/listener.rs | 71 +++++++- .../tests/connect_retry.rs | 153 ++++++++++++++++++ 2 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 backend/windmill-trigger-websocket/tests/connect_retry.rs diff --git a/backend/windmill-trigger-websocket/src/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs index d46faf1b1a..e34fe6e4f6 100644 --- a/backend/windmill-trigger-websocket/src/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -11,7 +11,7 @@ use serde::Deserialize; use serde_json::value::RawValue; use std::{borrow::Cow, collections::HashMap, sync::Arc}; use tokio::{net::TcpStream, sync::RwLock}; -use tokio_tungstenite::{tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{tungstenite, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use windmill_common::{ error::{to_anyhow, Error, Result}, jobs::JobTriggerKind, @@ -22,13 +22,30 @@ use windmill_common::{ }; use windmill_queue::PushArgsOwned; use windmill_trigger::filter::{check_filters, Filter}; -use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::listener::{update_rw_lock, ListeningTrigger}; use windmill_trigger::trigger_helpers::{ trigger_runnable, trigger_runnable_and_wait_for_raw_result, trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, }; use windmill_trigger::Listener; +const MAX_CONNECT_ATTEMPTS: u32 = 5; + +/// Whether a failed WebSocket connect is worth retrying: network-level IO +/// errors and HTTP 5xx/429 handshake responses are typically transient (edge +/// proxies like Cloudflare return 502/520 sporadically), while other errors +/// (bad URL, protocol mismatch, other 4xx) point at configuration and would +/// fail identically on every attempt. +fn is_transient_connect_error(err: &tungstenite::Error) -> bool { + match err { + tungstenite::Error::Io(_) => true, + tungstenite::Error::Http(resp) => { + resp.status().is_server_error() || resp.status().as_u16() == 429 + } + _ => false, + } +} + async fn send_initial_messages( listening_trigger: &ListeningTrigger, writer: &mut SplitSink>, Message>, @@ -176,12 +193,50 @@ impl Listener for WebsocketTrigger { validate_websocket_url_for_ssrf(&connect_url).await?; - let connection = connect_async_with_proxy(&*connect_url) - .await - .map(|conn| Some(conn)) - .map_err(|err| to_anyhow(err).into()); - - connection + // Gateway endpoints are often fronted by an edge proxy (e.g. Cloudflare) + // that sporadically answers the upgrade request with a transient 5xx + // instead of `101 Switching Protocols`, and a `get_consumer` error + // disables the trigger until a human re-enables it — so retry transient + // failures with backoff before giving up. The caller runs `loop_ping` + // concurrently so `last_server_ping` stays alive across the sleeps, and + // killpill cancels this future between awaits. + let mut attempt = 0; + loop { + attempt += 1; + match connect_async_with_proxy(&*connect_url).await { + Ok(conn) => return Ok(Some(conn)), + // Only retry in trigger mode: a failed connect there disables the + // trigger until a human re-enables it, while capture mode is an + // interactive test where instant feedback beats resilience (an + // `Io` error can also be a permanent misconfiguration, e.g. a + // typo'd host, which should surface immediately when iterating). + Err(err) + if listening_trigger.trigger_mode + && attempt < MAX_CONNECT_ATTEMPTS + && is_transient_connect_error(&err) => + { + let delay_secs = 1u64 << attempt; + tracing::warn!( + "Transient error connecting to WebSocket for trigger {} (attempt {}/{}), retrying in {}s: {}", + listening_trigger.path, + attempt, + MAX_CONNECT_ATTEMPTS, + delay_secs, + err + ); + update_rw_lock( + err_message.clone(), + Some(format!( + "Connection attempt {}/{} failed ({}), retrying in {}s...", + attempt, MAX_CONNECT_ATTEMPTS, err, delay_secs + )), + ) + .await; + tokio::time::sleep(std::time::Duration::from_secs(delay_secs)).await; + } + Err(err) => return Err(to_anyhow(err).into()), + } + } } async fn consume( &self, diff --git a/backend/windmill-trigger-websocket/tests/connect_retry.rs b/backend/windmill-trigger-websocket/tests/connect_retry.rs new file mode 100644 index 0000000000..2d21512692 --- /dev/null +++ b/backend/windmill-trigger-websocket/tests/connect_retry.rs @@ -0,0 +1,153 @@ +//! End-to-end tests for the websocket trigger connect-retry behavior: a mock +//! TCP server rejects the websocket upgrade with a configurable HTTP status a +//! number of times before completing a real handshake, and the tests assert +//! which failures `get_consumer` retries. +//! +//! This lives in an integration-test binary (own process) because it sets +//! ALLOW_PRIVATE_WEBSOCKET_URLS — the mock server listens on 127.0.0.1, which +//! the SSRF check blocks — and that process-global env var must not leak into +//! the crate's unit tests, which assert loopback URLs are rejected. + +use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, +}; + +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::TcpListener, + sync::{broadcast, RwLock}, +}; +use windmill_trigger::{listener::ListeningTrigger, Listener}; +use windmill_trigger_websocket::{ + WebsocketConfig, WebsocketTrigger, ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, +}; + +/// Mock server: rejects the first `failures` upgrade requests with +/// `status_line` and closes, then completes real websocket handshakes and +/// parks the connection open. Returns the bound address and the +/// connection-attempt counter. +async fn mock_ws_server( + failures: u32, + status_line: &'static str, +) -> (std::net::SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let attempts = Arc::new(AtomicU32::new(0)); + let served = attempts.clone(); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let n = served.fetch_add(1, Ordering::SeqCst); + if n < failures { + // Drain the request head, then reject the upgrade. + let mut reader = BufReader::new(&mut socket); + let mut line = String::new(); + loop { + line.clear(); + let read = reader.read_line(&mut line).await.unwrap_or(0); + if read == 0 || line == "\r\n" { + break; + } + } + socket + .write_all( + format!("HTTP/1.1 {status_line}\r\nContent-Length: 0\r\n\r\n").as_bytes(), + ) + .await + .ok(); + } else if let Ok(ws) = tokio_tungstenite::accept_async(socket).await { + tokio::spawn(async move { + let _open = ws; + std::future::pending::<()>().await + }); + } + } + }); + (addr, attempts) +} + +fn trigger(url: String, trigger_mode: bool) -> ListeningTrigger { + ListeningTrigger { + path: "f/test/ws".to_string(), + is_flow: false, + workspace_id: "test".to_string(), + edited_by: "test".to_string(), + permissioned_as: "u/test".to_string(), + trigger_config: WebsocketConfig { + url, + filters: vec![], + filter_logic: "and".to_string(), + initial_messages: None, + url_runnable_args: None, + can_return_message: false, + can_return_error_result: false, + heartbeat: None, + }, + script_path: "f/test/script".to_string(), + trigger_mode, + error_handling: None, + suspended_mode: false, + } +} + +async fn get_consumer_result( + lt: &ListeningTrigger, + err_message: Arc>>, +) -> windmill_common::error::Result::Consumer>> { + // The static-URL path of `get_consumer` never touches the DB; a lazy pool + // satisfies the signature without a running postgres. + let db: windmill_common::DB = + sqlx::Pool::connect_lazy("postgres://unused:unused@127.0.0.1:1/unused").unwrap(); + let (_killpill_tx, killpill_rx) = broadcast::channel::<()>(1); + WebsocketTrigger + .get_consumer(&db, lt, err_message, killpill_rx) + .await +} + +#[tokio::test] +async fn transient_502s_are_retried_until_the_upgrade_succeeds() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(2, "502 Bad Gateway").await; + let lt = trigger(format!("ws://{addr}"), true); + let err_message = Arc::new(RwLock::new(None)); + + let consumer = get_consumer_result(<, err_message.clone()) + .await + .expect("connect should succeed after retries"); + + assert!(consumer.is_some(), "expected an established connection"); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + // Retry progress was reported through the shared status lock. + let status = err_message.read().await.clone().unwrap(); + assert!(status.contains("attempt 2/5"), "got status: {status}"); + assert!(status.contains("502"), "got status: {status}"); +} + +#[tokio::test] +async fn non_transient_http_errors_fail_on_the_first_attempt() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(u32::MAX, "404 Not Found").await; + let lt = trigger(format!("ws://{addr}"), true); + + let err = get_consumer_result(<, Arc::new(RwLock::new(None))) + .await + .expect_err("a 404 upgrade response should not be retried"); + + assert!(err.to_string().contains("404"), "got error: {err}"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn capture_mode_fails_fast_even_on_transient_errors() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(u32::MAX, "502 Bad Gateway").await; + let lt = trigger(format!("ws://{addr}"), false); + + let err = get_consumer_result(<, Arc::new(RwLock::new(None))) + .await + .expect_err("capture mode should surface the first failure immediately"); + + assert!(err.to_string().contains("502"), "got error: {err}"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); +} From 5af91a677cad88faccba702e3556fc4fb7b6e640 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 2 Jul 2026 16:32:01 +0200 Subject: [PATCH 166/273] fix: stale AI chat context picker after workspace item changes (#9893) * fix: revalidate workspace items cache on context picker open The chat context picker (and workspace drill pickers) read from a module-level cache that was only invalidated from two editor save paths, so items created or deleted anywhere else stayed stale until a full page reload. Make the loader do real stale-while-revalidate: cached items render instantly and each workspace+kind is re-fetched once per picker mount, keeping the state reference stable when nothing changed. Co-Authored-By: Claude Fable 5 * fix: retry failed picker revalidation and handle its rejection Mark a workspace+kind as revalidated only after the fetch succeeds, so a failed revalidation is retried on the next ensureLoaded call instead of stranding stale data for the rest of the mount. Catch the rejection (callers fire-and-forget) and log it. Also dedupe the stale-while-revalidate rationale to its canonical comment site in workspacePicker.ts. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../components/workspaceItemsLoader.svelte.ts | 40 ++++++++++++++++--- .../src/lib/components/workspacePicker.ts | 14 ++++--- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/workspaceItemsLoader.svelte.ts b/frontend/src/lib/components/workspaceItemsLoader.svelte.ts index 45d64bd01b..4ac65f805b 100644 --- a/frontend/src/lib/components/workspaceItemsLoader.svelte.ts +++ b/frontend/src/lib/components/workspaceItemsLoader.svelte.ts @@ -21,13 +21,23 @@ import { * Both getters are read inside the returned closures so changing * workspace or kinds after mount Just Works. */ +function itemsEqual(a: WorkspaceItem[] | undefined, b: WorkspaceItem[]): boolean { + if (!a || a.length !== b.length) return false + return a.every( + (item, i) => + item.path === b[i].path && + item.summary === b[i].summary && + item.kind === b[i].kind && + item.raw_app === b[i].raw_app + ) +} + export function useWorkspaceItemsLoader( workspace: () => string | undefined, kinds: () => readonly WorkspaceItemKind[] ) { // Seed from the module-level cache so kinds already fetched in this - // session render on the first frame. Re-fetching `ensureLoaded` later - // quietly swaps in fresh data (stale-while-revalidate). + // session render on the first frame. let loaded = $state>>( (() => { const ws = untrack(workspace) @@ -41,18 +51,38 @@ export function useWorkspaceItemsLoader( })() ) let loadingKind = $state>>({}) + // Workspace+kind pairs this loader instance has refreshed — at most one + // background re-fetch per kind per mount. Marked on success only, so a + // failed fetch is retried by the next call instead of stranding stale data. + const revalidated = new Set() async function ensureLoaded(kind: WorkspaceItemKind) { const ws = workspace() if (!ws) return + const revalidateKey = `${ws}:${kind}` // `loaded[kind]` read inside `untrack` so callers wiring this into // a reactive context (DrillPicker's onFilterChange effect) don't // subscribe to a signal `ensureLoaded` itself writes — that would // re-fire the effect on every assignment and busy-loop. - if (!untrack(() => loaded[kind])) loadingKind[kind] = true + const hasCached = !!untrack(() => loaded[kind]) + if (hasCached && revalidated.has(revalidateKey)) return + if (!hasCached) loadingKind[kind] = true try { - const items = await loadKind(ws, kind) - loaded[kind] = items + const items = await loadKind(ws, kind, { revalidate: true }) + revalidated.add(revalidateKey) + // Keep the reference stable when nothing changed so an open picker's + // derived tree isn't rebuilt under the user on every revalidation. + if ( + !itemsEqual( + untrack(() => loaded[kind]), + items + ) + ) + loaded[kind] = items + } catch (e) { + // Callers fire-and-forget; surface the failure without an + // unhandled rejection. Cached items (if any) keep rendering. + console.error(`Failed to load workspace ${kind}s`, e) } finally { loadingKind[kind] = false } diff --git a/frontend/src/lib/components/workspacePicker.ts b/frontend/src/lib/components/workspacePicker.ts index ba2985ecbb..92ea48af20 100644 --- a/frontend/src/lib/components/workspacePicker.ts +++ b/frontend/src/lib/components/workspacePicker.ts @@ -44,9 +44,10 @@ type WorkspaceCache = { } /** Module-level session cache. Persists across picker mounts within a single - * page session. NOT invalidated automatically — call `invalidate()` after - * creating/deleting an item if the picker may be opened again before a full - * reload. */ + * page session so cached kinds render on the first frame. Pickers revalidate + * once per mount via `loadKind(..., { revalidate: true })`, so stale entries + * self-heal on the next open; call `invalidate()` after creating/deleting an + * item when even a brief flash of the stale list must be avoided. */ const cache = new Map() const inflight = new Map>() /** Bumped by `invalidate()`. Each in-flight `loadKind` captures the version @@ -87,11 +88,14 @@ export function invalidate(workspace: string, kind?: WorkspaceItemKind) { export async function loadKind( workspace: string, - kind: WorkspaceItemKind + kind: WorkspaceItemKind, + opts?: { revalidate?: boolean } ): Promise { const existing = cache.get(workspace)?.[kind] - if (existing) return existing + if (existing && !opts?.revalidate) return existing const key = cacheKey(workspace, kind) + // An in-flight fetch is already hitting the network, so it satisfies a + // revalidate request too. const flying = inflight.get(key) if (flying) return flying From 37bb57474e8336823bb31527f2a708ef41cd39c4 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 2 Jul 2026 23:18:49 +0200 Subject: [PATCH 167/273] fix: make SMTP username and password optional in frontend validation (#9895) --- .../src/lib/components/instanceSettings/SmtpSettings.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte b/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte index c297b1a32f..f7607c5d6c 100644 --- a/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte @@ -5,10 +5,6 @@ smtpSettings.smtp_host && smtpSettings.smtp_host.trim() !== '' && smtpSettings.smtp_port && - smtpSettings.smtp_username && - smtpSettings.smtp_username.trim() !== '' && - smtpSettings.smtp_password && - smtpSettings.smtp_password.trim() !== '' && smtpSettings.smtp_from && smtpSettings.smtp_from.trim() !== '' ) From 186ac4933b79aed57fce23ebcf3b525fcfd1c474 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 2 Jul 2026 23:30:28 +0200 Subject: [PATCH 168/273] fix(frontend): home New submenus fall back below, hugging the right edge (#9894) * fix(frontend): home New submenus fall back below hugging the right edge Co-Authored-By: Claude Fable 5 * fix(frontend): re-hug submenu on window resize even without a melt reposition Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../components/home/CreateActionsMenu.svelte | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte index 146e390419..4d8adb88e9 100644 --- a/frontend/src/lib/components/home/CreateActionsMenu.svelte +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -243,7 +243,9 @@ forceVisible: true }) - // per-row fan-out submenus, anchored toward the page center (popover hugs the right edge) + // per-row fan-out submenus: open to the right when there's room. The popover hugs + // the right viewport edge on most screens, so the default flip (right → left) would + // cover the doc panel — fall back below the trigger row instead. const { elements: { subTrigger: wacSubTrigger, subMenu: wacSubMenu }, states: { subOpen: wacSubOpen } @@ -251,7 +253,7 @@ positioning: { placement: 'right-start', gutter: 4, - flip: true, + flip: { fallbackPlacements: ['bottom-end', 'bottom-start', 'top-end', 'top-start'] }, fitViewport: true, overflowPadding: 8 } @@ -263,12 +265,43 @@ positioning: { placement: 'right-end', gutter: 4, - flip: true, + flip: { fallbackPlacements: ['bottom-end', 'bottom-start', 'top-end', 'top-start'] }, fitViewport: true, overflowPadding: 8 } }) + // When a fan-out submenu falls below its row ('bottom'/'top' data-side), melt aligns + // it to the row's right edge, but free viewport space may remain beside the popover — + // slide it right until it hugs the viewport edge. Melt repositions on scroll/resize + // via style/data-side writes, so re-apply on attribute mutations. + function hugViewportRight(node: HTMLElement) { + let delta = 0 + const apply = () => { + const side = node.getAttribute('data-side') + const baseRight = node.getBoundingClientRect().right - delta + const next = + side === 'bottom' || side === 'top' + ? Math.max(0, document.documentElement.clientWidth - 8 - baseRight) + : 0 + if (Math.abs(next - delta) < 0.5) return + delta = next + node.style.transform = next ? `translateX(${next}px)` : '' + } + const observer = new MutationObserver(apply) + observer.observe(node, { attributeFilter: ['style', 'data-side'] }) + // melt only rewrites style when the reposition moves the submenu — a resize that + // changes clientWidth without moving it (scrollbar appearing, zoom) needs a direct hook + window.addEventListener('resize', apply) + apply() + return { + destroy: () => { + observer.disconnect() + window.removeEventListener('resize', apply) + } + } + } + // attach the menu trigger to the design-system
- - - -
- {#if currentName} -

Current name
{currentName}

- {/if} - -
- - {#snippet actions()} - {/snippet} - + + From d15033cde6a474b548ebbaf18ff02223fc21f701 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 2 Jul 2026 21:57:26 +0000 Subject: [PATCH 171/273] fix: invalidate bun bundle cache on transitive relative-import changes (#9891) * fix: invalidate bun bundle cache on transitive relative-import changes Co-Authored-By: Claude Fable 5 * fix: do not memoize transient fetch errors in bundle-key import cache Co-Authored-By: Claude Fable 5 * fix: use regular comment on lazy_static block (deny unused_doc_comments) Co-Authored-By: Claude Fable 5 * fix: align bundle-key import version selection with loader content endpoint Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- ...44cfa928469ce3c1a53ec273ecc997f6e61f8.json | 23 +++ ...1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json | 23 --- backend/src/main.rs | 4 + backend/tests/bun_jobs.rs | 121 ++++++++++++++++ backend/windmill-common/src/lib.rs | 44 ++++++ backend/windmill-worker/src/bun_executor.rs | 131 +++++++++++++++--- 6 files changed, 301 insertions(+), 45 deletions(-) create mode 100644 backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json delete mode 100644 backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json diff --git a/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json new file mode 100644 index 0000000000..e171d44af5 --- /dev/null +++ b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8" +} diff --git a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json b/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json deleted file mode 100644 index f65ea6d2d4..0000000000 --- a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b" -} diff --git a/backend/src/main.rs b/backend/src/main.rs index 94b8e2f4f6..0d7bef4d70 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1708,6 +1708,10 @@ async fn process_notify_event( match *source_type { "script" => { windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + // Bundle-cache key resolution for imported scripts; evicted + // together with the content-side caches below so key and + // inlined content flip to the new version in the same window. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&key); // Evict the relative-import latest-hash cache so a redeployed // imported script flips the content cache to its new version // across all replicas within a poll interval (see #6769). Keyed diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index d51e51f58b..fc766253bc 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,3 +1,4 @@ +use futures::StreamExt; use sqlx::postgres::Postgres; use sqlx::Pool; use uuid::Uuid; @@ -818,6 +819,126 @@ export function main() { Ok(()) } +// ============================================================================ +// Bundle cache invalidation on transitive relative-import change +// ============================================================================ + +async fn insert_deployed_bun_script(db: &Pool, path: &str, hash: i64, content: &str) { + // What gen_bun_lockfile stores for a script with no npm dependencies; a + // bare '' lock fails split_lockfile when the script is run directly. + const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n"; + // Runtime query to avoid touching the sqlx offline cache. + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"properties\":{},\"required\":[],\"type\":\"object\"}', '', '', $2, $3, 'bun', $4)", + ) + .bind(content) + .bind(path) + .bind(hash) + .bind(EMPTY_BUN_LOCK) + .execute(db) + .await + .unwrap(); +} + +fn run_main_script_job(hash: i64) -> RunJob { + RunJob::from(JobPayload::ScriptHash { + path: "f/stale_bundle/main_script".to_string(), + hash: windmill_common::scripts::ScriptHash(hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bun, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }) +} + +/// Editing a script that a runnable imports only TRANSITIVELY (main -> mid -> +/// leaf) must invalidate the runnable's cached bundle: the leaf's code is +/// inlined in the bundle, so the cache key has to cover the whole closure, not +/// just direct imports. +#[sqlx::test(fixtures("base"))] +async fn test_bun_transitive_import_change_rebundles(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Hashes/paths unique across this test binary: the script/hash caches are + // process-global while parallel tests each run in their own DB. + const LEAF_V1: i64 = 41230001; + const MID: i64 = 41230002; + const MAIN: i64 = 41230003; + const LEAF_V2: i64 = 41230004; + + insert_deployed_bun_script( + &db, + "f/stale_bundle/leaf", + LEAF_V1, + r#"export function leafValue() { return "V1_FROM_LEAF"; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/mid", + MID, + r#"import { leafValue } from "./leaf"; +export function midValue() { return `M(${leafValue()})`; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/main_script", + MAIN, + r#"import { midValue } from "./mid"; +export function main() { return midValue(); }"#, + ) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + in_test_worker( + &db, + async move { + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V1_FROM_LEAF)")); + + // Deploy a new version of ONLY the leaf; main_script and mid keep + // their hash, content, and lock byte-identical. + insert_deployed_bun_script( + &db2, + "f/stale_bundle/leaf", + LEAF_V2, + r#"export function leafValue() { return "V2_FROM_LEAF"; }"#, + ) + .await; + + // Tests don't run the notify_event poll loop, so replay what its + // `notify_runnable_version_change` handler (main.rs) does on deploy: + // evict the leaf's latest-hash cache entries. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&( + "test-workspace".to_string(), + "f/stale_bundle/leaf".to_string(), + )); + windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE + .remove(&format!("test-workspace:f/stale_bundle/leaf")); + + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V2_FROM_LEAF)")); + }, + port, + ) + .await; + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index aaead77925..baa3e4b500 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -268,6 +268,10 @@ lazy_static::lazy_static! { pub static ref INSTANCE_NAME: String = rd_string(5); pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + // Latest non-archived version per (workspace, path) for bundle cache keying — + // looser predicate than DEPLOYED_SCRIPT_HASH_CACHE (no lock requirement), so + // the two must not share entries. See get_latest_script_hash_for_import_cached. + pub static ref IMPORTED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref DYNAMIC_INPUT_CACHE: Cache> = Cache::new(1000); pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); @@ -1602,6 +1606,46 @@ pub async fn get_latest_script_hash<'e, E: sqlx::PgExecutor<'e>>( return Ok(hash); } +/// Latest non-archived hash for an imported `path`, for bundle cache keying. +/// MUST select the same row as the bundler's content endpoint +/// (`raw_script_by_path_internal`: `archived = false ORDER BY created_at DESC`, +/// no lock predicate) — a stricter filter here would let the key point at an +/// older version than the content that gets inlined. Cached with the same +/// freshness contract as that endpoint's `RAW_SCRIPT_LATEST_HASH_CACHE`: +/// evicted by `notify_runnable_version_change` events, 60s TTL fallback. +pub async fn get_latest_script_hash_for_import_cached( + db: &DB, + w_id: &str, + script_path: &str, +) -> error::Result> { + let use_cache = !DEPLOYED_SCRIPT_CACHE_DISABLED.load(std::sync::atomic::Ordering::Relaxed); + let cache_key = (w_id.to_string(), script_path.to_string()); + if use_cache { + if let Some(cached) = IMPORTED_SCRIPT_HASH_CACHE.get(&cache_key) { + if cached.expires_at > std::time::Instant::now() { + return Ok(Some(cached.id)); + } + } + } + let hash = sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await?; + if let (true, Some(hash)) = (use_cache, hash) { + IMPORTED_SCRIPT_HASH_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: hash, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + } + Ok(hash) +} + pub async fn get_script_info_for_hash<'e, E: sqlx::PgExecutor<'e>>( db_authed: Option>>, db: E, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1e4f4aafd6..f6b093f766 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,6 +1,11 @@ #[cfg(feature = "deno_core")] use std::time::Instant; -use std::{collections::HashMap, fs, process::Stdio}; +use std::{ + collections::{HashMap, HashSet}, + fs, + process::Stdio, + sync::Arc, +}; use base64::Engine; use itertools::Itertools; @@ -27,9 +32,10 @@ use crate::{ NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ + cache, client::AuthedClient, jobs::JobKind, - scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang}, + scripts::{id_to_codebase_info, CodebaseInfo, ScriptHash, ScriptLang}, utils::WarnAfterExt, workspace_dependencies::WorkspaceDependenciesPrefetched, }; @@ -44,7 +50,6 @@ use tokio::io::AsyncReadExt; use windmill_common::{ error::{self, Result}, - get_latest_hash_for_path, worker::{write_file, Connection, DISABLE_BUNDLING}, DB, }; @@ -1242,16 +1247,101 @@ pub fn ensure_bundle_output_exists(bundle_path: &str) -> Result<()> { pub const BUN_BUNDLE_OBJECT_STORE_PREFIX: &str = "bun_bundle/"; -async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) -> Result { - let script_hash = get_latest_hash_for_path(db, w_id, script_path, false).await?; - let last_updated_at = sqlx::query_scalar!( - "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - w_id, - script_hash.0 .0 +// A script version's relative-import list never changes (content is immutable +// per hash), so parses are memoized without any invalidation. +lazy_static::lazy_static! { + static ref RELATIVE_IMPORTS_PER_HASH: quick_cache::sync::Cache>> = + quick_cache::sync::Cache::new(1000); +} + +const MAX_TRANSITIVE_IMPORT_PATHS: usize = 256; + +/// `(path, latest hash)` for the whole transitive closure of relative imports +/// of `inner_content` — the set of scripts whose code gets inlined into the +/// bundle, so all of them must key the bundle cache. Resolution goes through +/// `IMPORTED_SCRIPT_HASH_CACHE` (notify-evicted, 60s TTL fallback, same +/// version-selection predicate as the loader's content endpoint) and the +/// per-hash script/parse caches, so steady state costs no DB queries. +async fn collect_transitive_import_versions( + db: &DB, + w_id: &str, + script_path: &str, + inner_content: &str, +) -> Vec<(String, i64)> { + let conn = Connection::from(db.clone()); + let mut queue = crate::worker_lockfiles::extract_relative_imports( + inner_content, + script_path, + &Some(ScriptLang::Bun), ) - .fetch_one(db) - .await?; - Ok(last_updated_at.to_string()) + .unwrap_or_default(); + let mut visited: HashSet = HashSet::new(); + let mut versions: Vec<(String, i64)> = vec![]; + while let Some(path) = queue.pop() { + if !visited.insert(path.clone()) { + continue; + } + if visited.len() > MAX_TRANSITIVE_IMPORT_PATHS { + tracing::warn!( + "transitive relative-import closure of {script_path} exceeds \ + {MAX_TRANSITIVE_IMPORT_PATHS} scripts; bundle cache key covers only the first \ + {MAX_TRANSITIVE_IMPORT_PATHS}" + ); + break; + } + let hash = match windmill_common::get_latest_script_hash_for_import_cached(db, w_id, &path) + .await + { + Ok(Some(hash)) => hash, + // Not a deployed script at this path (deleted, or not a script): + // excluded from the key, matching what the bundler can inline. + Ok(None) => continue, + Err(e) => { + tracing::warn!( + "could not resolve import {path} while computing bundle cache key for \ + {script_path}: {e:#}" + ); + continue; + } + }; + versions.push((path.clone(), hash)); + let imports = match RELATIVE_IMPORTS_PER_HASH.get(&hash) { + Some(imports) => imports, + None => match cache::script::fetch(&conn, ScriptHash(hash)).await { + Ok((data, meta)) => { + let imports = Arc::new(match meta.language { + Some(ScriptLang::Bun) + | Some(ScriptLang::Bunnative) + | Some(ScriptLang::Deno) => { + crate::worker_lockfiles::extract_relative_imports( + &data.code, + &path, + &meta.language, + ) + .unwrap_or_default() + } + _ => vec![], + }); + RELATIVE_IMPORTS_PER_HASH.insert(hash, imports.clone()); + imports + } + // A fetch error is transient, not a property of the (immutable) + // content — memoizing it would drop this subtree from the key + // until worker restart. Skip caching and retry next run. + Err(e) => { + tracing::warn!( + "could not fetch import {path} (hash {hash}) while computing bundle \ + cache key for {script_path}: {e:#}" + ); + Arc::new(vec![]) + } + }, + }; + queue.extend(imports.iter().cloned()); + } + // deterministic key regardless of traversal order + versions.sort(); + versions } pub async fn compute_bundle_local_and_remote_path( @@ -1265,16 +1355,13 @@ pub async fn compute_bundle_local_and_remote_path( let mut input_src = format!("{inner_content}{lock}",); if let Some(db) = db { - let relative_imports = crate::worker_lockfiles::extract_relative_imports( - &inner_content, - script_path, - &Some(ScriptLang::Bun), - ); - for path in relative_imports.unwrap_or_default() { - if let Ok(updated_at) = get_script_import_updated_at(&db, w_id, &path).await { - input_src.push_str(&path); - input_src.push_str(&updated_at.to_string()); - } + // The bundle inlines the whole transitive relative-import closure, so a + // new deployed version of ANY script in it must change the key. + for (path, hash) in + collect_transitive_import_versions(db, w_id, script_path, inner_content).await + { + input_src.push_str(&path); + input_src.push_str(&hash.to_string()); } }; From 7c7d7474cc86a4052272032f281cc4d7a85db37b Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 2 Jul 2026 23:59:02 +0200 Subject: [PATCH 172/273] feat: support workspace forks on cloud using parent workspace limits (#9864) * feat: support workspace forks on cloud using parent workspace limits Co-Authored-By: Claude Opus 4.8 (1M context) * docs: clarify count_paid_seats approximates rather than mirrors billing seats Co-Authored-By: Claude Opus 4.8 (1M context) * fix: non-admin fork UI, attach cap, and fork-count for cloud forks Co-Authored-By: Claude Opus 4.8 (1M context) * fix: cloud fork billing cache on rename, usage display, attach cap edge Co-Authored-By: Claude Opus 4.8 (1M context) * feat: fork count in cloud quotas + fork billing points to parent Co-Authored-By: Claude Opus 4.8 (1M context) * fix: invalidate billing/fork caches on fork deletion for id reuse Co-Authored-By: Claude Opus 4.8 (1M context) * fix: gate fork usage remap on CLOUD_HOSTED, not just the cloud feature Co-Authored-By: Claude Opus 4.8 (1M context) * docs: note cloud feature vs CLOUD_HOSTED gating in backend guide Co-Authored-By: Claude Opus 4.8 (1M context) * fix: reserve fork-cap slots for an attach candidate's whole subtree Co-Authored-By: Claude Opus 4.8 (1M context) * fix: invalidate team-plan cache on delete, raise fork depth cap Co-Authored-By: Claude Opus 4.8 (1M context) * feat: cap fork nesting depth (MAX_FORK_DEPTH, default 5) Co-Authored-By: Claude Opus 4.8 (1M context) * fix: fork count/height robust to cycles and deleted intermediates Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): reset fork button loading state on creation error Co-Authored-By: Claude Opus 4.8 (1M context) * fix: invalidate billing cache for attached fork subtree; helper auth docs Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...0b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json | 22 ++ ...bc9b6a776a4941e68d8e2b31904c677493e2d.json | 22 ++ ...eaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json | 22 ++ ...dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json | 22 ++ ...1af05c5a6882f988ba5e0ec1f63828353987b.json | 22 ++ ...adcdd88b137bb892437a3ca67fd716772512b.json | 28 +++ backend/CLAUDE.md | 10 + .../windmill-api-workspaces/src/workspaces.rs | 177 +++++++++++++- .../src/workspaces_extra.rs | 33 ++- backend/windmill-api/openapi.yaml | 3 + backend/windmill-common/src/workspaces.rs | 216 +++++++++++++++++- .../tests/billing_workspace.rs | 211 +++++++++++++++++ backend/windmill-queue/src/jobs.rs | 35 ++- .../components/sessions/SessionForkBar.svelte | 9 +- .../sessions/WorkspaceFamilyPicker.svelte | 13 +- .../components/settings/CloudQuotas.svelte | 4 +- .../components/sidebar/WorkspaceMenu.svelte | 2 +- .../CreateWorkspaceInner.svelte | 39 ++-- .../src/routes/(root)/(logged)/+layout.svelte | 11 +- .../(logged)/workspace_settings/+page.svelte | 11 +- 20 files changed, 860 insertions(+), 52 deletions(-) create mode 100644 backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json create mode 100644 backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json create mode 100644 backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json create mode 100644 backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json create mode 100644 backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json create mode 100644 backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json create mode 100644 backend/windmill-common/tests/billing_workspace.rs diff --git a/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json new file mode 100644 index 0000000000..508eb32ed6 --- /dev/null +++ b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT COALESCE(MAX(depth), 0)::bigint AS \"depth!\" FROM chain\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "depth!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2" +} diff --git a/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json new file mode 100644 index 0000000000..44d9f8f36d --- /dev/null +++ b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS \"height!\" FROM tree\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d" +} diff --git a/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json new file mode 100644 index 0000000000..cb6bb20397 --- /dev/null +++ b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COUNT(DISTINCT id) AS \"count!\" FROM tree WHERE id != $1 AND NOT deleted\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba" +} diff --git a/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json new file mode 100644 index 0000000000..acfa08d0bf --- /dev/null +++ b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT id AS \"id!\" FROM tree WHERE id != $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d" +} diff --git a/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json new file mode 100644 index 0000000000..514024eb8d --- /dev/null +++ b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b" +} diff --git a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json new file mode 100644 index 0000000000..1930427b94 --- /dev/null +++ b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "developers!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b" +} diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 086e251962..ff5d9b2f22 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -11,3 +11,13 @@ cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh ``` Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. + +## Cloud vs self-hosted gating + +The `cloud` cargo feature is compiled into **all** EE builds, so `#[cfg(feature = "cloud")]` is **not** a "cloud-only" runtime gate — it only means the code is present. The real gate for behavior specific to the managed cloud (app.windmill.dev) is the runtime flag `*CLOUD_HOSTED` (`windmill_common::worker::CLOUD_HOSTED`, from the `CLOUD_HOSTED` env var; note it's loaded from `.env` via `dotenv`, so it won't show in `/proc//environ` — check the running behavior, not the exec env). + +Cloud-only logic must be behind `if *CLOUD_HOSTED { ... }`: feature-gate the helper so it compiles, then **runtime-gate the call**. `#[cfg(feature = "cloud")]` on its own is only sufficient for: +- pure helper/struct definitions (they only run when a gated caller invokes them), +- code already inside an `if *CLOUD_HOSTED { ... }` block, +- handlers that early-return on `!*CLOUD_HOSTED`, +- idempotent no-ops that are harmless off-cloud (e.g. cache invalidation). diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index bb77a50973..171ec94560 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -629,11 +629,13 @@ async fn list_pending_invites( } async fn is_premium( - authed: ApiAuthed, + _authed: ApiAuthed, Extension(_db): Extension, Path(_w_id): Path, ) -> JsonResult { - require_admin(authed.is_admin, &authed.username)?; + // Any workspace member (not just admins) may read whether the workspace is on a paid plan: it's a + // single boolean, and the frontend needs it to decide whether to surface premium-gated affordances + // (e.g. forking) to non-admin developers too. The `_authed` extractor still enforces membership. #[cfg(feature = "cloud")] let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) .await? @@ -3797,6 +3799,24 @@ lazy_static::lazy_static! { } }; + // Cloud only: how many forks a premium workspace may have per paid (developer) seat. + pub static ref MAX_FORKS_PER_SEAT: i64 = std::env::var("MAX_FORKS_PER_SEAT") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 0) + .unwrap_or(5); + + // How deep a fork chain may nest (root = depth 0, a direct fork = depth 1). A general guardrail + // for all builds, independent of the cloud per-seat cap: deep fork chains are a footgun and no + // real use case needs them. Clamped to [1, 20] so it's always a real limit and can never exceed + // the fork-walk recursion backstop (20) that billing/count resolution uses (a chain deeper than + // the backstop would truncate and mis-resolve its root). + pub static ref MAX_FORK_DEPTH: i64 = std::env::var("MAX_FORK_DEPTH") + .ok() + .and_then(|v| v.parse::().ok()) + .map(|v| v.clamp(1, 20)) + .unwrap_or(5); + } async fn create_workspace_require_superadmin() -> String { @@ -4996,10 +5016,12 @@ async fn create_workspace_fork_branch( Path(w_id): Path, Json(nw): Json, ) -> JsonResult> { + // Pre-check the fork guards before creating any git branch, so we don't leave orphaned branches + // behind when the follow-up create_workspace_fork would be rejected anyway. + enforce_fork_depth(&db, &w_id, 0).await?; + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); + enforce_cloud_fork_cap(&db, &w_id).await?; } if *DISABLE_WORKSPACE_FORK { @@ -5171,16 +5193,83 @@ async fn apply_forked_datatable( Ok(()) } +/// Cloud: require the fork/dev's root (billing) workspace be premium; returns the resolved root id. +#[cfg(feature = "cloud")] +async fn require_cloud_fork_premium(db: &DB, parent_workspace_id: &str) -> Result { + let root = + windmill_common::workspaces::get_billing_workspace_id(db, parent_workspace_id).await?; + if !windmill_common::workspaces::get_team_plan_status(db, &root) + .await? + .premium + { + return Err(Error::BadRequest( + "Creating a fork or dev workspace on the cloud requires a paid team plan. Upgrade the workspace first.".to_string(), + )); + } + Ok(root) +} + +/// Cloud: reject if adding `incoming` fork/dev workspaces would push `root`'s family over its per-seat +/// allotment. `incoming` is the number of workspaces the operation adds to the family — 1 for a plain +/// create, but `1 + candidate_subtree` for an attach whose candidate already has child forks. +#[cfg(feature = "cloud")] +async fn enforce_cloud_fork_count(db: &DB, root: &str, incoming: i64) -> Result<()> { + let seats = windmill_common::workspaces::count_paid_seats(db, root).await?; + let per_seat = *MAX_FORKS_PER_SEAT; + // Any premium workspace has at least one paid seat, so floor the seat count at 1. + let allowed = seats.max(1) * per_seat; + + let existing = windmill_common::workspaces::count_workspace_forks(db, root).await?; + let projected = existing + incoming; + if projected > allowed { + return Err(Error::BadRequest(format!( + "Fork limit reached: this would bring the workspace family to {projected} fork(s), over the cap of {allowed} ({seats} paid seat(s) × {per_seat} per seat). Delete a fork or add seats." + ))); + } + + Ok(()) +} + +/// Cloud-only guard for creating a fork/dev workspace. Forks piggyback on the parent's plan (a fork +/// inherits the root's premium and meters its usage into the root's bill), so forking is limited to +/// premium workspaces and capped at `MAX_FORKS_PER_SEAT` per paid (developer) seat of the root. +#[cfg(feature = "cloud")] +async fn enforce_cloud_fork_cap(db: &DB, parent_workspace_id: &str) -> Result<()> { + let root = require_cloud_fork_premium(db, parent_workspace_id).await?; + enforce_cloud_fork_count(db, &root, 1).await +} + +/// General guardrail (all builds): reject creating a fork/dev under `parent` when it would nest deeper +/// than `MAX_FORK_DEPTH`. `added_subtree_height` is the height of the subtree grafted below the new +/// node — 0 for a plain fork, or the candidate's own subtree height for an attach. +async fn enforce_fork_depth( + db: &DB, + parent_workspace_id: &str, + added_subtree_height: i64, +) -> Result<()> { + let parent_depth = + windmill_common::workspaces::fork_chain_depth(db, parent_workspace_id).await?; + // The new node sits one level below the parent; its deepest descendant adds the grafted height. + let resulting_depth = parent_depth + 1 + added_subtree_height; + if resulting_depth > *MAX_FORK_DEPTH { + return Err(Error::BadRequest(format!( + "Fork depth limit reached: forks can be nested at most {} level(s) deep, but this would create a fork at depth {}. Fork from a workspace closer to the root instead.", + *MAX_FORK_DEPTH, resulting_depth + ))); + } + Ok(()) +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, Path(parent_workspace_id): Path, Json(nw): Json, ) -> Result { + enforce_fork_depth(&db, &parent_workspace_id, 0).await?; + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); + enforce_cloud_fork_cap(&db, &parent_workspace_id).await?; } if nw.is_dev_workspace { @@ -5347,10 +5436,30 @@ async fn attach_dev_workspace( ) -> Result { require_admin(authed.is_admin, &authed.username)?; + // Attaching grafts the candidate (and its own fork subtree) under prod, so enforce the general + // depth limit on the deepest resulting node. + let candidate_height = + windmill_common::workspaces::fork_subtree_height(&db, &req.dev_workspace_id).await?; + enforce_fork_depth(&db, &prod_w_id, candidate_height).await?; + + // Attaching reparents a workspace under prod (one dev per prod, admin-gated) and it then draws + // prod's plan, so hold it to the same premium requirement as creating a fork. Only enforce the + // per-seat count when the attach actually adds a new workspace to the family: re-designating a + // workspace already under this root as its dev doesn't increase the descendant count. + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest( - "Dev workspaces are not available on app.windmill.dev".to_string(), - )); + let root = require_cloud_fork_premium(&db, &prod_w_id).await?; + // Only count against the cap when this attach adds workspaces to the family (candidate not + // already under this root). The candidate may itself have child forks, so reserve slots for its + // whole incoming subtree (the candidate + its descendants), not just one. + if windmill_common::workspaces::get_billing_workspace_id(&db, &req.dev_workspace_id).await? + != root + { + let incoming = + 1 + windmill_common::workspaces::count_workspace_forks(&db, &req.dev_workspace_id) + .await?; + enforce_cloud_fork_count(&db, &root, incoming).await?; + } } let dev_w_id = req.dev_workspace_id; @@ -5468,6 +5577,17 @@ async fn attach_dev_workspace( // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping // so per-workspace job tags route to the prod family immediately rather than after the TTL. windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + // Same reparent invalidates the billing-workspace mapping so its usage meters to prod at once. The + // candidate can bring its own fork subtree, whose descendants had resolved their (now-stale) root + // to the candidate's old family; invalidate them too so they meter to prod without waiting out the + // 60s TTL. Their immediate fork->parent links don't move, so the tag-routing cache needs no change. + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_billing_workspace_cache(&id); + } + } if req.lock_prod_deploy || req.lock_prod_forking { windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); @@ -6415,6 +6535,17 @@ async fn change_workspace_color( } async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Result { + // On cloud, a fork's executions meter against its billing root, so report the root's usage here too; + // otherwise the free-execs indicator would show the fork's own (often 0) count while enforcement + // applies the root's shared quota. Gated on `*CLOUD_HOSTED` (not just the `cloud` feature, which is + // compiled into all EE builds): self-hosted doesn't meter usage this way. Off-fork it resolves to + // `w_id` itself anyway. + #[cfg(feature = "cloud")] + let w_id = if *CLOUD_HOSTED { + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id).await? + } else { + w_id + }; let usage = sqlx::query_scalar!( " SELECT usage.usage FROM usage @@ -8230,6 +8361,10 @@ struct CloudQuotas { apps: QuotaInfo, variables: QuotaInfo, resources: QuotaInfo, + /// Fork/dev workspaces under this workspace's billing root vs the per-seat cap. `limit` is 0 for a + /// non-premium root (forking is premium-only). Family-wide: resolves to the billing root, so it + /// reads the same whether viewed from the root or one of its forks. + forks: QuotaInfo, } async fn get_cloud_quotas( @@ -8310,12 +8445,32 @@ async fn get_cloud_quotas( .await? .unwrap_or(0); + // Fork/dev workspaces vs the per-seat cap, resolved to the billing root. Non-premium roots can't + // fork, so their allowance is 0. + #[cfg(feature = "cloud")] + let forks = { + use windmill_common::workspaces::{ + count_paid_seats, count_workspace_forks, get_billing_workspace_id, get_team_plan_status, + }; + let root = get_billing_workspace_id(&db, &w_id).await?; + let used = count_workspace_forks(&db, &root).await?; + let limit = if get_team_plan_status(&db, &root).await?.premium { + count_paid_seats(&db, &root).await?.max(1) * *MAX_FORKS_PER_SEAT + } else { + 0 + }; + QuotaInfo { used, limit, prunable: 0 } + }; + #[cfg(not(feature = "cloud"))] + let forks = QuotaInfo { used: 0, limit: 0, prunable: 0 }; + Ok(Json(CloudQuotas { scripts: QuotaInfo { used: scripts_used, limit: 5000, prunable: scripts_prunable }, flows: QuotaInfo { used: flows_used, limit: 1000, prunable: flows_prunable }, apps: QuotaInfo { used: apps_used, limit: 1000, prunable: apps_prunable }, variables: QuotaInfo { used: variables_used, limit: 10000, prunable: 0 }, resources: QuotaInfo { used: resources_used, limit: 10000, prunable: 0 }, + forks, })) } diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 47d8c24b92..56abe689d5 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -726,10 +726,13 @@ pub(crate) async fn change_workspace_id( tx.commit().await?; // The children's parent_workspace_id changed (old root -> new root); invalidate their fork-parent - // routing cache so jobs route under the renamed root rather than the old (archived) one until the - // 300s TTL would otherwise expire. + // routing cache and their billing-workspace mapping so jobs route + meter under the renamed root + // rather than the old (archived) one, instead of waiting for the caches' TTLs. Deeper descendants + // (fork-of-fork) self-heal via the 60s billing-cache TTL. for child in &reparented_children { windmill_queue::tags::invalidate_fork_parent_cache(child); + #[cfg(feature = "cloud")] + windmill_common::workspaces::invalidate_billing_workspace_cache(child); } // Archive old workspace: disable schedules, cancel remaining jobs, set deleted=true @@ -1017,6 +1020,16 @@ pub(crate) async fn delete_workspace( .await? .flatten(); + // Capture direct child forks before the delete: the FK is ON DELETE SET NULL, so they're about to + // be orphaned (their billing root changes from this workspace's root to themselves). We drop their + // cached mappings after commit alongside the deleted id itself. + let orphaned_children: Vec = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id) .execute(&mut *tx) .await?; @@ -1050,6 +1063,22 @@ pub(crate) async fn delete_workspace( windmill_common::workspaces::invalidate_protection_rules_cache(&parent); } + // Workspace ids are reusable after permanent deletion, so drop every cached mapping keyed by the + // deleted id (and any just-orphaned children) — otherwise a recreated id could inherit the gone + // workspace's state within the caches' lifetimes. This covers fork->parent (tag routing) and + // fork->root (billing), plus the premium/team-plan status: TEAM_PLAN_CACHE has no TTL and is only + // evicted by the premium-change NOTIFY, so without this a reused id would keep the old workspace's + // premium indefinitely (free forks/usage). Deeper (grandchild) descendants self-heal via the 60s + // billing TTL. + for id in std::iter::once(&w_id).chain(orphaned_children.iter()) { + windmill_queue::tags::invalidate_fork_parent_cache(id); + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(id); + windmill_common::workspaces::invalidate_team_plan_cache(id); + } + } + Ok(format!("Deleted workspace {}", &w_id)) } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3b90927901..eec76ea009 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6221,12 +6221,15 @@ paths: $ref: "#/components/schemas/QuotaInfo" resources: $ref: "#/components/schemas/QuotaInfo" + forks: + $ref: "#/components/schemas/QuotaInfo" required: - scripts - flows - apps - variables - resources + - forks /w/{workspace}/workspaces/prune_versions: post: diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index cd3693cd92..5593a565ef 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -347,9 +347,215 @@ lazy_static::lazy_static! { pub static ref PUBLIC_APP_RATE_LIMIT_CACHE: Cache, i64)> = Cache::new(1000); } +#[cfg(feature = "cloud")] +lazy_static::lazy_static! { + // Maps a workspace id to its root (billing) ancestor. Value: (root_id, expiry_timestamp). + // Reparenting (attach/detach dev) is rare and self-heals via the 60s TTL, so a brief stale + // mapping only mis-attributes usage for <60s across other instances. + pub static ref BILLING_WORKSPACE_CACHE: Cache = Cache::new(5000); +} + +/// Resolve the "billing" workspace for `w_id`: the root ancestor of the fork/dev chain (the +/// workspace whose plan and usage a fork draws from). Returns `w_id` unchanged for a standalone +/// workspace, an unknown id, or a (malformed) cyclic chain. +/// +/// Unauthenticated metering helper: it only reads the parent chain and returns another workspace id, +/// so callers must already be authorized for `w_id` (or run in trusted server-side code); `w_id` is +/// expected to be a server-side id, not raw user input. +#[cfg(feature = "cloud")] +pub async fn get_billing_workspace_id(db: &crate::DB, w_id: &str) -> Result { + let now = chrono::Utc::now().timestamp(); + if let Some((root, expiry)) = BILLING_WORKSPACE_CACHE.get(w_id) { + if expiry > now { + return Ok(root); + } + } + + // The depth bound is a cycle-safety backstop kept well above the enforced `MAX_FORK_DEPTH`, so a + // truncated (root-not-found) result — which would fall back to `w_id` and mis-attribute billing — + // is unreachable for any real hierarchy; only a malformed cycle could hit it. + let root = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT id AS "id!" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1 + "#, + w_id + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("resolving billing workspace for {w_id}: {e:#}")))? + .unwrap_or_else(|| w_id.to_string()); + + BILLING_WORKSPACE_CACHE.insert(w_id.to_string(), (root.clone(), now + 60)); + Ok(root) +} + +/// Invalidate the billing-workspace mapping for a workspace (call after reparenting it). +#[cfg(feature = "cloud")] +pub fn invalidate_billing_workspace_cache(w_id: &str) { + BILLING_WORKSPACE_CACHE.remove(w_id); +} + +/// Invalidate the cached team-plan (premium/past-due) status for a workspace. `TEAM_PLAN_CACHE` has +/// no TTL — it's only evicted by the premium-change NOTIFY — so call this when a workspace id is +/// permanently deleted, otherwise a reused id could inherit the old workspace's premium status. +#[cfg(feature = "cloud")] +pub fn invalidate_team_plan_cache(w_id: &str) { + TEAM_PLAN_CACHE.remove(w_id); +} + +/// Depth of `w_id` in its fork chain: 0 for a root (no parent), 1 for a direct fork, and so on. Walks +/// the parent chain up to the root. The recursion bound is a cycle-safety backstop set well above the +/// enforced `MAX_FORK_DEPTH`; a (malformed) cyclic chain saturates it and so reads as "too deep", +/// which safely rejects rather than allows. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_chain_depth(db: &crate::DB, w_id: &str) -> Result { + let depth = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT COALESCE(MAX(depth), 0)::bigint AS "depth!" FROM chain + "#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("computing fork depth for {w_id}: {e:#}")))?; + Ok(depth) +} + +/// Height of the fork subtree rooted at `w_id`: 0 when it has no live child forks, 1 with direct +/// children, and so on. Used so that attaching a candidate which already has its own child forks can't +/// push the family past the depth limit. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_subtree_height(db: &crate::DB, w_id: &str) -> Result { + // The `deleted` filter is applied in the outer aggregation (not the recursive step, matching + // count_workspace_forks) so a live descendant under a soft-deleted intermediate is still measured + // at its true depth rather than pruned — otherwise the height could be underestimated and let the + // resulting chain exceed the depth limit. + let height = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS "height!" FROM tree + "#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("computing fork subtree height for {w_id}: {e:#}")))?; + Ok(height) +} + +/// Ids of every fork/dev workspace anywhere under `w_id` (excludes `w_id` itself), including live +/// descendants beneath a soft-deleted intermediate. Used to invalidate per-workspace caches for a +/// whole subtree after its ancestor is reparented. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn list_fork_descendants(db: &crate::DB, w_id: &str) -> Result> { + let ids = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT id AS "id!" FROM tree WHERE id != $1 + "#, + w_id + ) + .fetch_all(db) + .await + .map_err(|e| Error::internal_err(format!("listing fork descendants of {w_id}: {e:#}")))?; + Ok(ids) +} + +/// Count non-deleted fork/dev workspaces anywhere under `root` (excludes `root` itself). +/// +/// Unauthenticated metering helper: it reads workspace hierarchy for any `root` id, so callers must +/// already be authorized for that workspace (or run in trusted server-side code). `root` is expected +/// to be a server-resolved id, never raw user input. +#[cfg(feature = "cloud")] +pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result { + // The `deleted` filter is on the outer SELECT (not the recursive step) so that a live sub-fork + // whose intermediate parent was soft-deleted is still counted rather than pruned with it. The + // depth bound is a cycle-safety backstop kept well above the enforced `MAX_FORK_DEPTH`, so + // descendants are never silently dropped from the cap count. + let count = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT COUNT(DISTINCT id) AS "count!" FROM tree WHERE id != $1 AND NOT deleted + "#, + root + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("counting forks of {root}: {e:#}")))?; + Ok(count) +} + +/// Approximate paid seats of a workspace as `ceil(developers + operators/2)`, excluding disabled and +/// service-account members. Reuses billing's author/operator weighting, but counts provisioned +/// members rather than the active-user population billing meters, so it only ever loosens the fork +/// cap (never blocks a paid seat) — good enough for a soft guardrail. +/// +/// Unauthenticated metering helper: reads member counts for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +#[cfg(feature = "cloud")] +pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { + let row = sqlx::query!( + r#"SELECT + COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!", + COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS "operators!" + FROM usr WHERE workspace_id = $1"#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("counting paid seats of {w_id}: {e:#}")))?; + Ok(((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64) +} + #[cfg(feature = "cloud")] pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result { - let cached = TEAM_PLAN_CACHE.get(_w_id); + // A fork/dev workspace draws its plan from the root (billing) workspace. Resolve to the root and + // key the cache by it: the premium-change NOTIFY is keyed by the workspace whose premium row + // changed (the root), so keying by root keeps invalidation correct and lets forks share it. + let billing_w_id = get_billing_workspace_id(_db, _w_id).await?; + let cached = TEAM_PLAN_CACHE.get(&billing_w_id); if let Some(cached) = cached { return Ok(cached); } @@ -368,7 +574,7 @@ pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result Result Result, id: &str, parent: Option<&str>, deleted: bool) { + sqlx::query( + "INSERT INTO workspace (id, name, owner, parent_workspace_id, deleted) + VALUES ($1, $1, 'test-user', $2, $3)", + ) + .bind(id) + .bind(parent) + .bind(deleted) + .execute(db) + .await + .expect("insert workspace"); + // The resolver caches per id (60s TTL) in a process-global cache shared across tests, so drop + // any stale mapping for this id before the test reads it. + invalidate_billing_workspace_cache(id); +} + +async fn insert_member( + db: &Pool, + w_id: &str, + email: &str, + operator: bool, + disabled: bool, + is_service_account: bool, +) { + // `usr.username` has a `proper_username` check (no `@`), so derive one from the email prefix. + let username = email.split('@').next().unwrap(); + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, operator, disabled, is_service_account, role) + VALUES ($1, $2, $3, false, $4, $5, $6, 'Developer')", + ) + .bind(w_id) + .bind(email) + .bind(username) + .bind(operator) + .bind(disabled) + .bind(is_service_account) + .execute(db) + .await + .expect("insert usr"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_workspace_resolves_to_root(db: Pool) { + insert_ws(&db, "bwt-root", None, false).await; + insert_ws(&db, "bwt-fork", Some("bwt-root"), false).await; + insert_ws(&db, "bwt-grandchild", Some("bwt-fork"), false).await; + + assert_eq!( + get_billing_workspace_id(&db, "bwt-root").await.unwrap(), + "bwt-root" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwt-fork").await.unwrap(), + "bwt-root" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwt-grandchild") + .await + .unwrap(), + "bwt-root" + ); + // Unknown / orphaned ids resolve to themselves. + assert_eq!( + get_billing_workspace_id(&db, "bwt-missing").await.unwrap(), + "bwt-missing" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_workspace_survives_cycles(db: Pool) { + insert_ws(&db, "bwc-a", None, false).await; + insert_ws(&db, "bwc-b", Some("bwc-a"), false).await; + // Introduce a cycle a -> b -> a; no row has a NULL parent, so resolution falls back to the input + // and the depth guard keeps it from looping forever. + sqlx::query("UPDATE workspace SET parent_workspace_id = 'bwc-b' WHERE id = 'bwc-a'") + .execute(&db) + .await + .unwrap(); + invalidate_billing_workspace_cache("bwc-a"); + invalidate_billing_workspace_cache("bwc-b"); + + assert_eq!( + get_billing_workspace_id(&db, "bwc-a").await.unwrap(), + "bwc-a" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwc-b").await.unwrap(), + "bwc-b" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn paid_seats_and_fork_count(db: Pool) { + insert_ws(&db, "seat-root", None, false).await; + // 2 developers + 2 operators counted -> ceil(2 + 0.5*2) = 3. + insert_member(&db, "seat-root", "dev1@w.dev", false, false, false).await; + insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await; + insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await; + insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await; + // These must NOT count towards seats. + insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await; + insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await; + + assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3); + + insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; + insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; + // A deleted fork itself is not counted... + insert_ws(&db, "seat-fork-deleted", Some("seat-root"), true).await; + // ...but a live sub-fork under it still is (deleted filter is on the outer SELECT, not the walk). + insert_ws(&db, "seat-deleted-child", Some("seat-fork-deleted"), false).await; + // A grandchild fork still counts. + insert_ws(&db, "seat-fork1-child", Some("seat-fork1"), false).await; + + // Live: fork1, fork2, fork1-child, deleted-child -> 4 (seat-fork-deleted excluded). + assert_eq!(count_workspace_forks(&db, "seat-root").await.unwrap(), 4); + // A standalone workspace has no forks. + assert_eq!(count_workspace_forks(&db, "seat-fork2").await.unwrap(), 0); + + // list_fork_descendants returns every descendant id (deleted included, for cache invalidation): + // fork1, fork2, fork-deleted, deleted-child, fork1-child -> 5. + let mut descendants = list_fork_descendants(&db, "seat-root").await.unwrap(); + descendants.sort(); + assert_eq!( + descendants, + vec![ + "seat-deleted-child", + "seat-fork-deleted", + "seat-fork1", + "seat-fork1-child", + "seat-fork2", + ] + ); + assert!(list_fork_descendants(&db, "seat-fork2") + .await + .unwrap() + .is_empty()); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_cache_invalidation_reflects_reparent(db: Pool) { + insert_ws(&db, "inv-root-a", None, false).await; + insert_ws(&db, "inv-root-b", None, false).await; + insert_ws(&db, "inv-fork", Some("inv-root-a"), false).await; + + // Resolve + cache: fork -> root-a. + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-a" + ); + + // Reparent in the DB, as delete+recreate-under-another-root (or attach) would. + sqlx::query("UPDATE workspace SET parent_workspace_id = 'inv-root-b' WHERE id = 'inv-fork'") + .execute(&db) + .await + .unwrap(); + + // The cached mapping survives until invalidated (this is the staleness the delete/attach/rename + // paths must clear). + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-a" + ); + + // After invalidation (what delete_workspace / attach_dev_workspace / change_workspace_id now call), + // it re-resolves to the new root. + invalidate_billing_workspace_cache("inv-fork"); + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-b" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn fork_depth_and_subtree_height(db: Pool) { + // Chain: root -> f1 -> f2 -> f3 + insert_ws(&db, "fd-root", None, false).await; + insert_ws(&db, "fd-f1", Some("fd-root"), false).await; + insert_ws(&db, "fd-f2", Some("fd-f1"), false).await; + insert_ws(&db, "fd-f3", Some("fd-f2"), false).await; + + // Depth walks up to the root: root = 0, each fork adds one. + assert_eq!(fork_chain_depth(&db, "fd-root").await.unwrap(), 0); + assert_eq!(fork_chain_depth(&db, "fd-f1").await.unwrap(), 1); + assert_eq!(fork_chain_depth(&db, "fd-f3").await.unwrap(), 3); + // An unknown id has no chain, so depth 0 (treated as a root by the guard). + assert_eq!(fork_chain_depth(&db, "fd-missing").await.unwrap(), 0); + + // Height walks down: the deepest live descendant below the node. + assert_eq!(fork_subtree_height(&db, "fd-root").await.unwrap(), 3); + assert_eq!(fork_subtree_height(&db, "fd-f2").await.unwrap(), 1); + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 0); + + // A deleted leaf doesn't add to the height. + insert_ws(&db, "fd-f3-del", Some("fd-f3"), true).await; + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 0); + + // ...but a LIVE descendant below a soft-deleted intermediate still counts at its true depth + // (the walk traverses through the deleted node; only the aggregation filters deleted). + insert_ws(&db, "fd-f3-live-gc", Some("fd-f3-del"), false).await; + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 2); +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c92cea9ef3..99f33da3bf 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1914,22 +1914,31 @@ fn apply_completed_job_cloud_usage( tokio::task::spawn(async move { let additional_usage = _duration / 1000; let result = tokio::time::timeout(std::time::Duration::from_secs(10), async move { + // Fork/dev execution-seconds meter against the root (billing) workspace; resolves to + // `w_id` itself off-fork. + let billing_w_id = + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id) + .await + .unwrap_or_else(|e| { + tracing::error!("Failed to resolve billing workspace for {w_id}: {e:#}"); + w_id.clone() + }); // Update workspace usage let workspace_result = sqlx::query!( "INSERT INTO usage (id, is_workspace, month_, usage) VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage", - &w_id, + &billing_w_id, additional_usage as i32 ) .execute(&db) .await; if let Err(e) = workspace_result { - tracing::error!("Failed to update workspace usage for {}: {:#}", w_id, e); + tracing::error!("Failed to update workspace usage for {}: {:#}", billing_w_id, e); } - match windmill_common::workspaces::get_team_plan_status(&db, &w_id).await { + match windmill_common::workspaces::get_team_plan_status(&db, &billing_w_id).await { Ok(team_plan_status) => { // Update user usage for non-premium workspaces if !team_plan_status.premium { @@ -5124,8 +5133,13 @@ async fn push_inner<'c, 'd>( ) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { #[cfg(feature = "cloud")] if *CLOUD_HOSTED { + // A fork/dev workspace draws its plan and usage from the root (billing) workspace, so its + // executions are metered against the parent's quota/bill. Resolves to `workspace_id` itself + // for a standalone workspace (no behavior change off-fork). + let billing_w_id = + windmill_common::workspaces::get_billing_workspace_id(db, workspace_id).await?; let team_plan_status = - windmill_common::workspaces::get_team_plan_status(db, workspace_id).await?; + windmill_common::workspaces::get_team_plan_status(db, &billing_w_id).await?; // we track only non flow steps let (workspace_usage, user_usage) = if !matches!( job_payload, @@ -5134,12 +5148,12 @@ async fn push_inner<'c, 'd>( // Check current usage with SELECT (fast, no row locks) // Only check user usage for non-premium workspaces let (current_workspace_usage, current_user_usage) = - check_usage_limits(db, workspace_id, email, !team_plan_status.premium).await?; + check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?; // Spawn async task to update usage counters in the background increment_usage_async( db.clone(), - workspace_id.to_string(), + billing_w_id.clone(), if !team_plan_status.premium { Some(email.to_string()) } else { @@ -5242,7 +5256,7 @@ async fn push_inner<'c, 'd>( WHERE is_workspace IS TRUE AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) AND id = $1", - workspace_id + billing_w_id ) .fetch_optional(db) .await? @@ -5269,6 +5283,13 @@ async fn push_inner<'c, 'd>( ))); } + // These two burst guards intentionally stay keyed to `workspace_id`, not the + // billing root: the shared caps are the monthly usage above (metered to the + // root) and the per-user in-queue/concurrent guards further up (keyed by email, + // global across the whole family). Keying these to the root would count the + // root's own queue rather than this workspace's load or the family's; a true + // family-shared burst cap would need a family-wide subquery, not worth the + // hot-path cost for this downgrade-only soft guard. let in_queue_workspace = sqlx::query_scalar!( "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", workspace_id diff --git a/frontend/src/lib/components/sessions/SessionForkBar.svelte b/frontend/src/lib/components/sessions/SessionForkBar.svelte index 982dc68c0f..a0aafee456 100644 --- a/frontend/src/lib/components/sessions/SessionForkBar.svelte +++ b/frontend/src/lib/components/sessions/SessionForkBar.svelte @@ -11,7 +11,7 @@ } from 'lucide-svelte' import { Button } from '$lib/components/common' import WorkspaceFamilyPicker from './WorkspaceFamilyPicker.svelte' - import { userStore, userWorkspaces, workspaceStore } from '$lib/stores' + import { isPremiumStore, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { goto } from '$lib/navigation' import { canCreateFork } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' @@ -52,9 +52,12 @@ const isFork = $derived(!!parentWorkspaceId) // Same gate as the sidebar WorkspaceMenu / SessionWorkspaceBar. - // When forking isn't available the diff/review surface is moot. + // When forking isn't available the diff/review surface is moot. On cloud, forking is a + // premium-only feature (backend caps it per paid seat). const forksAllowed = $derived( - !isCloudHosted() && canCreateFork($userStore) && $workspaceStore !== 'admins' + (!isCloudHosted() || $isPremiumStore) && + canCreateFork($userStore) && + $workspaceStore !== 'admins' ) let diffDrawer: ForkDiffDrawer | undefined = $state(undefined) diff --git a/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte b/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte index 6987730b87..91a5922871 100644 --- a/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte @@ -2,6 +2,7 @@ import { tick, type Snippet } from 'svelte' import { enterpriseLicense, + isPremiumStore, userStore, userWorkspaces, workspaceStore, @@ -114,12 +115,14 @@ rootRulesetsResource.loading || rootUserInfoResource.loading || !canDeployRoot ) - // Structural gate: hidden on cloud, in the admins workspace, or when the user - // can't fork. DisableWorkspaceForking on the active workspace (a locked prod) - // doesn't apply when there's a dev to fork from instead — the dev isn't - // locked, and devOfRoot only resolves when the user is a member of it. + // Structural gate: hidden in the admins workspace, or when the user can't fork; on cloud, forking + // is premium-only (backend caps it per paid seat). DisableWorkspaceForking on the active workspace + // (a locked prod) doesn't apply when there's a dev to fork from instead — the dev isn't locked, and + // devOfRoot only resolves when the user is a member of it. const forksGateOpen = $derived( - !isCloudHosted() && $workspaceStore !== 'admins' && (canCreateFork($userStore) || !!devOfRoot) + (!isCloudHosted() || $isPremiumStore) && + $workspaceStore !== 'admins' && + (canCreateFork($userStore) || !!devOfRoot) ) // A fork is a new workspace, so it's subject to the community-edition cap on // the number of non-'admins' workspaces (backend _check_nb_of_workspaces, diff --git a/frontend/src/lib/components/settings/CloudQuotas.svelte b/frontend/src/lib/components/settings/CloudQuotas.svelte index 61158f13ec..ec597101ee 100644 --- a/frontend/src/lib/components/settings/CloudQuotas.svelte +++ b/frontend/src/lib/components/settings/CloudQuotas.svelte @@ -18,6 +18,7 @@ apps: QuotaInfo variables: QuotaInfo resources: QuotaInfo + forks: QuotaInfo } | undefined = $state(undefined) @@ -87,7 +88,8 @@ { label: 'Flows', key: 'flows', prunable: true }, { label: 'Apps', key: 'apps', prunable: true }, { label: 'Variables', key: 'variables', prunable: false }, - { label: 'Resources', key: 'resources', prunable: false } + { label: 'Resources', key: 'resources', prunable: false }, + { label: 'Forks', key: 'forks', prunable: false } ] diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index 71e5636512..53ac410cc8 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -217,7 +217,7 @@ {/if} - {#if !strictWorkspaceSelect && !isCloudHosted() && canCreateFork($userStore) && $workspaceStore !== 'admins'} + {#if !strictWorkspaceSelect && (!isCloudHosted() || $isPremiumStore) && canCreateFork($userStore) && $workspaceStore !== 'admins'}
{ - let gitSyncJobIds = await WorkspaceService.createWorkspaceForkGitBranch({ - workspace: $workspaceStore!, - requestBody: { - id: prefixed_id, - name, - color: colorEnabled && workspaceColor ? workspaceColor : undefined, - is_dev_workspace: createAsDevWorkspace, - // Send the lock intent in this first phase too so the backend can reject a non-admin's - // locked-dev request before any branch is created (avoids dangling branches). - lock_prod_deploy: createAsDevWorkspace && lockProdDeploy, - lock_prod_forking: createAsDevWorkspace && lockProdForking, - copy_members: copyMembers - } - }) + let gitSyncJobIds: string[] + try { + gitSyncJobIds = await WorkspaceService.createWorkspaceForkGitBranch({ + workspace: $workspaceStore!, + requestBody: { + id: prefixed_id, + name, + color: colorEnabled && workspaceColor ? workspaceColor : undefined, + is_dev_workspace: createAsDevWorkspace, + // Send the lock intent in this first phase too so the backend can reject a non-admin's + // locked-dev request before any branch is created (avoids dangling branches). + lock_prod_deploy: createAsDevWorkspace && lockProdDeploy, + lock_prod_forking: createAsDevWorkspace && lockProdForking, + copy_members: copyMembers + } + }) + } catch (e) { + // The backend can reject here (fork cap, depth limit, premium, non-admin lock). Reset the + // loading state and surface the error rather than leaving the button spinning. + forkCreationError = `Failed to create fork '${prefixed_id}'` + errorMsgs.push(e?.body ?? e ?? 'Unknown error') + forkCreationLoading = false + sendUserToast(`Could not create fork '${prefixed_id}' ${e?.body ?? e}`, true) + return + } try { await Promise.all( diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index e91ad87583..030cf67fd0 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -147,8 +147,15 @@ } catch (e) { console.error('Could not persist username to local storage', e) } - if (isCloudHosted() && user?.is_admin) { - isPremiumStore.set(await WorkspaceService.getIsPremium({ workspace })) + // Populate for all members (not just admins) so non-admin developers also get premium-gated + // affordances like the fork entry points on cloud. The `is_premium` endpoint is a boolean + // and no longer admin-gated. Best-effort: a failure here must not block user-store init. + if (isCloudHosted()) { + try { + isPremiumStore.set(await WorkspaceService.getIsPremium({ workspace })) + } catch (e) { + console.error('Could not fetch premium status', e) + } } } else { userStore.set(undefined) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 8fbec8f204..94628c8b17 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -1377,7 +1377,16 @@ /> {:else if tab == 'premium'} - + {#if currentWsForDevTab?.parent_workspace_id} + + This workspace is a fork of {currentWsForDevTab.parent_workspace_id}. It + runs on the parent's plan and its executions count toward the parent's usage and + bill, so there is no separate subscription here. Manage billing, seats, and quotas + from the parent workspace's settings. + + {:else} + + {/if} {:else if tab == 'slack'} Date: Thu, 2 Jul 2026 22:57:19 +0000 Subject: [PATCH 173/273] feat(s3): replace CE 50MB upload cap with 10GiB workspace storage quota (#9874) * fix(s3_proxy): enforce CE 50MB upload cap on multipart uploads Co-Authored-By: Claude Fable 5 * feat(s3): replace CE 50MB upload cap with 10GiB workspace storage quota Co-Authored-By: Claude Fable 5 * fix(s3): gate CE quota OSS stubs to not(enterprise) to match callers Co-Authored-By: Claude Fable 5 * fix(s3): delta-aware CE storage quota + guard usage-load retry loop Account for the overwritten object's size in the quota check so valid same-size overwrites near quota are not rejected (Codex review), and stop the storage-usage $effect from re-firing on persistent API errors (Pi review). Co-Authored-By: Claude Fable 5 * fix(s3): count chunked PUTs; revert overreaching volume quota copy Volumes write to workspace storage via a separate worker-side path with its own 50MB-per-file cap that this PR does not change, so revert the drawer copy that claimed they count toward the 10GiB quota (Codex review). Bump ee-repo-ref for the chunked-PUT accounting fix. Co-Authored-By: Claude Fable 5 * fix(s3): add SQLx cache for CE usage-bump/quota queries; exclude volumes Regenerate the missing offline SQLx cache for the not(enterprise) bump and remaining-quota queries so the private CE offline build compiles, and bump ee-repo-ref for the volumes/-prefix exclusion from the counted quota (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): always HEAD for CE upload delta so overwrites don't inflate usage Bump ee-repo-ref for the fast-path overwrite-accounting fix (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): reserve volumes/ prefix on CE write surfaces to close quota bypass Reject direct writes to the reserved volume prefix on the app-upload surface and add the OSS stub; bump ee-repo-ref (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): refuse new multipart work when CE workspace is at quota Bump ee-repo-ref for the multipart-initiate/part quota gate (Codex review). Co-Authored-By: Claude Fable 5 * feat(s3): reserve in-flight multipart bytes against CE storage quota Add workspace_multipart_inflight table + grants, SQLx cache for the reservation queries, and bump ee-repo-ref. Bounds abandoned multipart uploads that the list-based recount can't see (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): clear multipart reservation only after a successful complete Add exclude-upload arg to the OSS quota stub/caller and the SQLx cache for the updated remaining-quota query; bump ee-repo-ref (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): per-part multipart reservation; commit only on part success Per-part workspace_multipart_inflight schema (upload_id, part_id) so retries replace rather than double-count; SQLx cache for the reworked queries; bump ee-repo-ref (Codex review). Co-Authored-By: Claude Fable 5 * perf(s3): HEAD the multipart overwrite target once per upload, not per part SQLx cache for the stored-credit lookup; bump ee-repo-ref. Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to bea5a8b5120d6d69cab1ad4611ebe463559bd200 This commit updates the EE repository reference after PR #640 was merged in windmill-ee-private. Previous ee-repo-ref: 6e6ff86f1939cf74736b7d435bf6851416437523 New ee-repo-ref: bea5a8b5120d6d69cab1ad4611ebe463559bd200 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- ...464b5ff9f34e43fbcc38154939461aadfca9f.json | 18 +++ ...5a055b6972eb00e4dc1818f7a68bcf04c1ba7.json | 23 ++++ ...e0d5d1c3769d095352002daf8c11d10eac57b.json | 30 +++++ ...1bdb31f26e82fb0dc97bef7114d85959b6129.json | 35 ++++++ ...1cd50b297f80e5392715f54f32fab4bf346fc.json | 22 ++++ ...ebbe13e6298749b19570f9f8c4a0c7958b0ce.json | 36 ++++++ ...03ff9c96d613fb2d9b698361f4ede8031e5d2.json | 15 +++ ...01d9c16a9d4b5b422159fc239ff5502a7b71f.json | 15 +++ ...aaff40a634fc5530e202f65fad6a241f2671a.json | 19 +++ ...ed3e9625f54cd89cfd595665e4473c5f3ea6c.json | 37 ++++++ ...b610b028e88d5d2dbbea44377bf9dc416cd03.json | 22 ++++ ...5274749d7a436a753813da098758514a7493d.json | 22 ++++ ...6fea272ac289fc17ea9fc5aa237f77778b58f.json | 29 +++++ ...48af40192832c07c1c7049def3a80057bb683.json | 28 +++++ ...0ebf972bcc7f821fbf1e0171266efd8d46616.json | 16 +++ ...2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json | 34 +++++ ...406d1773657c62d67e55dac666f9469d50e6a.json | 15 +++ ...e7c80dd0ef92d78224792bec4c3323e14023c.json | 36 ++++++ ...05b693965f1bca6fef7418b19f98671c9204c.json | 16 +++ backend/ee-repo-ref.txt | 2 +- ...702064737_workspace_storage_usage.down.sql | 1 + ...60702064737_workspace_storage_usage.up.sql | 17 +++ ...3513_workspace_multipart_inflight.down.sql | 1 + ...213513_workspace_multipart_inflight.up.sql | 32 +++++ backend/windmill-api/openapi.yaml | 49 ++++++++ backend/windmill-api/src/apps.rs | 49 +++++++- backend/windmill-api/src/args.rs | 38 +++++- backend/windmill-api/src/job_helpers_oss.rs | 70 ++++++++++- .../workspaceSettings/StorageSettings.svelte | 119 ++++++++++++++++-- 29 files changed, 833 insertions(+), 13 deletions(-) create mode 100644 backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json create mode 100644 backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json create mode 100644 backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json create mode 100644 backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json create mode 100644 backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json create mode 100644 backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json create mode 100644 backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json create mode 100644 backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json create mode 100644 backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json create mode 100644 backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json create mode 100644 backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json create mode 100644 backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json create mode 100644 backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json create mode 100644 backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json create mode 100644 backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json create mode 100644 backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json create mode 100644 backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json create mode 100644 backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json create mode 100644 backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json create mode 100644 backend/migrations/20260702064737_workspace_storage_usage.down.sql create mode 100644 backend/migrations/20260702064737_workspace_storage_usage.up.sql create mode 100644 backend/migrations/20260702213513_workspace_multipart_inflight.down.sql create mode 100644 backend/migrations/20260702213513_workspace_multipart_inflight.up.sql diff --git a/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json new file mode 100644 index 0000000000..708922a0cb --- /dev/null +++ b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, storage, inflight_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, upload_id)\n DO UPDATE SET inflight_bytes = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f" +} diff --git a/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json new file mode 100644 index 0000000000..16063b9a3f --- /dev/null +++ b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7" +} diff --git a/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json new file mode 100644 index 0000000000..291dc00310 --- /dev/null +++ b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2 AND created_at > now() - ($3::text)::interval)::bigint as \"other_reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b" +} diff --git a/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json new file mode 100644 index 0000000000..137af7638b --- /dev/null +++ b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129" +} diff --git a/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json new file mode 100644 index 0000000000..18af751ca0 --- /dev/null +++ b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('workspace_multipart_inflight'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc" +} diff --git a/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json new file mode 100644 index 0000000000..cd2dd23319 --- /dev/null +++ b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3))::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce" +} diff --git a/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json new file mode 100644 index 0000000000..be54b4a25d --- /dev/null +++ b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2" +} diff --git a/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json new file mode 100644 index 0000000000..7250aff598 --- /dev/null +++ b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at < now() - ($2::text)::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f" +} diff --git a/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json new file mode 100644 index 0000000000..c0207b6bc7 --- /dev/null +++ b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, part_id, storage, part_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, upload_id, part_id)\n DO UPDATE SET part_bytes = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a" +} diff --git a/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json new file mode 100644 index 0000000000..253fe79753 --- /dev/null +++ b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n -- reservation of every OTHER in-flight upload\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2\n AND created_at > now() - ($4::text)::interval\n GROUP BY upload_id) t)::bigint as \"other_reserved!\",\n -- this upload's already-recorded parts, excluding the part being (re)uploaded\n (SELECT COALESCE(SUM(part_bytes), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 AND part_id <> $3)::bigint as \"this_other_parts!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "this_other_parts!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c" +} diff --git a/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json new file mode 100644 index 0000000000..0c3fdb8d0a --- /dev/null +++ b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_xact_lock(hashtext('workspace_storage_usage'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_xact_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03" +} diff --git a/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json new file mode 100644 index 0000000000..691016a610 --- /dev/null +++ b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') as \"key!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d" +} diff --git a/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json new file mode 100644 index 0000000000..0f1bcc2271 --- /dev/null +++ b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT inflight_bytes, target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inflight_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f" +} diff --git a/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json new file mode 100644 index 0000000000..1148679230 --- /dev/null +++ b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(SUM(bytes), 0)::bigint as \"total!\",\n COALESCE(MIN(computed_at) < now() - interval '10 minutes', true) as \"stale!\"\n FROM workspace_storage_usage WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683" +} diff --git a/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json new file mode 100644 index 0000000000..96608e1bc2 --- /dev/null +++ b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, GREATEST($3::bigint, 0), to_timestamp(0))\n ON CONFLICT (workspace_id, storage)\n DO UPDATE SET bytes = GREATEST(workspace_storage_usage.bytes + $3::bigint, 0)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616" +} diff --git a/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json new file mode 100644 index 0000000000..abb128f463 --- /dev/null +++ b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT storage, bytes, computed_at FROM workspace_storage_usage\n WHERE workspace_id = $1 ORDER BY storage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "storage", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "computed_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b" +} diff --git a/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json new file mode 100644 index 0000000000..1fd1fbaece --- /dev/null +++ b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_storage_usage WHERE workspace_id = $1 AND storage != ALL($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a" +} diff --git a/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json new file mode 100644 index 0000000000..5609adaad8 --- /dev/null +++ b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3)\n GROUP BY upload_id) t)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c" +} diff --git a/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json new file mode 100644 index 0000000000..4f0731f5e4 --- /dev/null +++ b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, $3, now())\n ON CONFLICT (workspace_id, storage) DO UPDATE SET bytes = $3, computed_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5e76f395d2..45adcf5f9f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -c3852ecb36bd0be1a74c63169e513888f3347850 +bea5a8b5120d6d69cab1ad4611ebe463559bd200 diff --git a/backend/migrations/20260702064737_workspace_storage_usage.down.sql b/backend/migrations/20260702064737_workspace_storage_usage.down.sql new file mode 100644 index 0000000000..e32480a602 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_storage_usage; diff --git a/backend/migrations/20260702064737_workspace_storage_usage.up.sql b/backend/migrations/20260702064737_workspace_storage_usage.up.sql new file mode 100644 index 0000000000..d7274e8742 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.up.sql @@ -0,0 +1,17 @@ +-- Cached per-(workspace, storage) byte usage of workspace object storage, +-- refreshed by listing the storage location and adjusted optimistically as +-- uploads complete. Read on every workspace-storage write in CE builds to +-- enforce the storage quota, and by the storage_usage endpoint in all builds. +CREATE TABLE workspace_storage_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + storage VARCHAR(255) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, storage) +); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role +-- that set them (same gap as workspace_diff, notify_event, script_trigger). +GRANT ALL ON workspace_storage_usage TO windmill_user; +GRANT ALL ON workspace_storage_usage TO windmill_admin; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql new file mode 100644 index 0000000000..e72edd0b12 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_multipart_inflight; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql new file mode 100644 index 0000000000..9768eaf842 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql @@ -0,0 +1,32 @@ +-- Reservation for the parts of in-flight (initiated but not yet completed) +-- multipart uploads to workspace object storage. Uncommitted parts occupy +-- object-store capacity but are invisible to the list-based storage recount +-- until completion, so CE folds this reservation into the remaining quota to +-- bound abandoned uploads. One row per uploaded part so a re-uploaded part +-- (same part_id) replaces rather than double-counts; a part is recorded only +-- after its upstream upload succeeds. Rows are removed on successful complete +-- and lazily expired after a TTL (abort/abandon rely on the TTL, which matches +-- when the object store reaps the uncommitted parts). +-- part_id - S3 part number or Azure block id (string) +-- part_bytes - size of that part +-- target_existing_size - size of the object the upload will overwrite (0 if new), +-- credited so an overwrite only reserves the net growth +CREATE TABLE workspace_multipart_inflight ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + upload_id VARCHAR(512) NOT NULL, + part_id VARCHAR(256) NOT NULL, + storage VARCHAR(255) NOT NULL, + part_bytes BIGINT NOT NULL DEFAULT 0, + target_existing_size BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, upload_id, part_id) +); + +CREATE INDEX idx_workspace_multipart_inflight_created_at + ON workspace_multipart_inflight (created_at); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role that +-- set them (same gap as workspace_storage_usage, notify_event, script_trigger). +GRANT ALL ON workspace_multipart_inflight TO windmill_user; +GRANT ALL ON workspace_multipart_inflight TO windmill_admin; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index eec76ea009..e661c3aeb8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -19901,6 +19901,55 @@ paths: application/json: schema: {} + /w/{workspace}/job_helpers/storage_usage: + get: + summary: Get the storage usage of the workspace object storage, per configured storage. On Community Edition, also returns the workspace storage quota. + operationId: getStorageUsage + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: refresh + in: query + description: recount usage by listing the storage instead of returning cached values + schema: + type: boolean + responses: + "200": + description: Storage usage + content: + application/json: + schema: + type: object + properties: + total_bytes: + type: integer + format: int64 + quota_bytes: + type: integer + format: int64 + description: only present on Community Edition, where workspace storage is capped + storages: + type: array + items: + type: object + properties: + storage: + type: string + bytes: + type: integer + format: int64 + computed_at: + type: string + format: date-time + required: + - storage + - bytes + - computed_at + required: + - total_bytes + - storages + /w/{workspace}/job_helpers/list_stored_files: get: summary: List the file keys available in a workspace object storage diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 38fa5ff862..99d6156687 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -7,6 +7,11 @@ use std::{collections::HashMap, sync::Arc}; * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ +#[cfg(all(feature = "parquet", not(feature = "enterprise")))] +use crate::job_helpers_oss::{ + bump_storage_usage, ce_upload_budget, reject_reserved_volume_key, + spawn_storage_usage_recount_floored, +}; use crate::{ auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, @@ -3544,7 +3549,49 @@ async fn upload_s3_file_from_app( ]) .into(); - let _put_result = upload_file_from_req(s3_client, &file_key, request, options).await?; + // Only workspace storage is quota-metered; a custom-resource upload lands in + // the user's own bucket and is neither capped nor counted. An overwrite of an + // existing key only spends the difference over its current size. + let _is_workspace_storage = query.s3_resource_path.is_none(); + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + reject_reserved_volume_key(&file_key)?; + } + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + let (max_size, _existing_size) = if _is_workspace_storage { + let content_length = request + .headers() + .get(http::header::CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.parse::().ok()); + let budget = ce_upload_budget(&db, &w_id, &s3_client, &file_key, content_length).await?; + (Some(budget.max_size), budget.existing_size) + } else { + (None, 0) + }; + #[cfg(any(not(feature = "parquet"), feature = "enterprise"))] + let max_size: Option = None; + + match upload_file_from_req(s3_client, &file_key, request, options, max_size).await { + Ok((_, _size)) => + { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + bump_storage_usage( + &db, + &w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64 - _existing_size, + ) + .await; + } + } + Err(e) => { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + spawn_storage_usage_recount_floored(&db, &w_id); + return Err(e); + } + } let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims { file_key: file_key.clone(), diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index 3793e94b02..dc8b36fbaf 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -90,6 +90,10 @@ impl RawWebhookArgs { db: &DB, w_id: &str, ) -> Result>, Error> { + #[cfg(not(feature = "enterprise"))] + use crate::job_helpers_oss::{ + bump_storage_usage, ce_storage_quota_remaining, spawn_storage_usage_recount_floored, + }; use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; @@ -139,8 +143,38 @@ impl RawWebhookArgs { .into_stream() .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); - upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) - .await?; + // file_key is always freshly random here, so this never + // overwrites an existing object; the full size is the delta. + #[cfg(not(feature = "enterprise"))] + let max_size = Some(ce_storage_quota_remaining(db, w_id, None).await? as usize); + #[cfg(feature = "enterprise")] + let max_size: Option = None; + + match upload_file_internal( + s3_client.clone(), + &file_key, + bytes_stream, + options, + max_size, + ) + .await + { + Ok((_, _size)) => { + #[cfg(not(feature = "enterprise"))] + bump_storage_usage( + db, + w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64, + ) + .await; + } + Err(e) => { + #[cfg(not(feature = "enterprise"))] + spawn_storage_usage_recount_floored(db, w_id); + return Err(e); + } + } files.entry(name).or_insert(vec![]).push(serde_json::json!({ "s3": &file_key diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 23f20459c3..c6d8e397f2 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -87,7 +87,8 @@ pub async fn upload_file_from_req( _file_key: &str, _req: axum::extract::Request, _options: PutMultipartOpts, -) -> error::Result { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) @@ -99,12 +100,77 @@ pub async fn upload_file_internal( _file_key: &str, _stream: impl Stream> + Unpin, _options: PutMultipartOpts, -) -> error::Result<()> { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } +// These stubs stand in for the CE quota helpers in a pure-OSS build; their only +// callers (apps.rs / args.rs uploads) are `not(enterprise)`, so gate them the +// same way — an enterprise-without-private build compiles neither. +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_storage_quota_remaining( + _db: &DB, + _w_id: &str, + _exclude_upload_id: Option<&str>, +) -> error::Result { + Ok(i64::MAX) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn reject_reserved_volume_key(_file_key: &str) -> error::Result<()> { + Ok(()) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub struct CeUploadBudget { + pub max_size: usize, + pub existing_size: i64, +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_upload_budget( + _db: &DB, + _w_id: &str, + _s3_client: &Arc, + _file_key: &str, + _content_length: Option, +) -> error::Result { + Ok(CeUploadBudget { max_size: usize::MAX, existing_size: 0 }) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn bump_storage_usage(_db: &DB, _w_id: &str, _storage: &str, _delta: i64) {} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn spawn_storage_usage_recount_floored(_db: &DB, _w_id: &str) {} + #[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn download_s3_file_internal( _authed: OptJobAuthed, diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 329ea53e14..be49e3c564 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -1,7 +1,7 @@ + + (open = false)}> + (open = false)}> + {#snippet actions()} +
+ {#each macros as m (m.name)} +
+
+ + {macroSignature(m)} + + + {m.is_table ? 'table' : 'scalar'} + +
+ 
{m.body}
+
+ {/each} + + {/each} + + {/if} + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index c709866a5f..8b5e90170c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -10,6 +10,7 @@ Loader2, Play, RotateCw, + SquareFunction, Tag, Target, Timer, @@ -37,6 +38,9 @@ freshness?: string tag?: string retry?: { count: number; delay?: string } + // Macros this script provides (deployed/drafted `// macros` library). + // Non-empty renders the ƒ chip marking the node as a macro library. + macros?: { name: string; params: string; is_table: boolean }[] // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState @@ -199,6 +203,17 @@ ×{r.count} {/if} + {#if data.macros && data.macros.length > 0} +
1 ? 's' : ''}:\n${data.macros + .map((m) => `• ${m.name}(${m.params})${m.is_table ? ' → table' : ''}`) + .join('\n')}`} + > + + ×{data.macros.length} +
+ {/if} {#if data.runState} {@const rs = data.runState}
= { retry: true, materialize: true, dataTests: true, - columnLineage: true + columnLineage: true, + macros: true, + useLibs: true } // Parser-parity guard: this TS parser (drives the live graph preview) and @@ -78,6 +80,10 @@ type Fixture = { // Snake_case `ColumnLineage` serde shape — TS parser emits it verbatim, // so the comparison is 1:1. Absent === []. column_lineage?: Array> + // `// macros` marker. Absent === false. + macros?: boolean + // `// use ` accumulation, declaration order, deduped. Absent === []. + use_libs?: string[] } } @@ -179,6 +185,10 @@ describe('parsePipelineAnnotations matches the shared Rust fixture corpus', () = expect(got.dataTests, 'data tests').toEqual(f.expected.data_tests ?? []) expect(got.columnLineage, 'column lineage').toEqual(f.expected.column_lineage ?? []) + + expect(got.macros, 'macros').toBe(f.expected.macros ?? false) + + expect(got.useLibs, 'use_libs').toEqual(f.expected.use_libs ?? []) }) } }) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts index 5f4b063e3f..315c7526f3 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts @@ -95,6 +95,33 @@ describe('parsePipelineAnnotations: retry', () => { }) }) +describe('parsePipelineAnnotations: macros + use', () => { + it('parses the bare macros marker', () => { + const out = parsePipelineAnnotations('// macros\nCREATE MACRO m(a) AS a;') + expect(out.macros).toBe(true) + }) + + it('macros marker is strict — trailing prose and variants rejected', () => { + expect(parsePipelineAnnotations('// macros are defined below\n').macros).toBe(false) + expect(parsePipelineAnnotations('// macros_v2\n').macros).toBe(false) + expect(parsePipelineAnnotations('-- macros \nSELECT 1;').macros).toBe(true) + }) + + it('use accumulates in order and dedups', () => { + const out = parsePipelineAnnotations( + '// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\n' + ) + expect(out.useLibs).toEqual(['f/lib/stats', 'f/lib/dates']) + }) + + it('use rejects prose, slashless and multi-token values', () => { + const out = parsePipelineAnnotations( + '// use this script to compute\n// use standalone\n// use f/lib/ok extra\n' + ) + expect(out.useLibs).toEqual([]) + }) +}) + describe('parsePipelineAnnotations: combined', () => { it('parses all keywords together', () => { const code = [ diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index f58ffa54ad..f33a03fff5 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -173,6 +173,12 @@ export type PipelineAnnotations = { dataTests: DataTest[] // `// column <- .
[, …]` — accumulating column lineage. columnLineage: ColumnLineage[] + // Bare `// macros` (alone on the line, like `// pipeline`) — marks this + // DuckDB script as a workspace macro library. + macros: boolean + // `// use ` — force-inject the named macro library into + // this script's jobs. Accumulating, declaration order, deduped. + useLibs: string[] } // Tokenize a `key=value [key="quoted value"] ...` option string. Bare @@ -510,7 +516,9 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { triggerAssets: [], nativeTriggers: [], dataTests: [], - columnLineage: [] + columnLineage: [], + macros: false, + useLibs: [] } for (const rawLine of code.split('\n')) { @@ -530,6 +538,26 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { continue } + const afterMacros = consumeKeyword(inner, 'macros') + if (afterMacros !== undefined) { + // Strict like `pipeline`: keyword alone on the line, so prose such + // as `// macros are defined below` never false-positives. + if (afterMacros.trim() === '') out.macros = true + continue + } + + // `// use ` — accumulating. The argument must be a + // single whitespace-free token containing `/` (all script paths do), + // so prose like `// use this script to …` is dropped fail-safe. + const afterUse = consumeKeyword(inner, 'use') + if (afterUse !== undefined) { + const path = afterUse.trim() + if (path && !/\s/.test(path) && path.includes('/') && !out.useLibs.includes(path)) { + out.useLibs.push(path) + } + continue + } + const afterPart = consumeKeyword(inner, 'partitioned') if (afterPart !== undefined) { if (!out.partition) { diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index cbb5476516..71a09318e6 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -19,6 +19,7 @@ export type PipelineOutputKind = | 'materialize' | 's3_parquet' | 's3_object' + | 'macros' export type PipelineOutputKindMeta = { id: PipelineOutputKind @@ -56,6 +57,11 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [ label: 'S3 Object', description: 'Generic file (JSON/CSV/binary)' }, + { + id: 'macros', + label: 'Macro library', + description: 'Reusable DuckDB macros, callable from every script in the workspace' + }, { id: 'none', label: 'No output', @@ -77,7 +83,7 @@ const LANG_COMPATIBILITY: Record = { // single SELECT. The Python/TS `wmll.ducklake` helper currently takes a SQL // SELECT (not in-memory rows), so a polyglot managed materialize is a // separate follow-up — those langs keep the `ducklake` raw-write kind. - duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'none'], + duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'macros', 'none'], postgresql: ['datatable', 'none'], mysql: ['none'], mssql: ['none'], @@ -176,6 +182,9 @@ export function autoOutputAsset( path: `/pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` } } + // A macro library produces no asset — its "output" is the registry + // entries the deploy records. + case 'macros': case 'none': return undefined } @@ -330,6 +339,16 @@ function header(ctx: TemplateContext): string { `${p} Strategy: add key=to merge (upsert), or append for insert-only; default replaces the partition` ] : [] + // Macro library: the `// macros` marker registers every CREATE MACRO below + // into the workspace registry at deploy. The hint must not start with a + // parser keyword — `Consumers` is safe. + const macrosLine = + outputKind === 'macros' + ? [ + `${p} macros`, + `${p} Consumers just call these by name; add \`${p} use \` in a consumer to force-inject the whole library` + ] + : [] // Discoverability hint — the three annotations users most often miss // when authoring their first pipeline script. Single line, real // example values (not placeholders) so users see the syntax. Docs @@ -337,7 +356,7 @@ function header(ctx: TemplateContext): string { // line separates it from the parsed annotations above (`// pipeline`, // `// on …`) so the editor reads as "real annotations, then a hint". const more = `${p} More: partitioned daily, freshness 1h, retry 3, tag heavy — https://www.windmill.dev/docs/pipelines/annotations` - return [`${p} pipeline`, ...lines, ...matLine, '', more, ''].join('\n') + return [`${p} pipeline`, ...lines, ...matLine, ...macrosLine, '', more, ''].join('\n') } // Bun / Deno bodies. These share the wmill SDK surface, so we treat them @@ -636,6 +655,18 @@ function bodyDuckdb(ctx: TemplateContext): string { ) } break + case 'macros': + // Library body: only CREATE [OR REPLACE] MACRO statements (plus plain + // setup). One scalar + one table example; bodies may only call macros + // defined EARLIER in the file (DuckDB bind-checks at creation). + lines.push( + `CREATE OR REPLACE MACRO safe_div(a, b, fallback := 0) AS`, + ` CASE WHEN b = 0 THEN fallback ELSE a / b END;`, + ``, + `CREATE OR REPLACE MACRO sample_rows(src, n) AS TABLE`, + ` SELECT * FROM query_table(src) LIMIT n;` + ) + break case 'none': default: // With an uploaded file but no output asset, at least surface its diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index 378c4e1469..3c98d57602 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -9,6 +9,9 @@ const ann = (over: Partial = {}): PipelineAnnotations => ({ triggerAssets: [], nativeTriggers: [], dataTests: [], + columnLineage: [], + macros: false, + useLibs: [], ...over }) @@ -452,4 +455,85 @@ describe('resolveGraph', () => { missing: true }) }) + + it('macro edges: base passes through; live `// use` adds an unsaved via_use edge', () => { + const base = baseGraph({ + runnables: [ + { + path: 'f/lib/stats', + usage_kind: 'script', + macros: [{ name: 'safe_div', params: 'a, b', is_table: false }] + }, + { path: 'f/x/cons', usage_kind: 'script' } + ], + macro_edges: [ + { + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: false + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { + scriptPath: 'f/x/other', + annotations: ann({ useLibs: ['f/lib/stats'] }) + } + }) + ) + // Detection edge preserved untouched. + expect(r.macro_edges).toContainEqual({ + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: false + }) + // Live `// use` synthesizes an unsaved whole-lib edge with the lib's names. + expect(r.macro_edges).toContainEqual({ + lib_path: 'f/lib/stats', + consumer_path: 'f/x/other', + macro_names: ['safe_div'], + via_use: true, + unsaved: true + }) + }) + + it('macro edges: removing the `// use` line of an overlaid consumer retires its via_use edge', () => { + const base = baseGraph({ + macro_edges: [ + { + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: true + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { scriptPath: 'f/x/cons', annotations: ann() } + }) + ) + expect(r.macro_edges).toEqual([]) + }) + + it('macro edges: draft `// macros` library gets the ƒ badge data from its body', () => { + const drafts = new Map([ + [ + 'f/lib/new', + { + script: { + content: '// macros\nCREATE OR REPLACE MACRO dbl(a) AS a * 2;' + } + } + ] + ]) + const r = resolveGraph(input({ drafts })) + const lib = r.runnables.find((x) => x.path === 'f/lib/new') + expect(lib?.macros).toEqual([{ name: 'dbl', params: 'a', is_table: false }]) + }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index e223379adb..849849c555 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -1,4 +1,4 @@ -import type { AssetGraphResponse, NativeTriggerKind } from './types' +import type { AssetGraphMacroEdge, AssetGraphResponse, NativeTriggerKind } from './types' import { mergeColumnLineage, parsePipelineAnnotations, @@ -146,15 +146,94 @@ export function resolveGraph(input: ResolveGraphInput): AssetGraphResponse { return false return true }) + // Mirror the backend's skip-if-empty: no `macro_edges` key at all when + // there is nothing to show (also keeps the no-macros response shape + // byte-identical to before the feature). + const macroEdges = resolveMacroEdges(input) return { ...base, assets: acc.assets, runnables: acc.runnables, edges: acc.edges, - triggers: [...baseTriggers, ...acc.extraTriggers] + triggers: [...baseTriggers, ...acc.extraTriggers], + ...(macroEdges.length > 0 || base.macro_edges ? { macro_edges: macroEdges } : {}) } } +/** + * Macro-library → consumer edges: the deployed base edges, with `// use` + * declarations of overlaid scripts (drafts + the open buffer) taking over + * their consumer's `via_use` edges so adding/removing a `// use` line updates + * the canvas live. Detection-based edges (macro calls in the deployed body) + * are backend-owned and only refresh on redeploy. + */ +function resolveMacroEdges(input: ResolveGraphInput): AssetGraphMacroEdge[] { + const { base, drafts, liveAnnotations } = input + const libMacroNames = new Map() + for (const r of base.runnables) { + if (r.usage_kind === 'script' && r.macros?.length) { + libMacroNames.set( + r.path, + r.macros.map((m) => m.name) + ) + } + } + const useByPath = new Map() + for (const [path, d] of drafts) { + useByPath.set(path, parsePipelineAnnotations(d.script.content).useLibs) + } + if (liveAnnotations.scriptPath) { + // `?? []` — callers may hand a minimal annotations object (tests, older + // call sites) that predates the field. + useByPath.set(liveAnnotations.scriptPath, liveAnnotations.annotations.useLibs ?? []) + } + const out: AssetGraphMacroEdge[] = [] + for (const e of base.macro_edges ?? []) { + if (e.via_use && useByPath.has(e.consumer_path)) continue + out.push({ ...e }) + } + for (const [path, libs] of useByPath) { + for (const lib of libs) { + const existing = out.find((e) => e.lib_path === lib && e.consumer_path === path) + if (existing) { + // Upgrade the detection edge in place: `// use` pulls in the whole + // library, so the edge covers every macro the lib defines. + existing.via_use = true + existing.unsaved = true + existing.macro_names = [ + ...new Set([...existing.macro_names, ...(libMacroNames.get(lib) ?? [])]) + ] + } else { + out.push({ + lib_path: lib, + consumer_path: path, + macro_names: libMacroNames.get(lib) ?? [], + via_use: true, + unsaved: true + }) + } + } + } + return out +} + +// Light regex extraction of a draft macro library's definitions for the live +// node badge. The strict grammar lives in the Rust `parse_macro_library` at +// deploy; this only needs name/params/table-ness for display (nested parens +// in a default value may truncate the shown signature, never the deploy). +const MACRO_DEF_RE = + /create\s+(?:or\s+replace\s+)?(?:temp(?:orary)?\s+)?(?:macro|function)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]*)\)\s*as\s+(table\b)?/gi + +export function extractDraftMacros( + content: string +): { name: string; params: string; is_table: boolean }[] { + const out: { name: string; params: string; is_table: boolean }[] = [] + for (const m of content.matchAll(MACRO_DEF_RE)) { + out.push({ name: m[1].toLowerCase(), params: m[2].trim(), is_table: m[3] !== undefined }) + } + return out +} + type ResolveContext = { draftedPaths: Set isDrafted: (kind: string, p: string) => boolean @@ -251,6 +330,9 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // duplicate (which would crash svelte-flow's keyed each), so the // canvas + trigger-node labels reflect that there's pending body // editing for this path. + // `// macros` library draft: extract the definitions for the live node + // badge (regex-light; the strict parse happens at deploy). + const draftMacros = parsed.macros ? extractDraftMacros(d.script.content) : [] const baseIdx = runnables.findIndex((r) => r.usage_kind === 'script' && r.path === path) if (baseIdx === -1) { runnables.push({ @@ -264,6 +346,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, column_lineage: mergedCL.length > 0 ? mergedCL : undefined, materialize_target: materializeTarget, + macros: draftMacros.length > 0 ? draftMacros : undefined, unsaved: true }) } else { @@ -276,6 +359,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, column_lineage: mergedCL.length > 0 ? mergedCL : undefined, materialize_target: materializeTarget, + macros: draftMacros.length > 0 ? draftMacros : undefined, unsaved: true } } diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 274d218e82..0db259a389 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -46,6 +46,11 @@ export interface AssetGraphRunnableNode { // produced asset's schema can evolve: only whole-table `replace` can, since // `append`/`merge`/partitioned writes INSERT into a fixed-schema table. materialize_strategy?: 'replace' | 'append' | 'merge' + // Macros this script provides to the workspace registry (deployed + // `// macros` library). Non-empty marks the node as a macro library; + // drives the "defines N macros" badge and the details-pane signature + // list. `params` is the verbatim parameter list. + macros?: { name: string; params: string; is_table: boolean }[] // Synthesized by the page from a local draft; the script doesn't exist // in the DB yet. Drives a dashed/lower-opacity rendering to mirror how // unsaved triggers are styled — visually distinct from persisted nodes. @@ -113,11 +118,24 @@ export type AssetGraphTrigger = missing?: boolean } +// Macro-library → consumer edge: the consumer calls `macro_names` of +// `lib_path`'s macros (deploy-recorded detection), or pulls in the whole +// library via `// use` (`via_use`, macro_names then lists the full library). +// `unsaved: true` marks a draft's `// use` overlay. +export interface AssetGraphMacroEdge { + lib_path: string + consumer_path: string + macro_names: string[] + via_use: boolean + unsaved?: boolean +} + export interface AssetGraphResponse { assets: AssetGraphAssetNode[] runnables: AssetGraphRunnableNode[] edges: AssetGraphEdge[] triggers: AssetGraphTrigger[] + macro_edges?: AssetGraphMacroEdge[] } export type AssetGraphNodeData = diff --git a/frontend/src/lib/components/assets/workspaceMacros.ts b/frontend/src/lib/components/assets/workspaceMacros.ts new file mode 100644 index 0000000000..d25f9d1bea --- /dev/null +++ b/frontend/src/lib/components/assets/workspaceMacros.ts @@ -0,0 +1,32 @@ +import { AssetService, type ListWorkspaceMacrosResponse } from '$lib/gen' + +export type WorkspaceMacro = ListWorkspaceMacrosResponse[number] + +// Workspace macros are late-bound (the worker reads the registry per job), so +// mild staleness in editor surfaces is harmless — a short TTL keeps repeated +// editor mounts / drawer opens from refetching on every keystroke-driven +// remount while still picking up a lib deploy within seconds. +const TTL_MS = 30_000 +const cache = new Map() + +export async function listWorkspaceMacrosCached(workspace: string): Promise { + const hit = cache.get(workspace) + if (hit && Date.now() - hit.at < TTL_MS) return hit.items + const items = await AssetService.listWorkspaceMacros({ workspace }) + cache.set(workspace, { at: Date.now(), items }) + return items +} + +export function invalidateWorkspaceMacros(workspace: string) { + cache.delete(workspace) +} + +/** `name(params)` display signature, with the table-macro arrow. */ +export function macroSignature(m: WorkspaceMacro): string { + return `${m.name}(${m.params})${m.is_table ? ' → table' : ''}` +} + +/** Full `CREATE` statement for the copy button / documentation preview. */ +export function macroDefinitionSql(m: WorkspaceMacro): string { + return `CREATE OR REPLACE MACRO ${m.name}(${m.params}) AS ${m.is_table ? 'TABLE ' : ''}${m.body};` +} diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 47b4bf6abc..4d41f7ff05 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -29,6 +29,7 @@ PipelineMode } from '$lib/components/assets/AssetGraph/types' import PipelineModeToggle from '$lib/components/assets/AssetGraph/PipelineModeToggle.svelte' + import MacroExplorerDrawer from '$lib/components/assets/AssetGraph/MacroExplorerDrawer.svelte' import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' import { buildColumnGraph, @@ -83,6 +84,7 @@ Play, RefreshCw, Save, + SquareFunction, Target, Telescope } from 'lucide-svelte' @@ -218,6 +220,24 @@ } } + // Workspace-macro explorer drawer (all `// macros` libraries + their + // signatures/bodies). "Open" on a group selects the lib node when it's on + // this canvas; a lib living in another folder opens its script page. + let macroDrawer: MacroExplorerDrawer | undefined = $state() + function openMacroLib(path: string) { + const onCanvas = displayGraph?.runnables?.some( + (r) => r.usage_kind === 'script' && r.path === path + ) + if (onCanvas) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', path } + pe.activeDraftPath = undefined + panelHidden = false + focusPipelineNode(`script:${path}`) + } else { + window.open(`${base}/scripts/get/${path}`, '_blank') + } + } + // Draft autosave (the data_pipeline DraftService bundle) lives inside // PipelineGraphEditor now; the route just supplies its path to the indicator. let pipelineDraftPath = $derived(`f/${folder}/data_pipeline`) @@ -233,7 +253,9 @@ triggerAssets: [], nativeTriggers: [], dataTests: [], - columnLineage: [] + columnLineage: [], + macros: false, + useLibs: [] } } @@ -2105,6 +2127,15 @@ Activity {/if} +
+ + + + + + + + + {#each snapshots as s (s.asset)} + + + + + + {/each} + +
AssetSnapshotTime travel
+ {s.asset} + + @ {s.snapshot_id} + {#if s.partition} + + · partition {s.partition} + {/if} + + +
+
+
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 6ca81a5005..14acd3c89c 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -42,6 +42,7 @@ import DisplayResult from '$lib/components/DisplayResult.svelte' import DispatchEventsPanel from '$lib/components/runs/DispatchEventsPanel.svelte' + import UpstreamSnapshotsPanel from '$lib/components/runs/UpstreamSnapshotsPanel.svelte' import { enterpriseLicense, initialArgsStore, @@ -918,6 +919,7 @@ {#if job.id && job.workspace_id} + {/if} {/if} From 5d7fb6deca3e02e89d77e5d3856483beb8b8bfeb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 04:23:44 +0000 Subject: [PATCH 185/273] =?UTF-8?q?feat(pipelines):=20asset=20freshness=20?= =?UTF-8?q?=E2=80=94=20fresh/stale=20badge=20(CE)=20+=20watchdog=20(EE)=20?= =?UTF-8?q?(#9909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): passive asset freshness tracking on the graph Co-Authored-By: Claude Fable 5 * chore(pipelines): drop dead freshness-enforcement stub, document query ordering Co-Authored-By: Claude Fable 5 * feat(pipelines): freshness watchdog (EE) — auto re-run stale producers Co-Authored-By: Claude Fable 5 * fix(pipelines): watchdog review fixes — archived workspaces, badge kind parity, scan index Co-Authored-By: Claude Fable 5 * fix(pipelines): CI review — no singlestepflow in freshness, +N parity, completion-time fallback Co-Authored-By: Claude Fable 5 * fix(pipelines): CI review — history completedAt, freshness/asset trigger UI metadata Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 6f5fe0f7f56696fbef5a8349da38496c32e71666 This commit updates the EE repository reference after PR #643 was merged in windmill-ee-private. Previous ee-repo-ref: 1f13380354bf591ae25a2c20d36917534bcc5459 New ee-repo-ref: 6f5fe0f7f56696fbef5a8349da38496c32e71666 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- ...bc7e9ee85289ff5c52af43166928337c257e7.json | 3 +- ...91f3afd87eeea5e4e8d935b1b86eb02f061e7.json | 15 + ...adbb2860ce52d37a56b2861de1215260ecff8.json | 6 +- ...cb3adcf0175d0a71013f2497397c1c8ccc619.json | 3 +- ...66b49254036ca93f7a6415ceb4e316ed64eed.json | 29 ++ ...de8b6fcebe83002c74647666e3a6f5b8490dd.json | 23 ++ ...5ff3ae182ff7c0631d4fa44b67b11a133498f.json | 17 + ...c23caf312ec377dd91e5307f00d3fb8ec325d.json | 3 +- ...07667fc1b5e802dc2465ffd13178a042e4f89.json | 24 ++ ...f6d740a1720f7746c01f9ea693c70271ff55c.json | 17 + ...04af5a7eb488c76f783e690af0c2153b1d6a8.json | 3 +- ...6017386cf2767c953e0d152e7983a00218c6f.json | 15 + ...34128c3a6ade024d6662379857905a7447a2a.json | 32 ++ ...a995a8a555e4a9c2d1387dd8cee9c81efd40b.json | 15 + ...fbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json | 3 +- ...e33244fe8ab9a803248fc23f334034d24aad4.json | 3 +- ...5bd601b7dcabdf70e85543455cb4fc3986d74.json | 23 ++ ...bb6a8f961c119516e4c33dc928cd3b4f4edbc.json | 3 +- ...39a26f649c960a4ee8d0a584884a236129d43.json | 15 + ...77429c51e6166c414bcc5ce2e97fac25bcd77.json | 3 +- ...9746147aa9b8d0a81642a7b7cb4738a0cad66.json | 3 +- ...ad8bc7a2d226a9f22418782bcc66e46d0f706.json | 15 + ...e1ea2fe96a1b94f5a92953fb1c1642d15c016.json | 3 +- ...120e5b0f629f9f04e08a2982df33ff23ac7a9.json | 3 +- ...9352d4d1e3cb3a8a08f1c9c03055a1cab1235.json | 3 +- ...a088b5c0c1ad601146ff1c57d4a3ddc30556a.json | 34 ++ ...dba01f2f3a02bb36a8d7c20e1990dc81e0472.json | 22 ++ backend/ee-repo-ref.txt | 2 +- ...65613_pipeline_freshness_watchdog.down.sql | 5 + ...3165613_pipeline_freshness_watchdog.up.sql | 35 +++ .../windmill-parser/src/asset_parser.rs | 12 +- backend/src/monitor.rs | 31 ++ backend/tests/freshness_watchdog.rs | 294 ++++++++++++++++++ backend/windmill-api-assets/src/lib.rs | 57 ++++ backend/windmill-api-scripts/src/scripts.rs | 10 - backend/windmill-api/openapi.yaml | 1 + backend/windmill-common/src/assets.rs | 4 + .../src/pipeline_advanced_oss.rs | 16 +- .../src/freshness_watchdog_oss.rs | 9 + backend/windmill-queue/src/lib.rs | 7 + backend/windmill-types/src/jobs.rs | 4 + .../assets/AssetGraph/AssetGraphCanvas.svelte | 1 + .../assets/AssetGraph/RunnableNode.svelte | 64 +++- .../AssetGraph/activeRunnables.svelte.ts | 49 ++- .../parsePipelineAnnotations.test.ts | 38 +++ .../AssetGraph/parsePipelineAnnotations.ts | 22 ++ .../AssetGraph/pipelineHistory.svelte.ts | 10 +- .../lib/components/assets/AssetGraph/types.ts | 5 + frontend/src/lib/components/triggers/utils.ts | 30 +- .../(logged)/pipeline/[folder]/+page.svelte | 13 +- 50 files changed, 998 insertions(+), 59 deletions(-) create mode 100644 backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json create mode 100644 backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json create mode 100644 backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json create mode 100644 backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json create mode 100644 backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json create mode 100644 backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json create mode 100644 backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json create mode 100644 backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json create mode 100644 backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json create mode 100644 backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json create mode 100644 backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json create mode 100644 backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json create mode 100644 backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json create mode 100644 backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json create mode 100644 backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql create mode 100644 backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql create mode 100644 backend/tests/freshness_watchdog.rs create mode 100644 backend/windmill-queue/src/freshness_watchdog_oss.rs diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index a779aa0e95..6efb66005d 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -35,7 +35,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json b/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json new file mode 100644 index 0000000000..906c710fdf --- /dev/null +++ b/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7" +} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json index f9bc97d29d..704883d4f1 100644 --- a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -39,7 +39,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } @@ -77,7 +78,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 3568d1723e..f5ee767768 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -128,7 +128,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json b/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json new file mode 100644 index 0000000000..60dbee76ab --- /dev/null +++ b/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT p.path AS \"path!\",\n (SELECT c.completed_at\n FROM v2_job j\n JOIN v2_job_completed c ON c.id = j.id\n WHERE j.workspace_id = $1\n AND j.runnable_path = p.path\n AND j.parent_job IS NULL\n -- No 'singlestepflow': flows may share a script's path, and\n -- a same-path flow run must not read as the script being\n -- fresh (false-fresh). Script retries land as native\n -- 'script' jobs; only the rare flow-wrapper fallback is\n -- missed, which errs stale. Kept in lockstep with the\n -- freshness watchdog's queries (freshness_watchdog_ee).\n AND j.kind IN ('script', 'preview')\n AND c.status = 'success'\n ORDER BY j.created_at DESC\n LIMIT 1) AS last_success_at\n FROM unnest($2::text[]) AS p(path)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "last_success_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed" +} diff --git a/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json b/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json new file mode 100644 index 0000000000..5b3e70acbf --- /dev/null +++ b/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.runnable_path = $2\n AND j.parent_job IS NULL\n AND j.kind IN ('script', 'preview')\n AND (q.running = true OR q.scheduled_for <= now())\n ) AS \"in_flight!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "in_flight!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd" +} diff --git a/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json b/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json new file mode 100644 index 0000000000..6dc5fdb9a6 --- /dev/null +++ b/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE pipeline_freshness_state\n SET attempts = attempts + 1,\n last_push_at = now(),\n next_attempt_at = now()\n + (LEAST($3::bigint, $4::bigint * (1::bigint << LEAST(attempts + 1, 20)))::text\n || ' seconds')::interval\n WHERE workspace_id = $1 AND script_path = $2 AND next_attempt_at <= now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f" +} diff --git a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json index 7d950f6d8f..7474818bf3 100644 --- a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json +++ b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json @@ -80,7 +80,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json b/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json new file mode 100644 index 0000000000..580666f07d --- /dev/null +++ b/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 FROM v2_job j\n JOIN v2_job_completed c ON c.id = j.id\n WHERE j.workspace_id = $1\n AND j.runnable_path = $2\n AND j.parent_job IS NULL\n AND j.kind IN ('script', 'preview')\n AND c.status = 'success'\n AND c.completed_at > now() - ($3::bigint::text || ' seconds')::interval\n ) AS \"fresh!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "fresh!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89" +} diff --git a/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json b/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json new file mode 100644 index 0000000000..18e0936d3a --- /dev/null +++ b/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO pipeline_freshness_state\n (workspace_id, script_path, attempts, last_push_at, next_attempt_at)\n VALUES ($1, $2, 1, now(),\n now() + (LEAST($3::bigint, $4::bigint * 2)::text || ' seconds')::interval)\n ON CONFLICT (workspace_id, script_path) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c" +} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json index 08ebe6bba5..fcc16e9a7c 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json @@ -161,7 +161,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json b/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json new file mode 100644 index 0000000000..4e8c7acee2 --- /dev/null +++ b/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE pipeline_freshness_state SET next_attempt_at = now() - interval '1 second'\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f" +} diff --git a/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json b/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json new file mode 100644 index 0000000000..5c8ab6fb5b --- /dev/null +++ b/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT ON (workspace_id, path)\n workspace_id AS \"workspace_id!\", path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND content ILIKE '%freshness%'\n -- Workspace archival stops all execution but leaves script rows\n -- intact for unarchival; without this the watchdog would keep\n -- resurrecting runs in a workspace the admin shut down.\n AND EXISTS (SELECT 1 FROM workspace w\n WHERE w.id = script.workspace_id AND w.deleted = false)\n ORDER BY workspace_id, path, created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "content!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a" +} diff --git a/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json b/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json new file mode 100644 index 0000000000..3f6a4a8136 --- /dev/null +++ b/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM pipeline_freshness_state s\n WHERE NOT EXISTS (\n SELECT 1 FROM unnest($1::text[], $2::text[]) AS w(workspace_id, script_path)\n WHERE w.workspace_id = s.workspace_id AND w.script_path = s.script_path\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b" +} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index ad9e57801e..ab01730c02 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -35,7 +35,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index 9a21f228ea..405904604a 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -191,7 +191,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json b/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json new file mode 100644 index 0000000000..d5b10b13b9 --- /dev/null +++ b/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT attempts FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "attempts", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index 3efa843923..092d15e592 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -166,7 +166,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json b/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json new file mode 100644 index 0000000000..ce77275faf --- /dev/null +++ b/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pipeline_freshness_state (workspace_id, script_path, attempts) VALUES ($1, $2, 3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43" +} diff --git a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json index f6ff25a4bf..419ab26383 100644 --- a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json +++ b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json @@ -80,7 +80,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 8dc66064dc..b9d33a6b5f 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -111,7 +111,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json b/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json new file mode 100644 index 0000000000..cbf96dad52 --- /dev/null +++ b/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pipeline_freshness_state (workspace_id, script_path) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706" +} diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index d5365ffe94..d97c02d26b 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -111,7 +111,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json index 33a5534b42..052d83fcd9 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json @@ -251,7 +251,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 470c651020..a35373a959 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -191,7 +191,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json b/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json new file mode 100644 index 0000000000..8c80c14ea4 --- /dev/null +++ b/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", created_by AS \"created_by!\",\n args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'freshness'\n ORDER BY created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false, + true + ] + }, + "hash": "e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a" +} diff --git a/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json b/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json new file mode 100644 index 0000000000..8257722c5c --- /dev/null +++ b/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) AS \"count!\" FROM pipeline_freshness_state WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 857e48ba09..0cd164d1ce 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -81d8a449effdc540a6e8810668dd5d4aea0c485a +6f5fe0f7f56696fbef5a8349da38496c32e71666 diff --git a/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql b/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql new file mode 100644 index 0000000000..aed2f9d32a --- /dev/null +++ b/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql @@ -0,0 +1,5 @@ +-- Postgres has no ALTER TYPE ... DROP VALUE for enums. The 'freshness' value +-- stays even on rollback, consistent with prior job_trigger_kind additions +-- (see 20260510174213_asset_trigger_dispatch). +DROP INDEX IF EXISTS idx_script_pipeline_freshness_scan; +DROP TABLE IF EXISTS pipeline_freshness_state; diff --git a/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql b/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql new file mode 100644 index 0000000000..acf9545c96 --- /dev/null +++ b/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql @@ -0,0 +1,35 @@ +-- Attribution for runs pushed by the pipeline freshness watchdog (the EE +-- background loop that re-runs a `// freshness`-annotated producer whose +-- output aged past its window). +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'freshness'; + +-- Per-(workspace, script) watchdog state: exponential-backoff bookkeeping so +-- a persistently failing producer isn't re-pushed on every scan tick, and an +-- atomic claim so concurrent servers can't double-push in the same tick +-- (claim = the UPDATE/INSERT that advances next_attempt_at; only the winner +-- pushes). Rows exist only while a script is stale — observing it fresh (or +-- its annotation gone) deletes the row, resetting the backoff. +CREATE TABLE pipeline_freshness_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + script_path VARCHAR(510) NOT NULL, + attempts INTEGER NOT NULL DEFAULT 1, + last_push_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path) +); + +-- Written only by the server monitor loop on the raw (non-RLS) pool, but +-- granted like every other app table so a future user-transaction reader +-- doesn't hit the recurring missing-GRANT class of bug. +GRANT ALL ON pipeline_freshness_state TO windmill_user; +GRANT ALL ON pipeline_freshness_state TO windmill_admin; + +-- The watchdog's ~60s candidate scan (latest deployed pipeline members) +-- filters on this exact predicate and orders by (workspace_id, path, +-- created_at DESC); without a matching partial index it seq-scans the whole +-- script-version heap on every tick, on instances that mostly have zero +-- pipeline scripts. (idx_script_pipeline_path is text_pattern_ops for +-- prefix LIKE — it can't serve this ordering.) +CREATE INDEX idx_script_pipeline_freshness_scan + ON script (workspace_id, path, created_at DESC) + WHERE auto_kind = 'pipeline' AND archived = false AND deleted = false; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index c22359f6b0..299e3a70f9 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -77,10 +77,12 @@ pub struct ParseAssetsOutput { #[serde(skip_serializing_if = "Option::is_none", default)] pub partition: Option, // `// freshness ` — SLA stating outputs must be at most - // `duration` old. Active backstop: when no other trigger has fired the - // script within the window, a watchdog re-runs it. Distinct from - // schedule (which is producer cadence); freshness is consumer SLA and - // applies regardless of which trigger last fired. + // `duration` old. Drives passive monitoring in CE (the asset graph + // colors the node's badge fresh/stale against its last successful run) + // and the enterprise watchdog (windmill-queue `freshness_watchdog`), + // which re-runs a stale unpartitioned producer. Distinct from schedule + // (which is producer cadence); freshness is consumer SLA and applies + // regardless of which trigger last fired. #[serde(skip_serializing_if = "Option::is_none", default)] pub freshness: Option, // `// trigger all` → AND join barrier; default (`any`) = OR (current @@ -637,7 +639,7 @@ fn parse_kv_opts(s: &str) -> BTreeMap { // - `on ` → asset / native trigger edge (including // the marker-only `on schedule` form) // - `partitioned [opts]` → partition declaration -// - `freshness ` → SLA / active backstop +// - `freshness ` → SLA window (badge + EE watchdog) // - `tag ` → worker-tag override (annotation wins // over UI-set value at deploy) // - `retry []` → cascade-only retry policy diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 817887eaad..60b4024c0a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -178,6 +178,14 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + // Ops kill switch for the pipeline freshness watchdog (a background + // pusher — being able to stop it without a redeploy matters more than + // for read-only monitors). + pub static ref DISABLE_FRESHNESS_WATCHDOG: bool = std::env::var("DISABLE_FRESHNESS_WATCHDOG") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + pub static ref WORKERS_NAMES: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); @@ -2994,6 +3002,28 @@ pub async fn monitor_db( } }; + // run every ~60s (2 iterations * 30s). Enterprise feature: the active + // `// freshness` backstop lives in windmill-queue's `freshness_watchdog` + // (`private`); OSS gets a no-op stub. Runtime-gated on an Enterprise + // license like the audit export above. Safe on concurrent servers — the + // watchdog claims per-script state rows atomically before pushing. + let pipeline_freshness_watchdog_f = async { + if server_mode + && !*DISABLE_FRESHNESS_WATCHDOG + && iteration.is_some() + && iteration.as_ref().unwrap().should_run(2) + { + if let Some(db) = conn.as_sql() { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) { + windmill_queue::freshness_watchdog::tick(db).await; + } + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -3021,6 +3051,7 @@ pub async fn monitor_db( manage_audit_partitions_f, export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, + pipeline_freshness_watchdog_f, ); } diff --git a/backend/tests/freshness_watchdog.rs b/backend/tests/freshness_watchdog.rs new file mode 100644 index 0000000000..17851d0a1e --- /dev/null +++ b/backend/tests/freshness_watchdog.rs @@ -0,0 +1,294 @@ +//! End-to-end tests for the pipeline freshness watchdog (Enterprise). +//! +//! `windmill_queue::freshness_watchdog::tick` is called directly against +//! seeded `script` / `v2_job(_completed)` rows — no worker or API server is +//! needed, since the watchdog's job ends at the push (the pushed job sitting +//! in `v2_job_queue` is itself part of the assertions). Covers: staleness on +//! never-ran and aged-out members, the fresh short-circuit + state reset, +//! the in-flight suppression, the backoff claim, and the skip rules +//! (partitioned, malformed window, non-pipeline scripts). + +#![cfg(feature = "private")] + +use sqlx::{Pool, Postgres}; +use windmill_queue::freshness_watchdog::tick; +use windmill_test_utils::initialize_tracing; + +const WS: &str = "test-workspace"; +const PATH: &str = "u/test-user/freshness_producer"; + +/// Seed a deployed pipeline-member script. Mirrors the deploy path's output: +/// `auto_kind = 'pipeline'`, empty (non-NULL) lock so run-by-path resolution +/// treats it as deployed, hash derived from path+content for uniqueness. +async fn seed_pipeline_script( + db: &Pool, + path: &str, + content: &str, +) -> anyhow::Result<()> { + let mut h = 0i64; + for b in path.bytes().chain(content.bytes()) { + h = h.wrapping_mul(31).wrapping_add(b as i64); + } + sqlx::query( + r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, tag, lock, auto_kind) + VALUES ($1, $2, $3, '', '', $4, 'test-user', 'bash'::script_lang, 'bash', '', 'pipeline') + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(h) + .bind(path) + .bind(content) + .execute(db) + .await?; + // Process-global deployed-script caches are keyed by (workspace, path) / + // (workspace, hash) and would leak between #[sqlx::test] isolated DBs + // that reuse both — resolve everything from this test's own DB. + windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) +} + +/// Seed a completed root run of `path` that finished `age_s` seconds ago. +async fn seed_completed_run( + db: &Pool, + path: &str, + age_s: i64, + success: bool, +) -> anyhow::Result<()> { + let id = uuid::Uuid::new_v4(); + sqlx::query( + r#"INSERT INTO v2_job (id, workspace_id, runnable_path, kind, created_at, + created_by, permissioned_as, permissioned_as_email, tag) + VALUES ($1, $2, $3, 'script'::job_kind, + now() - ($4::bigint::text || ' seconds')::interval, + 'test-user', 'u/test-user', 'test@windmill.dev', 'bash')"#, + ) + .bind(id) + .bind(WS) + .bind(path) + .bind(age_s) + .execute(db) + .await?; + sqlx::query( + r#"INSERT INTO v2_job_completed (id, workspace_id, duration_ms, status, started_at, completed_at) + VALUES ($1, $2, 0, CASE WHEN $3 THEN 'success'::job_status ELSE 'failure'::job_status END, + now() - ($4::bigint::text || ' seconds')::interval, + now() - ($4::bigint::text || ' seconds')::interval)"#, + ) + .bind(id) + .bind(WS) + .bind(success) + .bind(age_s) + .execute(db) + .await?; + Ok(()) +} + +/// Jobs the watchdog pushed: (path, created_by, args) rows attributed to +/// `trigger_kind = 'freshness'`. +async fn fetch_pushed( + db: &Pool, +) -> anyhow::Result)>> { + let rows = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", created_by AS "created_by!", + args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'freshness' + ORDER BY created_at"#, + WS, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.runnable_path, r.created_by, r.args.map(|a| a.0))) + .collect()) +} + +async fn state_row(db: &Pool, path: &str) -> anyhow::Result> { + Ok(sqlx::query_scalar!( + "SELECT attempts FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path, + ) + .fetch_optional(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn never_ran_member_is_pushed_once(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 30s\necho hi\n").await?; + + tick(&db).await; + + let pushed = fetch_pushed(&db).await?; + assert_eq!(pushed.len(), 1, "one watchdog push expected"); + let (path, created_by, args) = &pushed[0]; + assert_eq!(path, PATH); + assert_eq!(created_by, &format!("freshness-{PATH}")); + let args = args.as_ref().expect("args recorded"); + assert_eq!( + args.get("_wmill_skip_asset_dispatch"), + Some(&serde_json::json!(true)), + "watchdog runs must not re-fire the cascade" + ); + assert_eq!( + args.pointer("/trigger/kind"), + Some(&serde_json::json!("freshness")) + ); + assert_eq!(state_row(&db, PATH).await?, Some(1), "claim row recorded"); + + // Second tick: the pushed job is queued-and-due, so the in-flight guard + // suppresses a duplicate regardless of backoff. + tick(&db).await; + assert_eq!( + fetch_pushed(&db).await?.len(), + 1, + "no duplicate while queued" + ); + + // Simulate the queued job vanishing without a completion: the backoff + // claim (next_attempt_at in the future) now carries the suppression. + sqlx::query!("DELETE FROM v2_job_queue WHERE workspace_id = $1", WS) + .execute(&db) + .await?; + tick(&db).await; + assert_eq!(fetch_pushed(&db).await?.len(), 1, "backoff holds the retry"); + + // Force the backoff window open: the watchdog retries and escalates. + sqlx::query!( + "UPDATE pipeline_freshness_state SET next_attempt_at = now() - interval '1 second' + WHERE workspace_id = $1 AND script_path = $2", + WS, + PATH, + ) + .execute(&db) + .await?; + tick(&db).await; + assert_eq!(fetch_pushed(&db).await?.len(), 2, "due retry pushed"); + assert_eq!(state_row(&db, PATH).await?, Some(2), "attempts escalated"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn fresh_member_is_skipped_and_state_reset(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 1h\necho hi\n").await?; + seed_completed_run(&db, PATH, 10, true).await?; + // Leftover backoff row from an earlier staleness episode. + sqlx::query!( + "INSERT INTO pipeline_freshness_state (workspace_id, script_path, attempts) VALUES ($1, $2, 3)", + WS, + PATH, + ) + .execute(&db) + .await?; + + tick(&db).await; + + assert!( + fetch_pushed(&db).await?.is_empty(), + "fresh member not pushed" + ); + assert_eq!(state_row(&db, PATH).await?, None, "backoff reset on fresh"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn aged_out_member_is_pushed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 1h\necho hi\n").await?; + // Old success outside the window + a recent failure: still stale. + seed_completed_run(&db, PATH, 7200, true).await?; + seed_completed_run(&db, PATH, 60, false).await?; + + tick(&db).await; + + assert_eq!(fetch_pushed(&db).await?.len(), 1, "aged-out member pushed"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn partitioned_malformed_and_plain_members_are_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Partitioned: freshness means partition-gap detection, out of scope. + seed_pipeline_script( + &db, + "u/test-user/partitioned", + "# pipeline\n# partitioned daily\n# freshness 1h\necho hi\n", + ) + .await?; + // Malformed window: fails safe to unwatched. + seed_pipeline_script( + &db, + "u/test-user/malformed", + "# pipeline\n# freshness soonish\necho hi\n", + ) + .await?; + // Freshness only in prose (parser must reject; ILIKE prefilter passes). + seed_pipeline_script( + &db, + "u/test-user/prose", + "# pipeline\n# ensure freshness of data below\necho hi\n", + ) + .await?; + + tick(&db).await; + + assert!(fetch_pushed(&db).await?.is_empty(), "no member is watched"); + let rows = sqlx::query_scalar!( + r#"SELECT COUNT(*) AS "count!" FROM pipeline_freshness_state WHERE workspace_id = $1"#, + WS, + ) + .fetch_one(&db) + .await?; + assert_eq!(rows, 0, "no state rows for unwatched members"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn archived_workspace_is_not_resurrected(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // Workspace archival stops all execution but keeps script rows for + // unarchival — the watchdog must not keep pushing runs there. + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 30s\necho hi\n").await?; + sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", WS) + .execute(&db) + .await?; + + tick(&db).await; + + assert!( + fetch_pushed(&db).await?.is_empty(), + "no pushes into an archived workspace" + ); + assert_eq!( + state_row(&db, PATH).await?, + None, + "no state bookkeeping either" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn state_of_unwatched_member_is_cleaned_up(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // A stale-bookkeeping row whose script no longer declares freshness + // (e.g. annotation removed and redeployed) must not survive the sweep. + sqlx::query!( + "INSERT INTO pipeline_freshness_state (workspace_id, script_path) VALUES ($1, $2)", + WS, + "u/test-user/gone", + ) + .execute(&db) + .await?; + + tick(&db).await; + + assert_eq!(state_row(&db, "u/test-user/gone").await?, None); + Ok(()) +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index b4b0583863..2eb24cee6c 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -632,6 +632,14 @@ struct GraphRunnableNode { partition_kind: Option, #[serde(skip_serializing_if = "Option::is_none", default)] freshness: Option, + // Completion time of the most recently started successful run of this + // pipeline member. The canvas checks it against the `// freshness` window + // to color the badge fresh/stale. The badge itself is passive; on EE the + // freshness watchdog (windmill-queue) separately re-runs stale + // unpartitioned producers. Absent when no successful run is visible to + // the caller (job RLS applies). + #[serde(skip_serializing_if = "Option::is_none", default)] + last_success_at: Option>, #[serde(skip_serializing_if = "Option::is_none", default)] tag: Option, #[serde(skip_serializing_if = "Option::is_none", default)] @@ -920,6 +928,46 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // Newest successful completed run per pipeline member, for the passive + // freshness status on the canvas. Correlated per-path lookup walks + // ix_job_root_job_index_by_path_2 newest-first until the first success, + // so cost is bounded by the member count, not run history. "Newest" is + // by created_at (the index order), not completed_at: with overlapping + // runs of one path this can pick an earlier completion, erring toward + // stale — never toward false-fresh. Inside the user tx so job-visibility + // RLS applies — a caller who can't see the runs gets no timestamp rather + // than leaked completion times. + let member_paths: Vec = pipeline_member_paths + .iter() + .map(|r| r.path.clone()) + .collect(); + let last_success_rows = sqlx::query!( + r#" + SELECT p.path AS "path!", + (SELECT c.completed_at + FROM v2_job j + JOIN v2_job_completed c ON c.id = j.id + WHERE j.workspace_id = $1 + AND j.runnable_path = p.path + AND j.parent_job IS NULL + -- No 'singlestepflow': flows may share a script's path, and + -- a same-path flow run must not read as the script being + -- fresh (false-fresh). Script retries land as native + -- 'script' jobs; only the rare flow-wrapper fallback is + -- missed, which errs stale. Kept in lockstep with the + -- freshness watchdog's queries (freshness_watchdog_ee). + AND j.kind IN ('script', 'preview') + AND c.status = 'success' + ORDER BY j.created_at DESC + LIMIT 1) AS last_success_at + FROM unnest($2::text[]) AS p(path) + "#, + &w_id, + &member_paths, + ) + .fetch_all(&mut *tx) + .await?; + // Existing scripts / flows in the workspace. Used to filter out // orphan trigger rows whose `script_path` no longer resolves — those // would otherwise be added to `runnable_set` below and surface as @@ -1010,6 +1058,11 @@ async fn asset_graph( (r.path.clone(), lineage) }) .collect(); + let last_success_by_path: std::collections::HashMap> = + last_success_rows + .into_iter() + .filter_map(|r| r.last_success_at.map(|t| (r.path, t))) + .collect(); let pipeline_member_script_paths: std::collections::HashSet = pipeline_member_paths.into_iter().map(|r| r.path).collect(); let existing_script_paths: std::collections::HashSet = @@ -1215,6 +1268,10 @@ async fn asset_graph( freshness: ann .and_then(|a| a.freshness.as_ref()) .map(|f| f.duration.clone()), + last_success_at: (usage_kind == AssetUsageKind::Script) + .then(|| last_success_by_path.get(&path)) + .flatten() + .copied(), tag: ann.and_then(|a| a.tag.clone()), retry: ann.and_then(|a| a.retry.clone()), data_tests: ann.map(|a| a.data_tests.clone()).unwrap_or_default(), diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 4f9de4f416..49a4091f07 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1252,16 +1252,6 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// freshness` is parsed but enforcement is a not-yet-implemented - // enterprise feature (skeleton in windmill_common::pipeline_advanced). - // Surface a clear TODO at deploy rather than silently accepting an - // annotation that does nothing. - if pipeline_annotations.freshness.is_some() { - tracing::warn!( - "{}", - windmill_common::pipeline_advanced::freshness_enforcement_todo() - ); - } // `// materialize` materializes a `ducklake:///` target from a // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake // target would otherwise deploy, register a producer in the asset graph, then diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 01ae7229e6..fbad5fffd4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -25592,6 +25592,7 @@ components: - google - github - asset + - freshness TriggerMode: description: job trigger mode diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 0757ef6894..722ad29747 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -352,6 +352,10 @@ mod debounce_duration_tests { assert_eq!(parse_duration_secs("5m"), Some(300)); assert_eq!(parse_duration_secs("2h"), Some(7200)); assert_eq!(parse_duration_secs(" 1d "), Some(86400)); + // Explicit plus sign comes free with i64 parsing; the TS mirror + // (parseDurationSecs) matches it — keep the two in lockstep. + assert_eq!(parse_duration_secs("+5m"), Some(300)); + assert_eq!(parse_duration_secs("+45"), Some(45)); } #[test] diff --git a/backend/windmill-common/src/pipeline_advanced_oss.rs b/backend/windmill-common/src/pipeline_advanced_oss.rs index 009e783bd5..15bb4b127a 100644 --- a/backend/windmill-common/src/pipeline_advanced_oss.rs +++ b/backend/windmill-common/src/pipeline_advanced_oss.rs @@ -1,16 +1,12 @@ -//! OSS fallback: pipeline freshness/SLA enforcement and partition backfills -//! are enterprise features; their implementations live in windmill-ee-private -//! (see `pipeline_advanced_ee`). In the public build the entry points report -//! that the enterprise edition is required. +//! OSS fallback: pipeline partition backfills are an enterprise feature; +//! their implementations live in windmill-ee-private (see +//! `pipeline_advanced_ee`). In the public build the entry points report that +//! the enterprise edition is required. (Freshness lives elsewhere: the +//! fresh/stale badge is CE in the assets API, the active watchdog is +//! windmill-queue's `freshness_watchdog`.) use crate::error::Error; -pub fn freshness_enforcement_todo() -> Error { - Error::internal_err( - "Pipeline freshness/SLA enforcement requires the enterprise edition".to_string(), - ) -} - pub fn backfill_todo() -> Error { Error::internal_err("Pipeline partition backfill requires the enterprise edition".to_string()) } diff --git a/backend/windmill-queue/src/freshness_watchdog_oss.rs b/backend/windmill-queue/src/freshness_watchdog_oss.rs new file mode 100644 index 0000000000..319fb25f62 --- /dev/null +++ b/backend/windmill-queue/src/freshness_watchdog_oss.rs @@ -0,0 +1,9 @@ +//! OSS fallback for the pipeline freshness watchdog. The active backstop — +//! re-running a `// freshness`-annotated producer whose output aged past its +//! window — is an enterprise feature (see `freshness_watchdog_ee`). In the +//! public build the tick is a no-op; CE keeps the passive fresh/stale badge +//! on the asset graph. + +use windmill_common::DB; + +pub async fn tick(_db: &DB) {} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 6f689c7026..59ac7f1fa8 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -14,6 +14,13 @@ pub mod cascade_oss; pub use cascade_ee as cascade; #[cfg(not(feature = "private"))] pub use cascade_oss as cascade; +#[cfg(feature = "private")] +pub mod freshness_watchdog_ee; +pub mod freshness_watchdog_oss; +#[cfg(feature = "private")] +pub use freshness_watchdog_ee as freshness_watchdog; +#[cfg(not(feature = "private"))] +pub use freshness_watchdog_oss as freshness_watchdog; pub mod jobs; #[cfg(feature = "private")] pub mod jobs_ee; diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 448e6f4d01..f9372b949d 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -46,6 +46,9 @@ pub enum JobTriggerKind { // A run dispatched because an upstream pipeline script wrote an asset // this runnable subscribes to via `// on s3://...` annotations. Asset, + // A run pushed by the pipeline freshness watchdog (EE) because the + // script's `// freshness` window elapsed without a successful run. + Freshness, } impl std::fmt::Display for JobTriggerKind { @@ -68,6 +71,7 @@ impl std::fmt::Display for JobTriggerKind { JobTriggerKind::Github => "github", JobTriggerKind::CiTest => "ci_test", JobTriggerKind::Asset => "asset", + JobTriggerKind::Freshness => "freshness", }; write!(f, "{}", kind) } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 1c5cf2dde6..6352e6adc9 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -401,6 +401,7 @@ in_pipeline: r.in_pipeline ?? false, partition_kind: r.partition_kind, freshness: r.freshness, + last_success_at: r.last_success_at, tag: r.tag, retry: r.retry, macros: r.macros, diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index 8b5e90170c..6bbf8f5656 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -22,12 +22,13 @@ import { preventDefault, stopPropagation } from 'svelte/legacy' import type { GraphUsageKind } from './types' import type { RunnableRunState } from './activeRunnables.svelte' + import { parseDurationSecs } from './parsePipelineAnnotations' import { NODE } from '$lib/components/graph/util' import DropdownV2 from '$lib/components/DropdownV2.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' import type { Item } from '$lib/utils' import { workspaceStore } from '$lib/stores' - import { sendUserToast } from '$lib/utils' + import { sendUserToast, msToReadableTimeShort } from '$lib/utils' interface Props { data: { @@ -36,6 +37,9 @@ in_pipeline?: boolean partition_kind?: 'daily' | 'hourly' | 'weekly' | 'monthly' | 'dynamic' freshness?: string + // Completion time (ISO) of the newest successful run visible to + // the caller. With `freshness`, drives the fresh/stale chip state. + last_success_at?: string tag?: string retry?: { count: number; delay?: string } // Macros this script provides (deployed/drafted `// macros` library). @@ -122,6 +126,44 @@ } } + // Freshness verdict: newest successful run (server `last_success_at`, + // or a newer one the session poll observed) vs the `// freshness` + // window. No verdict (undefined) for drafts — no run history — and for + // unparseable windows; the chip then stays neutral like the other + // annotation chips. + let freshnessWindowS = $derived(data.freshness ? parseDurationSecs(data.freshness) : undefined) + // Ticks so a node crosses fresh→stale while the canvas stays open (the + // graph payload is static between refetches). Armed only when a verdict + // is rendered. + let nowMs = $state(Date.now()) + $effect(() => { + if (freshnessWindowS === undefined || data.unsaved) return + const id = setInterval(() => (nowMs = Date.now()), 30_000) + return () => clearInterval(id) + }) + let lastSuccessMs = $derived.by(() => { + const server = data.last_success_at ? new Date(data.last_success_at).getTime() : undefined + const polled = data.runState?.lastSuccessAt + ? new Date(data.runState.lastSuccessAt).getTime() + : undefined + if (server === undefined) return polled + return polled === undefined ? server : Math.max(server, polled) + }) + let freshnessState = $derived.by((): 'fresh' | 'stale' | undefined => { + if (freshnessWindowS === undefined || data.unsaved) return undefined + if (lastSuccessMs === undefined) return 'stale' + return nowMs - lastSuccessMs <= freshnessWindowS * 1000 ? 'fresh' : 'stale' + }) + let freshnessTooltip = $derived.by(() => { + const base = `// freshness ${data.freshness}` + if (freshnessState === undefined) return base + if (lastSuccessMs === undefined) return `${base} — stale: no successful run yet` + const ago = msToReadableTimeShort(Math.max(0, nowMs - lastSuccessMs)) + return freshnessState === 'fresh' + ? `${base} — fresh: last successful run ${ago} ago` + : `${base} — stale: last successful run ${ago} ago` + }) + // Cascade + bounded-run options live on the Run button's caret popover // (whenever there's a cascade OR a bounded-run start — see `hasCaret` // below), so the kebab menu stays focused on lifecycle actions only. @@ -164,8 +206,8 @@ + guidelines). Only the freshness chip (when it has a verdict) + and the run-state chip below use semantic colors. --> {#if data.partition_kind}
{data.partition_kind}
{/if} + {#if data.freshness}
{data.freshness} diff --git a/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts index 917e21766a..f6682804c7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts @@ -1,8 +1,15 @@ import { JobService } from '$lib/gen' export type RunStatus = 'running' | 'success' | 'failure' -/** Per-runnable badge state: latest run status + runs observed this session. */ -export type RunnableRunState = { status: RunStatus; runs: number } +/** + * Per-runnable badge state: latest run status + runs observed this session. + * `lastSuccessAt` is the completion time (start + duration when the listing + * carries it, else start as a conservative lower bound) of the newest + * successful run seen by the poll — lets the freshness chip go green right + * after an in-session run, ahead of the next graph refetch (whose + * `last_success_at` would carry it). + */ +export type RunnableRunState = { status: RunStatus; runs: number; lastSuccessAt?: string } export type EventStatus = 'queued' | 'running' | 'success' | 'failure' /** One folder activity-log row (a job observed by the poll). */ @@ -14,6 +21,12 @@ export type PipelineEvent = { /** What started it, as far as the job listing reveals. */ source: 'schedule' | 'run' at: string + /** + * Completion time (start + duration) for completed rows. The freshness + * chip compares against completion — `at` is the start time and would + * read a long run as older than its output actually is. + */ + completedAt?: string /** * Queued jobs: when the job is due to start. A future value means a * scheduled run waiting for its cron tick, not pipeline activity. @@ -45,7 +58,8 @@ function statesEq(a: Map, b: Map() const countedJobIds = new Set() // Job ids we've observed in-flight at least once. The catch-up pulse is @@ -241,10 +255,25 @@ export function useActiveRunnableIds( const prev = completedHistory.get(id) const status: RunStatus = (j as any).success === true ? 'success' : 'failure' const ts = startedTs ?? new Date(pollStartedMs).toISOString() + // Freshness compares against COMPLETION time (that's + // when the output materialized — the server-side + // last_success_at is completed_at too). The listing + // only carries started_at, so add duration_ms; when + // absent, the start is a conservative lower bound + // (errs stale, never false-fresh). + const durationMs = (j as any).duration_ms + const doneTs = + typeof durationMs === 'number' && startedTs + ? new Date(new Date(startedTs).getTime() + durationMs).toISOString() + : ts completedHistory.set(id, { runs: (prev?.runs ?? 0) + 1, lastStatus: !prev || ts >= prev.lastTs ? status : prev.lastStatus, - lastTs: !prev || ts >= prev.lastTs ? ts : prev.lastTs + lastTs: !prev || ts >= prev.lastTs ? ts : prev.lastTs, + lastSuccessTs: + status === 'success' && (!prev?.lastSuccessTs || doneTs >= prev.lastSuccessTs) + ? doneTs + : prev?.lastSuccessTs }) } } @@ -267,6 +296,10 @@ export function useActiveRunnableIds( : 'failure', source: (j as any).schedule_path ? 'schedule' : 'run', at: startedTs ?? new Date(pollStartedMs).toISOString(), + completedAt: + !isQueued && typeof (j as any).duration_ms === 'number' && startedTs + ? new Date(new Date(startedTs).getTime() + (j as any).duration_ms).toISOString() + : undefined, scheduledFor: isQueued ? ((j as any).scheduled_for as string | undefined) : undefined }) } @@ -289,7 +322,11 @@ export function useActiveRunnableIds( // previous badge state until a worker picks the job up. const snap = new Map() for (const [id, h] of completedHistory) { - snap.set(id, { status: runningThisTick.has(id) ? 'running' : h.lastStatus, runs: h.runs }) + snap.set(id, { + status: runningThisTick.has(id) ? 'running' : h.lastStatus, + runs: h.runs, + lastSuccessAt: h.lastSuccessTs + }) } for (const id of runningThisTick) { if (!snap.has(id)) snap.set(id, { status: 'running', runs: 0 }) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts index 315c7526f3..e9e23525ec 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { mergeColumnLineage, + parseDurationSecs, parsePipelineAnnotations, type ColumnLineage } from './parsePipelineAnnotations' @@ -188,3 +189,40 @@ describe('mergeColumnLineage', () => { expect(mergeColumnLineage([], annotated)).toEqual(annotated) }) }) + +// Mirror of the Rust `parse_duration_secs` tests (windmill-common assets.rs) +// — the freshness chip's staleness verdict depends on identical parsing. +describe('parseDurationSecs', () => { + it('parses suffixed durations', () => { + expect(parseDurationSecs('30s')).toBe(30) + expect(parseDurationSecs('5m')).toBe(300) + expect(parseDurationSecs('2h')).toBe(7200) + expect(parseDurationSecs('1d')).toBe(86400) + }) + + it('bare integer means seconds', () => { + expect(parseDurationSecs('45')).toBe(45) + }) + + it('tolerates surrounding whitespace', () => { + expect(parseDurationSecs(' 5 m ')).toBe(300) + }) + + it('accepts an explicit plus sign (Rust i64 parsing does)', () => { + expect(parseDurationSecs('+5m')).toBe(300) + expect(parseDurationSecs('+45')).toBe(45) + }) + + it('rejects malformed / non-positive input', () => { + expect(parseDurationSecs('')).toBeUndefined() + expect(parseDurationSecs('h')).toBeUndefined() + expect(parseDurationSecs('1.5h')).toBeUndefined() + expect(parseDurationSecs('-5m')).toBeUndefined() + expect(parseDurationSecs('0')).toBeUndefined() + expect(parseDurationSecs('fast')).toBeUndefined() + }) + + it('rejects values beyond i32 seconds (mirrors backend cap)', () => { + expect(parseDurationSecs('999999999d')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index f33a03fff5..964e48570a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -67,6 +67,28 @@ export type FreshnessSpec = { duration: string } +// Mirrors backend `parse_duration_secs` (windmill-common assets.rs): a bare +// integer means seconds, otherwise `` with an `s`/`m`/`h`/`d` suffix +// (e.g. `30s`, `5m`, `2h`, `1d`). Returns undefined for malformed or +// non-positive input so a typo'd `// freshness` window fails safe (the chip +// stays neutral instead of guessing a staleness verdict). +export function parseDurationSecs(s: string): number | undefined { + const t = s.trim() + if (!t) return undefined + const last = t[t.length - 1] + const mult = + last === 's' ? 1 : last === 'm' ? 60 : last === 'h' ? 3600 : last === 'd' ? 86400 : undefined + const num = (mult !== undefined ? t.slice(0, -1) : t).trim() + // `+?`: Rust's i64 parsing accepts an explicit plus sign (`+5m`), so the + // mirror must too — divergence here would leave the chip neutral for a + // window the deploy path and watchdog honor. + if (mult === undefined && !/^\+?\d+$/.test(t)) return undefined + if (!/^\+?\d+$/.test(num)) return undefined + const secs = Number(num) * (mult ?? 1) + if (!Number.isSafeInteger(secs) || secs <= 0 || secs > 2147483647) return undefined + return secs +} + // `// retry []` — see backend RetrySpec. Delay is kept as the // raw duration string and resolved to seconds at deploy. export type RetrySpec = { diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts index 50ad3e1d6e..d654085179 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts @@ -80,7 +80,15 @@ export function usePipelineHistory( kind: j.job_kind.startsWith('flow') ? 'flow' : 'script', status: j.success ? 'success' : 'failure', source: j.schedule_path ? 'schedule' : 'run', - at: j.started_at ?? j.created_at + at: j.started_at ?? j.created_at, + // Same completion-time derivation as the live poll — + // the freshness chip compares against completion, and + // `at` (start) would read a long run as older than its + // output actually is. + completedAt: + j.started_at != undefined + ? new Date(new Date(j.started_at).getTime() + j.duration_ms).toISOString() + : undefined }) } sawFullPage = rows.length === PER_PAGE diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 0db259a389..fb7b5b1de7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -22,6 +22,11 @@ export interface AssetGraphRunnableNode { // Raw `// freshness ` value, e.g. "1h", "30m". Surfaced for // the badge; the runtime parses it as needed. freshness?: string + // Completion time (ISO) of the newest successful run of this pipeline + // member visible to the caller. The freshness chip compares it against + // the `// freshness` window to render fresh/stale. Absent = no + // successful run found (or none visible under job RLS). + last_success_at?: string // `// tag ` worker-tag override. Surfaced for the badge so users // can see which worker pool will pick this script up at a glance. tag?: string diff --git a/frontend/src/lib/components/triggers/utils.ts b/frontend/src/lib/components/triggers/utils.ts index 231de5cea4..3d17491791 100644 --- a/frontend/src/lib/components/triggers/utils.ts +++ b/frontend/src/lib/components/triggers/utils.ts @@ -1,4 +1,14 @@ -import { Webhook, Mail, Calendar, Route, Unplug, Database, Terminal } from 'lucide-svelte' +import { + Webhook, + Mail, + Calendar, + Route, + Unplug, + Database, + Terminal, + Timer, + Zap +} from 'lucide-svelte' import KafkaIcon from '$lib/components/icons/KafkaIcon.svelte' import NatsIcon from '$lib/components/icons/NatsIcon.svelte' import MqttIcon from '$lib/components/icons/MqttIcon.svelte' @@ -82,7 +92,8 @@ export const jobTriggerKinds: JobTriggerKind[] = [ 'azure', 'google', 'github', - 'asset' + 'asset', + 'freshness' ] export type Trigger = { @@ -118,7 +129,12 @@ export const triggerIconMap = { cli: Terminal, nextcloud: NextcloudIcon, google: GoogleIcon, - github: GithubIcon + github: GithubIcon, + // Job-attribution-only kinds (no trigger CRUD page): the pipeline asset + // cascade and the freshness watchdog. Needed so the Runs filter and job + // detail render these trigger kinds instead of a blank label / no icon. + asset: Zap, + freshness: Timer } export const triggerDisplayNamesMap = { @@ -139,8 +155,12 @@ export const triggerDisplayNamesMap = { cli: 'CLI', nextcloud: 'Nextcloud', google: 'Google', - github: 'GitHub' -} as const satisfies Record + github: 'GitHub', + asset: 'Asset cascade', + freshness: 'Freshness' + // `asset` / `freshness` are job-attribution-only (JobTriggerKind, not + // TriggerType) — hence the union in the satisfies below. +} as const satisfies Record /** * Converts a TriggerType to a CaptureTriggerKind when a mapping exists diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 4d41f7ff05..845a93505b 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -1337,8 +1337,17 @@ ? 'failure' : 'success' const cur = m.get(id) - if (cur) cur.runs += 1 - else m.set(id, { status, runs: 1 }) + // Freshness compares against completion; `at` (start) is the + // fallback lower bound for rows without a duration. + const successAt = e.status === 'success' ? (e.completedAt ?? e.at) : undefined + if (cur) { + cur.runs += 1 + // Newest-first, so the first success per id is the latest one — + // it feeds the freshness chip between graph refetches. + if (successAt && !cur.lastSuccessAt) cur.lastSuccessAt = successAt + } else { + m.set(id, { status, runs: 1, lastSuccessAt: successAt }) + } } return m }) From 42e11c6570b62ffaa86598438fa8ddf462c4035f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 08:40:00 +0000 Subject: [PATCH 186/273] =?UTF-8?q?feat(pipelines):=20schema=20contracts?= =?UTF-8?q?=20=E2=80=94=20save-time=20consumer=20checks=20vs=20captured=20?= =?UTF-8?q?schemas=20(#9917)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): schema contracts — save-time consumer checks vs captured schemas Co-Authored-By: Claude Fable 5 * refactor: move schemaContractContext above schemaCanEvolve doc comment Co-Authored-By: Claude Fable 5 * fix: emit scd2/on_schema_change in CLI local graph, address review notes Co-Authored-By: Claude Fable 5 * fix: gate editor _current ignore-suppression on scd2, matching backend Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- ...368a6862e35350ae43383fb2335e2c82a9419.json | 29 + ...eb5a0665c90c4926f1afb689d6cd16bd5b722.json | 29 + ...3ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json | 41 ++ .../windmill-parser/src/asset_parser.rs | 67 ++ .../tests/fixtures/pipeline_annotations.json | 37 ++ .../tests/pipeline_annotations_parity.rs | 15 + backend/windmill-api-assets/src/lib.rs | 17 + backend/windmill-api-scripts/src/scripts.rs | 53 ++ backend/windmill-api/openapi.yaml | 77 +++ backend/windmill-common/src/assets.rs | 2 +- backend/windmill-common/src/lib.rs | 1 + .../windmill-common/src/schema_contracts.rs | 614 ++++++++++++++++++ cli/src/commands/pipeline/localGraph.ts | 33 +- docs/ducklake-materialization.md | 31 + docs/pipelines-vs-dbt.md | 44 +- frontend/src/lib/components/Editor.svelte | 91 ++- .../src/lib/components/ScriptBuilder.svelte | 6 + .../src/lib/components/ScriptEditor.svelte | 43 +- .../AssetGraph/AssetGraphDetailsPane.svelte | 12 +- .../AssetGraph/PipelineGraphEditor.svelte | 6 + .../parsePipelineAnnotations.parity.test.ts | 5 + .../AssetGraph/parsePipelineAnnotations.ts | 13 +- .../assets/AssetGraph/schemaContracts.test.ts | 276 ++++++++ .../assets/AssetGraph/schemaContracts.ts | 440 +++++++++++++ .../lib/components/assets/AssetGraph/types.ts | 11 +- .../(logged)/pipeline/[folder]/+page.svelte | 8 + 26 files changed, 1972 insertions(+), 29 deletions(-) create mode 100644 backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json create mode 100644 backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json create mode 100644 backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json create mode 100644 backend/windmill-common/src/schema_contracts.rs create mode 100644 frontend/src/lib/components/assets/AssetGraph/schemaContracts.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts diff --git a/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json b/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json new file mode 100644 index 0000000000..d05abd4f3a --- /dev/null +++ b/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path AS \"asset_path!\", usage_path AS \"producer_path!\"\n FROM asset\n WHERE workspace_id = $1 AND kind = 'ducklake' AND path = ANY($2)\n AND usage_kind = 'script' AND usage_access_type IN ('w', 'rw')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "producer_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419" +} diff --git a/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json b/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json new file mode 100644 index 0000000000..b554ef8b55 --- /dev/null +++ b/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, content\n FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND archived = false AND deleted = false\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722" +} diff --git a/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json b/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json new file mode 100644 index 0000000000..7cfc1f109f --- /dev/null +++ b/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (asset_path)\n asset_path, version, columns AS \"columns: Json>\", captured_at\n FROM materialized_asset_schema\n WHERE workspace_id = $1 AND asset_kind = 'ducklake' AND asset_path = ANY($2)\n ORDER BY asset_path, version DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "columns: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "captured_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896" +} diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 299e3a70f9..3e6d433405 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -252,6 +252,8 @@ pub struct RetrySpec { // history (`valid_from`/`valid_to`/`is_current`). The leading keyword `scd2` is a // recognized alias for `history`. `deletes=close` (scd2 only) also closes a key // that disappears from the snapshot; default leaves absent keys current. +// `on_schema_change=ignore` suppresses downstream schema-contract warnings for +// the produced asset (save-time metadata only; default `warn`). #[derive(Serialize, Debug, PartialEq, Clone)] pub struct MaterializeSpec { pub target_kind: AssetKind, @@ -276,6 +278,32 @@ pub struct MaterializeSpec { // `hard_deletes=close`). Default (false) leaves absent keys current. #[serde(skip_serializing_if = "std::ops::Not::not", default)] pub close_deleted: bool, + // `on_schema_change=ignore` opts this producer's asset out of downstream + // schema-contract warnings (gap #2b): consumers referencing columns the + // captured schema no longer has warn by default (`warn`); `ignore` declares + // the schema deliberately unstable and suppresses those warnings. Save-time + // metadata only — the materialize write strategy is unaffected. + #[serde(skip_serializing_if = "OnSchemaChange::is_warn", default)] + pub on_schema_change: OnSchemaChange, +} + +// dbt's `on_schema_change` narrowed to the save-time contract check: `warn` +// (default) surfaces consumer warnings, `ignore` suppresses them. dbt's `fail` +// is deliberately not offered — saves are never hard-blocked (a deliberate +// upstream reshape must not fail every consumer save); a CI/CLI gate can layer +// it on later without touching the grammar. +#[derive(Serialize, Debug, PartialEq, Eq, Clone, Copy, Default)] +#[serde(rename_all = "lowercase")] +pub enum OnSchemaChange { + #[default] + Warn, + Ignore, +} + +impl OnSchemaChange { + pub fn is_warn(&self) -> bool { + matches!(self, OnSchemaChange::Warn) + } } // `// data_test …` — a data-quality assertion run against the @@ -917,6 +945,14 @@ fn parse_materialize_spec(s: &str) -> Option { // `deletes=close` (scd2 only) opts into hard-delete-close; any other value // (or absence) keeps the soft-delete default. let close_deleted = opts.get("deletes").map(|v| v == "close").unwrap_or(false); + // `on_schema_change=ignore` suppresses downstream contract warnings; any + // other value (or absence) keeps the `warn` default, fail-safe like + // `deletes=` above. + let on_schema_change = if opts.get("on_schema_change").map(String::as_str) == Some("ignore") { + OnSchemaChange::Ignore + } else { + OnSchemaChange::Warn + }; Some(MaterializeSpec { target_kind, target_path: path.to_string(), @@ -926,6 +962,7 @@ fn parse_materialize_spec(s: &str) -> Option { scd2, track, close_deleted, + on_schema_change, }) } @@ -1623,6 +1660,36 @@ mod pipeline_annotation_tests { assert!(!out.materialize.expect("materialize").close_deleted); } + #[test] + fn materialize_on_schema_change_opt() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/orders on_schema_change=ignore", + ); + let m = out.materialize.expect("materialize"); + assert_eq!(m.on_schema_change, OnSchemaChange::Ignore); + // default is warn + let out = parse_pipeline_annotations("// materialize ducklake://a/orders"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Warn + ); + // unknown value keeps the warn default (fail-safe, like `deletes=`); + // `fail` is deliberately unrecognized in v1 + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders on_schema_change=fail"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Warn + ); + // composes with other opts + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history on_schema_change=ignore", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert_eq!(m.on_schema_change, OnSchemaChange::Ignore); + } + #[test] fn materialize_key_without_history_is_plain_merge() { let out = parse_pipeline_annotations("// materialize ducklake://a/dim key=id"); diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index f7977239c1..5ee2945d18 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -310,6 +310,43 @@ } } }, + { + "name": "materialize on_schema_change=ignore opt", + "code": "// pipeline\n// materialize ducklake://analytics/orders on_schema_change=ignore\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders", + "on_schema_change": "ignore" + } + } + }, + { + "name": "materialize on_schema_change unknown value keeps warn default (fail unrecognized in v1)", + "code": "// materialize ducklake://analytics/orders key=id on_schema_change=fail\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders", + "unique_key": "id", + "on_schema_change": "warn" + } + } + }, { "name": "materialize key without history is plain merge (SCD1, not scd2)", "code": "// materialize ducklake://analytics/dim key=id\nSELECT 1;", diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 2df4acd2e4..b51b33faf8 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -71,6 +71,13 @@ struct ExpectedMaterialize { track: Vec, #[serde(default)] close_deleted: bool, + // "warn" | "ignore"; absent === "warn" (the default). + #[serde(default = "default_on_schema_change")] + on_schema_change: String, +} + +fn default_on_schema_change() -> String { + "warn".to_string() } #[derive(Deserialize)] @@ -209,6 +216,14 @@ fn pipeline_annotation_fixtures_match() { m.close_deleted, e.close_deleted, "{ctx}: materialize close_deleted" ); + let osc = match m.on_schema_change { + windmill_parser::asset_parser::OnSchemaChange::Warn => "warn", + windmill_parser::asset_parser::OnSchemaChange::Ignore => "ignore", + }; + assert_eq!( + osc, e.on_schema_change, + "{ctx}: materialize on_schema_change" + ); } (got, want) => panic!( "{ctx}: materialize mismatch — got {:?}, want present={}", diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 2eb24cee6c..587307afbb 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -664,6 +664,13 @@ struct GraphRunnableNode { // `merge` / any partitioned write INSERTs into a fixed-schema table. #[serde(skip_serializing_if = "Option::is_none", default)] materialize_strategy: Option, + // `on_schema_change=ignore` on the managed materialize — the producer's + // opt-out from downstream schema-contract warnings. Threaded to the editor + // so its client-side contract mirror suppresses the same warnings the + // server check does. Only serialized when set to `ignore` (default `warn` + // is absent). Lockstep with TS `AssetGraphRunnableNode.materialize_on_schema_change`. + #[serde(skip_serializing_if = "Option::is_none", default)] + materialize_on_schema_change: Option, // Macros this script provides to the workspace registry (deployed // `// macros` library). Drives the library node state + details-pane // signature list. Lockstep with TS `AssetGraphRunnableNode.macros`. @@ -1289,8 +1296,12 @@ async fn asset_graph( } }), materialize_strategy: ann.and_then(|a| a.materialize.as_ref()).and_then(|m| { + // Precedence mirrors the runtime strategy derivation: + // scd2 (`history`) > append > merge (`key=`) > replace. if m.manual { None + } else if m.scd2 { + Some("scd2".to_string()) } else if m.append { Some("append".to_string()) } else if m.unique_key.is_some() { @@ -1299,6 +1310,12 @@ async fn asset_graph( Some("replace".to_string()) } }), + materialize_on_schema_change: ann + .and_then(|a| a.materialize.as_ref()) + .filter(|m| { + m.on_schema_change == windmill_common::assets::OnSchemaChange::Ignore + }) + .map(|_| "ignore".to_string()), macros: (usage_kind == AssetUsageKind::Script) .then(|| macros_by_provider.get(&path)) .flatten() diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 49a4091f07..e38d1925f0 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -141,6 +141,8 @@ pub fn workspaced_service() -> Router { // CI test results .route("/ci_test_results/{kind}/{*path}", get(get_ci_test_results)) .route("/ci_test_results_batch", post(get_ci_test_results_batch)) + // Save-time schema-contract check (pipelines gap #2b) + .route("/check_schema_contracts", post(check_schema_contracts)) } #[derive(Serialize, FromRow)] @@ -1305,6 +1307,14 @@ async fn create_script_internal<'c>( ns.path ); } + // `manual` materialize never captures a schema (no wrap codegen), so + // there is no contract for `on_schema_change` to mute downstream. + if m.manual && m.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Ignore { + tracing::warn!( + "script {}: `on_schema_change=ignore` on a `manual` materialize is inert — manual mode captures no schema, so consumers have no contract to check", + ns.path + ); + } } // `// macros` — this script is a workspace macro library: its body is // CREATE [OR REPLACE] MACRO statements plus plain setup, registered into @@ -3966,3 +3976,46 @@ async fn get_ci_test_results_batch( Ok(Json(result_map)) } + +#[derive(Deserialize)] +struct CheckSchemaContractsRequest { + language: ScriptLang, + content: String, +} + +#[derive(Serialize)] +struct CheckSchemaContractsResponse { + warnings: Vec, +} + +// Save-time schema-contract check (pipelines gap #2b): validate the given +// script content's asset references (body column reads, `// column` lineage, +// `// data_test relationships`) against the latest captured producer schemas +// and return WARNINGS — never errors, and deploy never blocks on this. The +// frontend calls it right after a successful deploy (post-commit, so a +// self-produced target resolves to the fresh content) and the editor mirrors +// the same diff client-side; this endpoint is the authoritative check. Parsing +// uses the same server path as deploy (`effective_script_assets` + +// `parse_pipeline_annotations`) so the verdict matches what deployed. +async fn check_schema_contracts( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + let assets = crate::asset_inference::effective_script_assets(&req.language, &req.content, None) + .unwrap_or_default(); + let ann = parse_pipeline_annotations(&req.content); + let mut tx = user_db.begin(&authed).await?; + let warnings = windmill_common::schema_contracts::check_schema_contracts( + &mut tx, + &w_id, + &assets, + &ann.column_lineage, + &ann.data_tests, + ann.materialize.as_ref(), + ) + .await?; + tx.commit().await?; + Ok(Json(CheckSchemaContractsResponse { warnings })) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fbad5fffd4..1f836cfb2b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9187,6 +9187,50 @@ paths: items: $ref: "#/components/schemas/CiTestResult" + /w/{workspace}/scripts/check_schema_contracts: + post: + summary: check a script's asset references against captured producer schemas + description: | + Save-time schema-contract check for data pipelines: validates the given + script content's asset references (body column reads, `// column` lineage, + `// data_test relationships`) against the latest captured producer schemas + and returns warnings. Warnings never block a save/deploy; an asset whose + producer declares `on_schema_change=ignore` is suppressed to a single + informational entry. + operationId: checkSchemaContracts + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - language + - content + properties: + language: + $ref: "#/components/schemas/ScriptLang" + content: + type: string + responses: + "200": + description: contract warnings (empty when all references match) + content: + application/json: + schema: + type: object + required: + - warnings + properties: + warnings: + type: array + items: + $ref: "#/components/schemas/ContractWarning" + /w/{workspace}/scripts/raw_temp/store: post: summary: store raw script content temporarily for CLI lock generation @@ -29903,6 +29947,39 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + ContractWarning: + description: | + One save-time schema-contract warning: a consumer reference that does + not match the referenced asset's latest captured schema. + `schema_version`/`captured_at` identify the capture the check ran + against (as-of the producer's last run, not its latest save). + type: object + required: [kind, asset_path, message] + properties: + kind: + type: string + enum: + - missing_column + - missing_lineage_source + - missing_relationship_column + - relationship_type_mismatch + - suppressed + asset_path: + type: string + column: + type: string + expected_type: + type: string + found_type: + type: string + schema_version: + type: integer + format: int64 + captured_at: + type: string + format: date-time + message: + type: string Volume: type: object required: diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 722ad29747..63309a1400 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -6,7 +6,7 @@ use crate::{error, scripts::ScriptHash}; pub use windmill_parser::asset_parser::{ merge_column_lineage, parse_pipeline_annotations, ColumnLineage, ColumnRef, DataTest, - PartitionKind, PipelineAnnotations, RetrySpec, TriggerSpec, PARTITION_TOKEN, + OnSchemaChange, PartitionKind, PipelineAnnotations, RetrySpec, TriggerSpec, PARTITION_TOKEN, }; pub use windmill_types::assets::*; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index baa3e4b500..acf97734d8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -65,6 +65,7 @@ pub mod materialization; pub mod min_version; pub mod notify_events; pub mod runtime_assets; +pub mod schema_contracts; pub mod workspace_dependencies; #[cfg(feature = "private")] diff --git a/backend/windmill-common/src/schema_contracts.rs b/backend/windmill-common/src/schema_contracts.rs new file mode 100644 index 0000000000..b605aae33b --- /dev/null +++ b/backend/windmill-common/src/schema_contracts.rs @@ -0,0 +1,614 @@ +//! Save-time schema-contract check (pipelines gap #2b): validate a consumer +//! script's asset references against the latest *captured* producer schema +//! (`materialized_asset_schema`, written post-materialize by #2a) and return +//! WARNINGS — never errors. A deliberate upstream reshape must not fail every +//! consumer save; blocking (`on_schema_change=fail`) is deliberately not +//! offered in v1. +//! +//! Ducklake-only: `// materialize` targets are ducklake-only in v1, so nothing +//! else has a captured schema to check against; an asset with no captured +//! schema produces no warnings (first deploy, datatable, external tables). +//! +//! The comparison itself (`diff_contract`) is pure so it can be unit-tested +//! and mirrored 1:1 by the editor-side TS check +//! (frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts); the +//! async wrapper owns the DB reads (schemas + producer resolution) and runs on +//! the caller's RLS-scoped transaction. + +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::types::Json; +use sqlx::{Postgres, Transaction}; +use windmill_parser::asset_parser::{ + ColumnLineage, DataTest, MaterializeSpec, OnSchemaChange, PARTITION_TOKEN, +}; +use windmill_types::assets::{AssetKind, AssetWithAltAccessType}; + +use crate::error::Result; +use crate::materialization::SchemaColumn; + +/// Columns the materialize engine adds/manages; never part of the captured +/// schema, so consumer reads of them must not warn (`_wm_partition` is +/// filtered out of the DESCRIBE capture on purpose). +const RESERVED_COLUMNS: &[&str] = &["_wm_partition"]; + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ContractWarningKind { + /// A column the body reads/writes is absent from the captured schema. + MissingColumn, + /// A `// column … <- .
` source column is absent. + MissingLineageSource, + /// A `// data_test relationships … -> .` ref column is absent. + MissingRelationshipColumn, + /// Relationship join columns have different captured types (may still + /// coerce at run time — phrased as "differs", not "will fail"). + RelationshipTypeMismatch, + /// Warnings for this asset were suppressed by the producer's + /// `on_schema_change=ignore` (one informational entry per asset). + Suppressed, +} + +/// One save-time contract warning. `schema_version`/`captured_at` identify the +/// capture the check ran against, so a stale-capture warning is +/// self-explaining (the schema is as-of the producer's last run, not its +/// latest save). +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct ContractWarning { + pub kind: ContractWarningKind, + /// Normalized ducklake asset path (`/
`). + pub asset_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub column: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub found_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub captured_at: Option>, + pub message: String, +} + +/// The latest captured schema of one asset, as loaded by the wrapper. +#[derive(Debug, Clone)] +pub struct CapturedSchema { + pub columns: Vec, + pub version: i64, + pub captured_at: DateTime, +} + +impl CapturedSchema { + /// Case-insensitive column lookup — DuckDB matches unquoted identifiers + /// case-insensitively, and the body parser preserves source casing while + /// DESCRIBE returns stored casing. + fn find(&self, name: &str) -> Option<&SchemaColumn> { + self.columns + .iter() + .find(|c| c.name.eq_ignore_ascii_case(name)) + } +} + +fn is_reserved(name: &str) -> bool { + RESERVED_COLUMNS + .iter() + .any(|r| r.eq_ignore_ascii_case(name)) +} + +/// Strip the `{partition}` token a declared URI may carry (`// on +/// ducklake://lake/t/{partition}` or pasted refs) so lookups hit the captured +/// path. Body-inferred paths never carry it, but annotation refs can. +pub fn normalize_asset_path(path: &str) -> String { + path.replace(&format!("/{}", PARTITION_TOKEN), "") + .replace(PARTITION_TOKEN, "") + .trim_end_matches('/') + .to_string() +} + +fn column_list(schema: &CapturedSchema) -> String { + schema + .columns + .iter() + .map(|c| c.name.as_str()) + .collect::>() + .join(", ") +} + +/// Pure comparison: consumer refs vs captured schemas. `schemas` is keyed by +/// normalized ducklake path (the `_current` → base-table fallback is resolved +/// by the wrapper before this runs); `ignored` holds paths whose producer +/// declared `on_schema_change=ignore`. +pub fn diff_contract( + assets: &[AssetWithAltAccessType], + column_lineage: &[ColumnLineage], + data_tests: &[DataTest], + materialize: Option<&MaterializeSpec>, + schemas: &HashMap, + ignored: &HashSet, +) -> Vec { + let mut warnings: Vec = vec![]; + + // W1 — body-read/written columns missing from the captured schema. Assets + // whose column set the parser could not derive (`columns: None`, e.g. + // wildcard SELECT or non-SQL access) are skipped fail-safe; a literal "*" + // key is skipped defensively for the same reason. + for a in assets { + if a.kind != AssetKind::Ducklake { + continue; + } + let Some(columns) = a.columns.as_ref() else { + continue; + }; + let path = normalize_asset_path(&a.path); + let Some(schema) = schemas.get(&path) else { + continue; + }; + for col in columns.keys() { + if col == "*" || is_reserved(col) { + continue; + } + if schema.find(col).is_none() { + warnings.push(ContractWarning { + kind: ContractWarningKind::MissingColumn, + asset_path: path.clone(), + column: Some(col.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "column `{col}` of ducklake://{path} is not in its captured schema \ + (v{}, columns: {})", + schema.version, + column_list(schema) + ), + }); + } + } + } + + // W2 — `// column` lineage source refs. Only annotation-declared lineage + // reaches this fn (AST-inferred lineage is redundant with W1 and can + // mis-attribute aliases). + for cl in column_lineage { + for input in &cl.inputs { + if input.from_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + continue; + } + let path = normalize_asset_path(&input.from_path); + let Some(schema) = schemas.get(&path) else { + continue; + }; + if is_reserved(&input.from_column) { + continue; + } + if schema.find(&input.from_column).is_none() { + warnings.push(ContractWarning { + kind: ContractWarningKind::MissingLineageSource, + asset_path: path.clone(), + column: Some(input.from_column.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// column {}` reads `{}` from ducklake://{path}, which is not in \ + its captured schema (v{})", + cl.column, input.from_column, schema.version + ), + }); + } + } + } + + // W3 — relationships data-test refs: the referenced column must exist; + // when both sides have captured types, flag a difference. Types come from + // DuckDB DESCRIBE on both sides so verbatim spellings are comparable; the + // runtime probe's IN-subquery still coerces, so a difference is "differs", + // never "will fail". + let own_schema = materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Ducklake) + .and_then(|m| schemas.get(&normalize_asset_path(&m.target_path))); + for dt in data_tests { + let DataTest::Relationships { column, to_kind, to_path, to_column } = dt else { + continue; + }; + if *to_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + continue; + } + let path = normalize_asset_path(to_path); + let Some(schema) = schemas.get(&path) else { + continue; + }; + match schema.find(to_column) { + None => { + warnings.push(ContractWarning { + kind: ContractWarningKind::MissingRelationshipColumn, + asset_path: path.clone(), + column: Some(to_column.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// data_test relationships {column}` references \ + ducklake://{path}.{to_column}, which is not in its captured schema \ + (v{})", + schema.version + ), + }); + } + Some(ref_col) => { + if let Some(own_col) = own_schema.and_then(|s| s.find(column)) { + if !own_col.data_type.eq_ignore_ascii_case(&ref_col.data_type) { + warnings.push(ContractWarning { + kind: ContractWarningKind::RelationshipTypeMismatch, + asset_path: path.clone(), + column: Some(to_column.clone()), + expected_type: Some(own_col.data_type.clone()), + found_type: Some(ref_col.data_type.clone()), + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// data_test relationships {column}` joins `{}` ({}) to \ + ducklake://{path}.{to_column} ({}) — captured types differ", + column, own_col.data_type, ref_col.data_type + ), + }); + } + } + } + } + } + + // W4 — producer opted the asset out (`on_schema_change=ignore`): drop its + // warnings, leaving one informational entry per suppressed asset so the + // response still records that a mismatch exists but was muted upstream. + if !ignored.is_empty() { + let mut suppressed_assets: Vec = vec![]; + warnings.retain(|w| { + if ignored.contains(&w.asset_path) { + if !suppressed_assets.contains(&w.asset_path) { + suppressed_assets.push(w.asset_path.clone()); + } + false + } else { + true + } + }); + for path in suppressed_assets { + warnings.push(ContractWarning { + kind: ContractWarningKind::Suppressed, + asset_path: path.clone(), + column: None, + expected_type: None, + found_type: None, + schema_version: None, + captured_at: None, + message: format!( + "schema mismatches on ducklake://{path} suppressed by its producer's \ + `on_schema_change=ignore`" + ), + }); + } + } + + warnings +} + +/// Load captured schemas + producer modes and run the contract check for one +/// consumer script's parsed refs. +/// +/// Runs on the caller's RLS-scoped transaction (`user_db`), consistent with +/// the `listAssetSchemas` read path: a producer script the caller cannot read +/// simply stays unresolved and keeps the default `warn` behavior. Draft-only +/// producers have no `asset` write edges yet and likewise default to `warn`. +pub async fn check_schema_contracts( + tx: &mut Transaction<'_, Postgres>, + workspace_id: &str, + assets: &[AssetWithAltAccessType], + column_lineage: &[ColumnLineage], + data_tests: &[DataTest], + materialize: Option<&MaterializeSpec>, +) -> Result> { + // Referenced ducklake paths (normalized) across every ref family the diff + // inspects — plus the consumer's own materialize target (for W3 types). + let mut paths: HashSet = HashSet::new(); + for a in assets { + if a.kind == AssetKind::Ducklake && a.columns.is_some() { + paths.insert(normalize_asset_path(&a.path)); + } + } + for cl in column_lineage { + for input in &cl.inputs { + if input.from_kind == windmill_parser::asset_parser::AssetKind::Ducklake { + paths.insert(normalize_asset_path(&input.from_path)); + } + } + } + for dt in data_tests { + if let DataTest::Relationships { to_kind, to_path, .. } = dt { + if *to_kind == windmill_parser::asset_parser::AssetKind::Ducklake { + paths.insert(normalize_asset_path(to_path)); + } + } + } + if let Some(m) = materialize { + if m.target_kind == windmill_parser::asset_parser::AssetKind::Ducklake { + paths.insert(normalize_asset_path(&m.target_path)); + } + } + if paths.is_empty() { + return Ok(vec![]); + } + + // A managed scd2 producer (re)creates a `_current` view with the base + // table's columns; only the base table's schema is captured. Include the + // base path in the lookup so `_current` readers can fall back to it (the + // fallback itself is gated on the producer's spec below). + let mut lookup_paths: HashSet = paths.clone(); + for p in &paths { + if let Some(base) = p.strip_suffix("_current") { + if !base.is_empty() { + lookup_paths.insert(base.to_string()); + } + } + } + let lookup_vec: Vec = lookup_paths.into_iter().collect(); + + let schema_rows = sqlx::query!( + r#"SELECT DISTINCT ON (asset_path) + asset_path, version, columns AS "columns: Json>", captured_at + FROM materialized_asset_schema + WHERE workspace_id = $1 AND asset_kind = 'ducklake' AND asset_path = ANY($2) + ORDER BY asset_path, version DESC"#, + workspace_id, + &lookup_vec, + ) + .fetch_all(&mut **tx) + .await?; + let mut schemas: HashMap = schema_rows + .into_iter() + .map(|r| { + ( + r.asset_path, + CapturedSchema { + columns: r.columns.0, + version: r.version, + captured_at: r.captured_at, + }, + ) + }) + .collect(); + + // Producer resolution: write edges on the referenced assets → latest + // non-archived producer content → parsed `// materialize` spec. Drives + // both the `on_schema_change=ignore` suppression and the `_current` + // fallback. Flow writers are excluded — they cannot carry the annotation. + let paths_vec: Vec = paths.iter().cloned().collect(); + let producer_edges = sqlx::query!( + r#"SELECT DISTINCT path AS "asset_path!", usage_path AS "producer_path!" + FROM asset + WHERE workspace_id = $1 AND kind = 'ducklake' AND path = ANY($2) + AND usage_kind = 'script' AND usage_access_type IN ('w', 'rw')"#, + workspace_id, + &paths_vec, + ) + .fetch_all(&mut **tx) + .await?; + + let mut ignored: HashSet = HashSet::new(); + if !producer_edges.is_empty() { + let producer_paths: Vec = producer_edges + .iter() + .map(|e| e.producer_path.clone()) + .collect::>() + .into_iter() + .collect(); + // Same latest-content pattern as the asset-graph endpoint; NOT + // `get_latest_script_hash`, whose `lock IS NOT NULL` filter transiently + // excludes a just-deployed producer pending its dependency job. + let producer_rows = sqlx::query!( + r#"SELECT DISTINCT ON (path) path, content + FROM script + WHERE workspace_id = $1 AND path = ANY($2) + AND archived = false AND deleted = false + ORDER BY path, created_at DESC"#, + workspace_id, + &producer_paths, + ) + .fetch_all(&mut **tx) + .await?; + let producer_specs: HashMap> = producer_rows + .into_iter() + .map(|r| { + ( + r.path, + windmill_parser::asset_parser::parse_pipeline_annotations(&r.content) + .materialize, + ) + }) + .collect(); + + for edge in &producer_edges { + let Some(Some(spec)) = producer_specs.get(&edge.producer_path) else { + continue; + }; + if spec.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + continue; + } + let target = normalize_asset_path(&spec.target_path); + // `on_schema_change=ignore` — any producer declaring it wins. + if spec.on_schema_change == OnSchemaChange::Ignore + && (target == edge.asset_path + || (spec.scd2 && format!("{target}_current") == edge.asset_path)) + { + ignored.insert(edge.asset_path.clone()); + } + // `_current` fallback: a managed scd2 producer's view has exactly + // the base table's columns. + if spec.scd2 + && !spec.manual + && format!("{target}_current") == edge.asset_path + && !schemas.contains_key(&edge.asset_path) + { + if let Some(base) = schemas.get(&target).cloned() { + schemas.insert(edge.asset_path.clone(), base); + } + } + } + } + + Ok(diff_contract( + assets, + column_lineage, + data_tests, + materialize, + &schemas, + &ignored, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use windmill_parser::asset_parser::{parse_pipeline_annotations, ColumnRef}; + use windmill_types::assets::AssetUsageAccessType; + + fn schema(cols: &[(&str, &str)]) -> CapturedSchema { + CapturedSchema { + columns: cols + .iter() + .map(|(n, t)| SchemaColumn { name: n.to_string(), data_type: t.to_string() }) + .collect(), + version: 2, + captured_at: DateTime::::MIN_UTC, + } + } + + fn read_asset(path: &str, cols: &[&str]) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: path.to_string(), + kind: AssetKind::Ducklake, + access_type: Some(AssetUsageAccessType::R), + alt_access_type: None, + columns: Some( + cols.iter() + .map(|c| (c.to_string(), AssetUsageAccessType::R)) + .collect::>(), + ), + } + } + + #[test] + fn missing_read_column_warns_case_insensitively() { + let schemas = HashMap::from([( + "lake/orders".to_string(), + schema(&[("Order_ID", "BIGINT"), ("amount_usd", "DOUBLE")]), + )]); + let assets = vec![read_asset("lake/orders", &["order_id", "amount"])]; + let w = diff_contract(&assets, &[], &[], None, &schemas, &HashSet::new()); + // order_id matches case-insensitively; amount is gone + assert_eq!(w.len(), 1); + assert_eq!(w[0].kind, ContractWarningKind::MissingColumn); + assert_eq!(w[0].column.as_deref(), Some("amount")); + assert_eq!(w[0].schema_version, Some(2)); + } + + #[test] + fn unknown_columns_wildcard_and_reserved_are_skipped() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + // columns: None (wildcard SELECT) — skipped entirely + let mut a = read_asset("lake/orders", &[]); + a.columns = None; + assert!(diff_contract(&[a], &[], &[], None, &schemas, &HashSet::new()).is_empty()); + // literal "*" and the reserved partition column are skipped + let a = read_asset("lake/orders", &["*", "_wm_partition", "id"]); + assert!(diff_contract(&[a], &[], &[], None, &schemas, &HashSet::new()).is_empty()); + } + + #[test] + fn asset_without_captured_schema_is_silent() { + let assets = vec![read_asset("lake/unknown", &["whatever"])]; + assert!( + diff_contract(&assets, &[], &[], None, &HashMap::new(), &HashSet::new()).is_empty() + ); + } + + #[test] + fn partition_token_is_normalized() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + let assets = vec![read_asset("lake/orders/{partition}", &["gone"])]; + let w = diff_contract(&assets, &[], &[], None, &schemas, &HashSet::new()); + assert_eq!(w.len(), 1); + assert_eq!(w[0].asset_path, "lake/orders"); + } + + #[test] + fn lineage_ref_missing_column_warns() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + let lineage = vec![ColumnLineage { + column: "total".to_string(), + inputs: vec![ColumnRef { + from_kind: windmill_parser::asset_parser::AssetKind::Ducklake, + from_path: "lake/orders".to_string(), + from_column: "amount".to_string(), + }], + }]; + let w = diff_contract(&[], &lineage, &[], None, &schemas, &HashSet::new()); + assert_eq!(w.len(), 1); + assert_eq!(w[0].kind, ContractWarningKind::MissingLineageSource); + } + + #[test] + fn relationships_missing_and_type_mismatch() { + let schemas = HashMap::from([ + ("lake/customers".to_string(), schema(&[("id", "VARCHAR")])), + ( + "lake/orders".to_string(), + schema(&[("customer_id", "BIGINT")]), + ), + ]); + let ann = parse_pipeline_annotations( + "// materialize ducklake://lake/orders\n\ + // data_test relationships customer_id -> ducklake://lake/customers.id\n\ + // data_test relationships customer_id -> ducklake://lake/customers.uuid\n\ + SELECT 1;", + ); + let w = diff_contract( + &[], + &[], + &ann.data_tests, + ann.materialize.as_ref(), + &schemas, + &HashSet::new(), + ); + assert_eq!(w.len(), 2); + assert!(w + .iter() + .any(|w| w.kind == ContractWarningKind::RelationshipTypeMismatch + && w.expected_type.as_deref() == Some("BIGINT") + && w.found_type.as_deref() == Some("VARCHAR"))); + assert!(w + .iter() + .any(|w| w.kind == ContractWarningKind::MissingRelationshipColumn + && w.column.as_deref() == Some("uuid"))); + } + + #[test] + fn ignored_asset_suppresses_to_single_note() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + let assets = vec![read_asset("lake/orders", &["a", "b"])]; + let ignored = HashSet::from(["lake/orders".to_string()]); + let w = diff_contract(&assets, &[], &[], None, &schemas, &ignored); + assert_eq!(w.len(), 1); + assert_eq!(w[0].kind, ContractWarningKind::Suppressed); + // and nothing at all when there was nothing to suppress + let assets = vec![read_asset("lake/orders", &["id"])]; + assert!(diff_contract(&assets, &[], &[], None, &schemas, &ignored).is_empty()); + } +} diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 9d40a0e4b7..98413876a5 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -44,9 +44,14 @@ export type GraphRunnable = { column_lineage?: unknown[]; // `// materialize ` target + strategy — the script's declared output, // so the UI anchors column lineage / the materialize badge to it (the producer - // write-edge is emitted separately). + // write-edge is emitted separately). `scd2` also identifies the producer of a + // `_current` view for the editor's schema-contract fallback. materialize_target?: { kind: string; path: string }; - materialize_strategy?: "replace" | "append" | "merge"; + materialize_strategy?: "replace" | "append" | "merge" | "scd2"; + // `on_schema_change=ignore` — producer's opt-out from downstream + // schema-contract warnings; only present when set (default `warn` is absent), + // mirroring the deployed graph node. + materialize_on_schema_change?: string; }; export type GraphEdge = { runnable_kind: string; @@ -110,6 +115,9 @@ type ParseAssetsRaw = { manual?: boolean; append?: boolean; unique_key?: string; + scd2?: boolean; + // "ignore" when set; the default `warn` is skipped in serialization. + on_schema_change?: string; }; // `// tag ` — the worker tag the deployed pipeline routes to. tag?: string; @@ -428,16 +436,18 @@ export async function buildLocalPipelineGraph(args: { // deployed pipeline would (both `pipeline run --local` and `/pipeline_dev`). pipelineScripts.push(out.tag ? { ...s, tag: out.tag } : s); const mat = out.materialize; - // Managed-materialize write strategy, derived like the deployed graph - // (append → append; key=→ merge; else replace). Manual mode has no - // managed strategy. + // Managed-materialize write strategy, derived like the deployed graph — + // precedence mirrors the runtime: scd2 (`history`) > append > merge + // (key=) > replace. Manual mode has no managed strategy. const materialize_strategy = mat && !mat.manual - ? mat.append - ? "append" - : mat.unique_key - ? "merge" - : "replace" + ? mat.scd2 + ? "scd2" + : mat.append + ? "append" + : mat.unique_key + ? "merge" + : "replace" : undefined; runnables.push({ path: s.path, @@ -453,6 +463,9 @@ export async function buildLocalPipelineGraph(args: { : {}), ...(mat ? { materialize_target: { kind: mat.target_kind, path: mat.target_path } } : {}), ...(materialize_strategy ? { materialize_strategy } : {}), + ...(mat?.on_schema_change === "ignore" + ? { materialize_on_schema_change: "ignore" } + : {}), }); for (const a of out.assets ?? []) { diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 4481d7555b..83c25f3eec 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -49,6 +49,8 @@ stays separate because it is cross-cutting (cascade + scheduling + materialize). // materialize ducklake://analytics/orders_daily append → managed, append // materialize ducklake://analytics/dim_customer key=id history → managed, SCD type 2 history // materialize manual ducklake://analytics/orders_daily → track-only escape hatch +// materialize ducklake://analytics/raw_events on_schema_change=ignore +// → managed, downstream contract warnings muted ``` - **managed (default)** — the script is *setup + one trailing `SELECT`*; Windmill @@ -366,6 +368,35 @@ load-bearing. - **Managed only.** `// materialize manual` + `// data_test` is rejected with a clear error (we can't know the manual script's target alias / partition col). +## Schema contracts (save-time, gap #2b) + +The captured schema (#2a) is read back as a *contract*: at save/deploy time, +every consumer's asset references — body-read/written columns, `// column` +lineage sources, `// data_test relationships` refs — are diffed against the +latest `materialized_asset_schema` version of each referenced ducklake asset, +and mismatches surface as **warnings** (deploy never blocks; dbt's +`on_schema_change=fail` is deliberately absent in v1). The diff lives in +`windmill_common::schema_contracts` (endpoint: +`POST /w/{ws}/scripts/check_schema_contracts`, called by the UI right after a +save) and is mirrored 1:1 by the editor (`schemaContracts.ts`): live Monaco +warning squiggles from the WASM buffer parse, plus column-name completion for +annotation refs fed by the same captured schemas. + +Comparison rules worth knowing: column names are case-insensitive (DuckDB +unquoted-identifier semantics); `_wm_partition` is whitelisted (it's excluded +from capture); `{partition}` tokens are stripped before lookup; a +`_current` reference falls back to the scd2 base table's capture; a +relationships join across two captured assets also flags a captured-type +*difference* (the runtime probe coerces, so it's "differs", not "will fail"). +An asset with no capture (never materialized, `manual` mode, any +`datatable://`) produces no warnings. + +The producer opts a deliberately unstable schema out with +`// materialize … on_schema_change=ignore` — consumers then get a single +informational "suppressed" note instead of per-column warnings. On `manual` +materialize the option is inert (nothing is captured) and deploy logs a +warning saying so. + ## Scoping decision: DuckLake vs DataTable **Make DuckLake the materialization/versioning substrate; keep DataTable as the diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index c073c88b8c..4ad8e65fb9 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -51,7 +51,7 @@ Asset-centric, polyglot, annotation-driven, event-aware: | Column lineage | No | **Shipped** (`// column`); docs site still TODO | | Snapshots / SCD2 | Yes (`key=… history`) | Managed strategy | | Selective execution grammar | No | UI/CLI surface | -| Schema contracts | No, but design metadata model | TODO with design work | +| Schema contracts | No, but design metadata model | **Shipped** (capture #2a + save-time check #2b) | | Packages / community / macros | No | **Shipped** (`// macros` workspace macro libraries) | | Semantic layer / metrics | No | Large additive scope | @@ -140,21 +140,43 @@ Today: `requestRunCascadeSignal` in the canvas, `// tag` annotation parsed. Graph + tags + last-run state has all the inputs. UI/CLI surface, not abstraction work. -### 6. Schema contracts +### 6. Schema contracts — **shipped** dbt: `contract: enforced` + `columns: [{name, data_type}]`. Compile-time check that model output matches the declaration. -Today: `// on datatable://users/active` is a string. Rename a column -upstream → downstream breaks at runtime, silently. +**Shipped**, capture-then-validate (no declaration to maintain): the schema a +managed `// materialize` run captures post-DESCRIBE into the versioned +`materialized_asset_schema` sidecar (#2a) *is* the contract, and every +consumer save/deploy is validated against the latest capture. Three surfaces, +same diff (`windmill_common::schema_contracts`, mirrored 1:1 in +`schemaContracts.ts`): -This is the item where the current asset abstraction is thinnest. -To do contracts well: capture output schemas after a run (substrate-specific -DESCRIBE), persist them as asset metadata, validate consumer references at -save time. The asset-as-typed-node model accommodates it — but **where** -schemas live (asset row, sidecar?), **when** they're captured (post-run? -edit-time?), and **how** versioning works are non-trivial design choices. -Worth doing intentionally now while the asset surface is still young. +- **Save-time (authoritative)**: `POST /w/{ws}/scripts/check_schema_contracts` + runs on the deployed content right after a save and returns **warnings — + never errors**. A deliberate upstream reshape must not fail every consumer + save; that's why dbt's `on_schema_change=fail` is deliberately not offered + in v1 (a CI/CLI gate can layer it on later without touching the grammar). +- **Editor flycheck**: the WASM parse that already runs on the open buffer + feeds the same diff, surfacing mismatches as live Monaco warning squiggles + anchored to the offending read / `// column` / `// data_test` line. +- **Autocompletion**: annotation refs (`// column out <- + ducklake://lake/orders.`, `// data_test relationships … ->`) complete + column names from the captured schema — the broken ref never gets typed. + +What's checked (ducklake-only — the only substrate with capture in v1; +datatable refs have no captured schema and stay silent): body-read/written +columns missing from the capture, `// column` lineage sources, and +`// data_test relationships` refs — including a captured-type *difference* on +the join columns when both sides are captured (phrased "differs", since the +runtime probe's IN-subquery still coerces). Column names compare +case-insensitively (DuckDB unquoted-identifier semantics); the managed +`_wm_partition` column is whitelisted; a `_current` scd2 view falls back +to its base table's capture (identical columns by construction). + +The producer-side escape hatch is `// materialize … on_schema_change=ignore`: +it declares the schema deliberately unstable and collapses downstream +warnings for that asset into a single informational note. Default is `warn`. ### 7. Packages / community / macros — **shipped** (macro libraries) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index ae413d2484..78a75734c9 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -91,6 +91,11 @@ listWorkspaceMacrosCached, macroDefinitionSql } from '$lib/components/assets/workspaceMacros' + import { + fetchLatestSchema, + normalizeAssetPath, + type ContractMarker + } from '$lib/components/assets/AssetGraph/schemaContracts' import * as htmllang from '$lib/svelteMonarch' import { conf, language } from '$lib/vueMonarch' @@ -162,6 +167,11 @@ preparedAssetsSqlQueries?: InferAssetsSqlQueryDetails[] | undefined // To execute preview scripts with the right worker group customTag?: string + // Live schema-contract diagnostics (pipelines gap #2b): owner-scoped + // warning markers computed by the caller (ScriptEditor's contract + // mirror) from the buffer's asset refs vs captured producer schemas. + // Warning severity only — contracts never block; empty clears. + schemaContractMarkers?: ContractMarker[] } let { @@ -194,7 +204,8 @@ enablePreprocessorSnippet = false, rawAppRunnableKey = undefined, preparedAssetsSqlQueries, - customTag + customTag, + schemaContractMarkers = [] }: Props = $props() $effect.pre(() => { @@ -676,6 +687,53 @@ }) } + let schemaContractCompletor: IDisposable | undefined = undefined + + // Column-name completion for pipeline annotation refs (`// column out <- + // ducklake://lake/orders.|`, `// data_test relationships col -> + // ducklake://lake/customers.|`): suggests the referenced asset's *captured* + // columns (with types) so a broken ref never gets typed — the prevention + // side of the schema-contract check. Only fires on annotation comment lines + // with a ducklake URI right before the cursor's `.`; schemas come from the + // short-TTL contract cache, so per-keystroke cost is a map lookup. + function addSchemaContractCompletions() { + schemaContractCompletor?.dispose() + schemaContractCompletor = languages.registerCompletionItemProvider(lang, { + triggerCharacters: ['.'], + provideCompletionItems: async function (model, position) { + // Read the store per request, not at registration — the provider + // outlives a workspace switch. + const workspace = $workspaceStore + if (!workspace) return { suggestions: [] } + const before = model.getLineContent(position.lineNumber).slice(0, position.column - 1) + if (!/^\s*(\/\/|--|#)\s*(column|data_test|on|materialize)\b/.test(before)) { + return { suggestions: [] } + } + const uri = before.match(/ducklake:\/\/([\w/.{}-]+?)\.$/) + if (!uri) return { suggestions: [] } + const schema = await fetchLatestSchema(workspace, normalizeAssetPath(uri[1])) + if (!schema) return { suggestions: [] } + const word = model.getWordUntilPosition(position) + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn + } + return { + suggestions: schema.columns.map((c) => ({ + label: c.name, + kind: languages.CompletionItemKind.Field, + detail: `${c.type} · captured schema v${schema.version}`, + insertText: c.name, + range, + sortText: 'a' + c.name + })) + } + } + }) + } + let sqlSchemaCompletor: IDisposable | undefined = undefined async function updateSchema(newSchemaRes: string | undefined) { @@ -1885,6 +1943,7 @@ sqlTypeCompletor && sqlTypeCompletor.dispose() resultCollectionCompletor && resultCollectionCompletor.dispose() workspaceMacroCompletor && workspaceMacroCompletor.dispose() + schemaContractCompletor && schemaContractCompletor.dispose() preprocessorCompletor && preprocessorCompletor.dispose() timeoutModel && clearTimeout(timeoutModel) changeChainStart = undefined @@ -1968,6 +2027,36 @@ : workspaceMacroCompletor?.dispose() }) + // Pipeline annotation grammar is language-agnostic (`//` / `--` / `#` + // comment headers), so contract-ref completions register for every script + // language that can be a pipeline member. The provider line-gates itself, + // so it is inert outside annotation lines. + $effect(() => { + initialized && ['duckdb', 'python3', 'bun', 'deno', 'nativets'].includes(scriptLang ?? '') + ? untrack(() => addSchemaContractCompletions()) + : schemaContractCompletor?.dispose() + }) + + // Schema-contract markers arrive as a prop because the mirror can finish + // computing before Monaco initializes on mount — reacting to `initialized` + // re-applies the pending set once the model exists. The ever-set latch + // keeps unrelated editors from calling setModelMarkers with [] forever. + let contractMarkersEverSet = false + $effect(() => { + const ms = schemaContractMarkers + if (!initialized || (ms.length === 0 && !contractMarkersEverSet)) return + contractMarkersEverSet = true + untrack(() => { + const model = editor?.getModel() + if (!model) return + meditor.setModelMarkers( + model, + 'schema-contracts', + ms.map((m) => ({ ...m, severity: MarkerSeverity.Warning })) + ) + }) + }) + $effect(() => { initialized && canHavePreprocessor(lang) && enablePreprocessorSnippet ? untrack(() => addPreprocessorCompletions(lang)) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index fbfa4f1c1b..7d639b2c75 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -41,6 +41,7 @@ } from '$lib/utils' import Path from './Path.svelte' import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' + import { notifyContractWarnings } from './assets/AssetGraph/schemaContracts' import ScriptEditor from './ScriptEditor.svelte' import { Alert, Button, Drawer, SecondsInput, Tab, TabContent, Tabs } from './common' import LanguageIcon from './common/languageIcons/LanguageIcon.svelte' @@ -654,6 +655,11 @@ // cache so it shows up immediately instead of after the 60s TTL. invalidateWorkspacePaths($workspaceStore!) + // Authoritative save-time schema-contract check (pipelines gap #2b): + // warn-only, post-commit so a self-produced target resolves to the + // content just deployed. Fire-and-forget — must never gate the deploy. + notifyContractWarnings($workspaceStore!, script.language, script.content) + if (!initialPath) { await CaptureService.moveCapturesAndConfigs({ workspace: $workspaceStore!, diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index f1944c73cc..50b5c8da95 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -105,6 +105,11 @@ import { canHavePreprocessor } from '$lib/script_helpers' import { assetEq, type AssetWithAltAccessType } from './assets/lib' import type { ColumnLineage } from './assets/AssetGraph/parsePipelineAnnotations' + import { + computeContractMarkers, + type ContractMarker, + type SchemaContractGraphContext + } from './assets/AssetGraph/schemaContracts' import { editor as meditor } from 'monaco-editor' import type { ReviewChangesOpts } from './copilot/chat/monaco-adapter' import GitRepoViewer from './GitRepoViewer.svelte' @@ -202,6 +207,11 @@ // regular /scripts/edit route keeps its current open-by-default UX; // the session preview opts in to save vertical real estate. initialTestPanelCollapsed?: boolean + // Producer-side facts for the live schema-contract diagnostics + // (`on_schema_change=ignore` suppression + scd2 `_current` fallback), + // built by the pipeline page from the resolved graph. Absent outside the + // pipeline editor — the check still runs, just without suppression. + schemaContractContext?: SchemaContractGraphContext } let { @@ -242,7 +252,8 @@ previewLayout = 'right', onTestStateChange, onTestJob, - initialTestPanelCollapsed = false + initialTestPanelCollapsed = false, + schemaContractContext = undefined }: Props = $props() $effect(() => { @@ -614,6 +625,35 @@ } ) + // Live schema-contract diagnostics (pipelines gap #2b): diff the buffer's + // asset refs against the captured producer schemas and surface mismatches + // as Monaco warning squiggles — the as-you-type mirror of the authoritative + // save-time check. The result is a prop on Editor (not an imperative call) + // because this can resolve before Monaco initializes on mount. Sequenced so + // a slow schema fetch can't overwrite the markers of a newer keystroke. + let contractMarkers: ContractMarker[] = $state([]) + let contractCheckSeq = 0 + watch([() => inferAssetsRes.current, () => schemaContractContext], () => { + const res = inferAssetsRes.current + const workspace = $workspaceStore + const seq = ++contractCheckSeq + if (!workspace || !res || res.status === 'error') { + contractMarkers = [] + return + } + const bufferCode = code + computeContractMarkers( + workspace, + bufferCode, + (res.assets ?? []) as AssetWithAltAccessType[], + schemaContractContext + ) + .then((markers) => { + if (seq === contractCheckSeq) contractMarkers = markers + }) + .catch((e) => console.error('schema-contract diagnostics failed', e)) + }) + watch([() => code, () => lang], () => { if (lang !== 'ansible') return inferAnsibleExecutionMode(code).then((v) => { @@ -2602,6 +2642,7 @@ bind:code={editorCode} bind:websocketAlive bind:this={editor} + schemaContractMarkers={contractMarkers} {yContent} awareness={wsProvider?.awareness} on:change={(e) => { diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 1abe21f76d..c9b921be2e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -1,5 +1,5 @@ + +{#if !provider || !model} +
Select a model to configure reasoning effort.
+{:else if !capability.supported} +
The selected model does not support reasoning effort.
+{:else} + + {#snippet buttonReplacement()} + + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/aiProviderStorage.ts b/frontend/src/lib/components/aiProviderStorage.ts index ce6af230bb..7f6d705d3a 100644 --- a/frontend/src/lib/components/aiProviderStorage.ts +++ b/frontend/src/lib/components/aiProviderStorage.ts @@ -50,6 +50,7 @@ export function isSameAsStoredConfig(config: ProviderConfig | undefined): boolea storedConfig !== undefined && storedConfig.kind === config?.kind && storedConfig.resource === config?.resource && - storedConfig.model === config?.model + storedConfig.model === config?.model && + storedConfig.reasoning_effort === config?.reasoning_effort ) } diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 4776e64c79..519c1a4021 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -319,7 +319,7 @@ export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' * model that reasons *by default* — omitting the field would silently keep * the default-on behavior. Undefined means omission is the correct off. */ -function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { +export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { switch (reasoningProviderFamily(provider, model)) { case 'googleai': // Gemini 2.5/3 think by default (dynamic budget / level). The backend diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index ab81f1ea14..b0508b2719 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -475,6 +475,9 @@ components: model: type: string description: Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro') + reasoning_effort: + type: string + description: Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default. required: - kind - resource @@ -1028,7 +1031,7 @@ components: - $ref: '#/components/schemas/InputTransform' description: | Boolean. If true, stream the AI response incrementally. - Streaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result + Streaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result memory: $ref: '#/components/schemas/MemoryTransform' output_schema: diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index 7f70280e9f..70bec07fe6 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -295,4 +295,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 9578f1b62d..434a52da63 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2664,7 +2664,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 26f06fb667..a2a5d294e7 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -381,4 +381,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file From dce247c6d2678a2c95bd728027e17ae3965638e2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 15:19:49 +0000 Subject: [PATCH 188/273] feat(pipeline): write-audit-publish for materialization data tests (#9911) * feat(pipeline): write-audit-publish for materialization data tests (EE) Co-Authored-By: Claude Fable 5 * docs: EE worktree E0583 troubleshooting + duckdb feature check row Co-Authored-By: Claude Fable 5 * docs: clarify EE symlink example (absolute target, EE repo layout) Co-Authored-By: Claude Fable 5 * fix(pipeline): move bootstrap DDL inside guarded WAP transaction Co-Authored-By: Claude Fable 5 * refactor(pipeline): move WAP guard SQL builder into EE, OSS keeps placement only Co-Authored-By: Claude Fable 5 * chore: bump ee-repo-ref to EE branch rebased on EE main Co-Authored-By: Claude Fable 5 * style: reword test comment as current invariant per AGENTS.md Co-Authored-By: Claude Fable 5 * chore: bump ee-repo-ref (EE module doc update) Co-Authored-By: Claude Fable 5 * refactor(pipeline): OSS emits typed materialize plan, EE owns WAP transform Co-Authored-By: Claude Fable 5 * test: make rewrite assertion build-aware; refresh oss module doc Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c This commit updates the EE repository reference after PR #644 was merged in windmill-ee-private. Previous ee-repo-ref: 63cabae75329429f647e01083936d70f8197dc9e New ee-repo-ref: 7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../windmill-parser/src/sql_materialize.rs | 213 +++++++++++++++--- .../src/pipeline_advanced_oss.rs | 27 ++- .../windmill-worker/src/duckdb_executor.rs | 43 +++- docs/ducklake-materialization.md | 20 +- docs/enterprise.md | 16 ++ docs/pipelines-vs-dbt.md | 6 + docs/validation.md | 1 + 8 files changed, 289 insertions(+), 39 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 0cd164d1ce..e31f18cdd0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6f5fe0f7f56696fbef5a8349da38496c32e71666 +7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index e40cd0f057..d684cf431b 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -440,7 +440,6 @@ impl<'a> MaterializeCodegen<'a> { "CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;" )); } - out.push("BEGIN TRANSACTION;".to_string()); // The rows to write, with the partition column appended when partitioned. let source = if self.partitioned { @@ -578,7 +577,7 @@ impl<'a> MaterializeCodegen<'a> { ]; // Hard-delete-close (`deletes=close`): the keys that vanished from the // snapshot — present-and-current in the table, absent from the SELECT. - // Captured before the transaction (like `changed`) and disjoint from it (a + // Captured before the close (like `changed`) and disjoint from it (a // key is either in the snapshot or not), so the two closes never overlap. if close_deleted { out.push(format!( @@ -646,6 +645,44 @@ pub fn snapshot_capture_sql(alias: &str) -> String { /// from the target ducklake's config and passes it in as `target_attach`. pub const TARGET_ALIAS: &str = "_wm_target"; +/// Structural role of one statement in a [`MaterializePlan`]. The public build +/// executes the plan verbatim, so the kinds are pure metadata there; they exist +/// so a downstream assembler (`pipeline_advanced::finalize_materialize_query`) +/// can reason about the plan without parsing SQL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MaterializeStmtKind { + /// Pre-write statement: user setup, the target ATTACH, referenced-asset + /// ATTACHes. + Setup, + /// Write work against the target: bootstrap DDL, SCD2 temp-table captures, + /// the mutation itself, the `_current` view. + Write, + /// The `BEGIN TRANSACTION;` marker emitted by the strategy codegen. + TxnBegin, + /// The `COMMIT;` marker emitted by the strategy codegen. + TxnCommit, + /// The trailing one-row summary read (asset / rows / snapshot_id / + /// data_tests breakdown). + Summary, +} + +/// One planned statement: its structural role and the SQL text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializeStmt { + pub kind: MaterializeStmtKind, + pub sql: String, +} + +/// The full ordered materialization plan [`build_wrap_blocks`] produces: +/// statements in execution order plus the compiled data-test checks (also +/// embedded in the summary statement's breakdown). Assembled into the final +/// statement list by `pipeline_advanced::finalize_materialize_query`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializePlan { + pub stmts: Vec, + pub checks: Vec, +} + /// Assemble the full ordered statement list the DuckDB executor runs for a /// managed `// materialize` script. This is the single entry point the worker /// calls; it composes the already-tested pieces (classifier split → target @@ -659,6 +696,14 @@ pub const TARGET_ALIAS: &str = "_wm_target"; /// the full `/
` for the result summary. The trailing statement is /// a one-row summary read (asset / rows / snapshot_id) that is both the job's /// result (a useful preview) and what the worker records. +/// +/// Returns a [`MaterializePlan`] — the statements plus their structural role +/// and the compiled data-test checks — rather than raw SQL: the executor hands +/// the plan to `windmill_common::pipeline_advanced::finalize_materialize_query` +/// (which this crate cannot depend on), whose public-build implementation +/// assembles the statements verbatim. Everything this function produces runs +/// as-is on the public build; the plan's structure is metadata about it, not a +/// second mode. pub fn build_wrap_blocks( plan: &WrapPlan, target_attach: &str, @@ -669,17 +714,9 @@ pub fn build_wrap_blocks( partitioned: bool, strategy: MaterializeStrategy, tests: &[DataTestResolved], -) -> Result, String> { +) -> Result { let target_qualified = format!("{TARGET_ALIAS}.{target_table}"); let scd2 = matches!(strategy, MaterializeStrategy::Scd2 { .. }); - let cg = MaterializeCodegen { - target_qualified: &target_qualified, - select_sql: &plan.output, - partition_col, - partition_value_sql, - partitioned, - strategy, - }; let ctx = DataTestCtx { target_qualified: &target_qualified, asset_path, @@ -689,27 +726,49 @@ pub fn build_wrap_blocks( scd2, }; let test_sql = build_data_test_checks(tests, &ctx)?; - let mut blocks: Vec = Vec::new(); + let cg = MaterializeCodegen { + target_qualified: &target_qualified, + select_sql: &plan.output, + partition_col, + partition_value_sql, + partitioned, + strategy, + }; + let mut stmts: Vec = Vec::new(); + let setup = |sql: String| MaterializeStmt { kind: MaterializeStmtKind::Setup, sql }; // Setup blocks come from the splitter with their `;` stripped — re-terminate // each so that when the executor re-joins and re-splits the assembled query, // adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH) // don't merge into one malformed statement. - blocks.extend(plan.setup.iter().map(|s| terminate(s))); - blocks.push(target_attach.to_string()); + stmts.extend(plan.setup.iter().map(|s| setup(terminate(s)))); + stmts.push(setup(target_attach.to_string())); // Referenced-asset ATTACHes (relationships tests) — read-only, before the // write and the summary that probes them. - blocks.extend(test_sql.attaches); - blocks.extend(cg.statements()); + stmts.extend(test_sql.attaches.into_iter().map(setup)); + // Classify the codegen statements by matching the exact transaction-marker + // literals this module emits (`BEGIN TRANSACTION;` / `COMMIT;`); everything + // else the codegen produces is write work. + stmts.extend(cg.statements().into_iter().map(|sql| { + let kind = match sql.as_str() { + "BEGIN TRANSACTION;" => MaterializeStmtKind::TxnBegin, + "COMMIT;" => MaterializeStmtKind::TxnCommit, + _ => MaterializeStmtKind::Write, + }; + MaterializeStmt { kind, sql } + })); // The summary read carries the per-test breakdown (when any tests apply). - blocks.push(materialize_result_sql( - &target_qualified, - asset_path, - partition_col, - partition_value_sql, - partitioned, - &test_sql.checks, - )); - Ok(blocks) + stmts.push(MaterializeStmt { + kind: MaterializeStmtKind::Summary, + sql: materialize_result_sql( + &target_qualified, + asset_path, + partition_col, + partition_value_sql, + partitioned, + &test_sql.checks, + ), + }); + Ok(MaterializePlan { stmts, checks: test_sql.checks }) } /// The trailing one-row summary the materialize run returns: the asset it @@ -1454,7 +1513,7 @@ mod tests { #[test] fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() { let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'"); - let blocks = build_wrap_blocks( + let blocks: Vec = build_wrap_blocks( &plan, "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');", "orders_daily", @@ -1465,7 +1524,11 @@ mod tests { MaterializeStrategy::Replace, &[], ) - .unwrap(); + .unwrap() + .stmts + .into_iter() + .map(|s| s.sql) + .collect(); // setup block first, then the target ATTACH, then codegen, then result. assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl")); // every setup block must be `;`-terminated so re-splitting can't merge it @@ -1488,6 +1551,102 @@ mod tests { assert!(last.contains("ducklake_snapshots('_wm_target')")); } + // -- materialize plan structure ------------------------------------------ + + fn plan_for(strategy: MaterializeStrategy, partitioned: bool) -> MaterializePlan { + let plan = ok("SELECT a, b FROM src"); + build_wrap_blocks( + &plan, + "ATTACH 'ducklake:…' AS _wm_target;", + "orders", + "main/orders", + "_wm_partition", + "'2026-06-19'", + partitioned, + strategy, + &[ + DataTestResolved::BuiltIn(DataTest::NotNull { column: "a".into() }), + DataTestResolved::BuiltIn(DataTest::Unique { column: "b".into() }), + ], + ) + .unwrap() + } + + fn kidx(plan: &MaterializePlan, pred: impl Fn(&MaterializeStmt) -> bool) -> usize { + plan.stmts + .iter() + .position(|s| pred(s)) + .expect("stmt present") + } + + #[test] + fn plan_tags_structure_and_carries_checks() { + use MaterializeStmtKind::*; + let plan = plan_for(MaterializeStrategy::Replace, true); + // leading statements are Setup, ending with the target ATTACH + assert!(plan.stmts[0].kind == Setup); + assert!(plan + .stmts + .iter() + .take_while(|s| s.kind == Setup) + .any(|s| s.sql.contains("_wm_target"))); + // txn markers are tagged, everything between them is Write + let begin = kidx(&plan, |s| s.kind == TxnBegin); + let commit = kidx(&plan, |s| s.kind == TxnCommit); + assert!(begin < commit); + assert!(plan.stmts[begin + 1..commit] + .iter() + .all(|s| s.kind == Write)); + // bootstrap DDL is Write work (it targets the table, not the session) + let bootstrap = kidx(&plan, |s| s.sql.starts_with("CREATE TABLE IF NOT EXISTS")); + assert_eq!(plan.stmts[bootstrap].kind, Write); + // summary is last and carries the breakdown; checks ride along + let last = plan.stmts.last().unwrap(); + assert_eq!(last.kind, Summary); + assert!(last.sql.contains("AS data_tests")); + assert_eq!(plan.checks.len(), 2); + assert!(plan.checks[0].name.contains("not_null(a)")); + } + + #[test] + fn plan_whole_table_replace_has_no_txn_markers() { + use MaterializeStmtKind::*; + let plan = plan_for(MaterializeStrategy::Replace, false); + assert!(!plan.stmts.iter().any(|s| s.kind == TxnBegin)); + assert!(!plan.stmts.iter().any(|s| s.kind == TxnCommit)); + assert_eq!( + plan.stmts.iter().filter(|s| s.kind == Write).count(), + 1, + "single atomic CREATE OR REPLACE" + ); + } + + #[test] + fn plan_scd2_captures_are_write_kind() { + use MaterializeStmtKind::*; + let plan = plan_for( + MaterializeStrategy::Scd2 { key: "a".into(), track: vec![], close_deleted: false }, + false, + ); + let capture = kidx(&plan, |s| s.sql.contains("TEMP TABLE _wm_scd2_changed")); + assert_eq!(plan.stmts[capture].kind, Write); + // no test declared ⇒ empty checks + let plain = ok("SELECT a FROM src"); + let no_tests = build_wrap_blocks( + &plain, + "ATTACH 'ducklake:…' AS _wm_target;", + "orders", + "main/orders", + "_wm_partition", + "''", + false, + MaterializeStrategy::Append, + &[], + ) + .unwrap(); + assert!(no_tests.checks.is_empty()); + } + // -- data tests --------------------------------------------------------- fn ctx_partitioned() -> DataTestCtx<'static> { diff --git a/backend/windmill-common/src/pipeline_advanced_oss.rs b/backend/windmill-common/src/pipeline_advanced_oss.rs index 15bb4b127a..3220b251a0 100644 --- a/backend/windmill-common/src/pipeline_advanced_oss.rs +++ b/backend/windmill-common/src/pipeline_advanced_oss.rs @@ -1,12 +1,27 @@ -//! OSS fallback: pipeline partition backfills are an enterprise feature; -//! their implementations live in windmill-ee-private (see -//! `pipeline_advanced_ee`). In the public build the entry points report that -//! the enterprise edition is required. (Freshness lives elsewhere: the -//! fresh/stale badge is CE in the assets API, the active watchdog is -//! windmill-queue's `freshness_watchdog`.) +//! OSS fallback for enterprise pipeline features (implementations in +//! windmill-ee-private, see `pipeline_advanced_ee`): partition backfill +//! reports that the enterprise edition is required, and materialization-plan +//! assembly runs the plan verbatim — dbt-like commit-then-test instead of the +//! enterprise write-audit-publish. (Freshness lives elsewhere: the fresh/stale +//! badge is CE in the assets API, the active watchdog is windmill-queue's +//! `freshness_watchdog`.) use crate::error::Error; pub fn backfill_todo() -> Error { Error::internal_err("Pipeline partition backfill requires the enterprise edition".to_string()) } + +/// Assemble a materialization plan into the statement list the DuckDB executor +/// runs. The public build executes the plan verbatim — dbt-like +/// commit-then-test: a failing `// data_test` still fails the run and stops +/// the cascade, but the written slice stays live. The enterprise +/// implementation (`pipeline_advanced_ee`) instead restructures the plan into +/// write-audit-publish, where a failing test rolls the whole write back before +/// anything is published. +pub fn finalize_materialize_query( + plan: windmill_parser::sql_materialize::MaterializePlan, + _asset_path: &str, +) -> Vec { + plan.stmts.into_iter().map(|s| s.sql).collect() +} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 09e3158200..a90c80acfc 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -242,7 +242,7 @@ fn build_materialized_query( } } - let blocks = build_wrap_blocks( + let mat_plan = build_wrap_blocks( &plan, &synthetic_attach, table, @@ -254,6 +254,11 @@ fn build_materialized_query( &resolved, ) .map_err(Error::ExecutionErr)?; + // Enterprise seam: assembles the plan into the final statement list — + // verbatim on the public build (commit-then-test), restructured into + // write-audit-publish (guarded transaction, rollback on violation) on EE. + let blocks = + windmill_common::pipeline_advanced::finalize_materialize_query(mat_plan, &m.target_path); Ok(Some((Some(blocks.join("\n")), meta))) } @@ -1240,6 +1245,11 @@ pub async fn do_duckdb( // committed (like dbt), so the slice is recorded `Failed` and the // cascade stops. The error lists *every* test so the user sees the // whole picture, not just the first failure. + // Under enterprise write-audit-publish an in-transaction guard + // (the enterprise `finalize_materialize_query` restructure) already aborted a failing run before + // COMMIT — that surfaces on the Err path above with the same + // breakdown in the error string, and nothing was published; this + // post-commit path then only ever sees passing counts. let tests = extract_data_tests(&result); // Captured output schema (gap #2a) — recorded only on the successful // path below, not on the failure paths (a failed run shouldn't @@ -2498,6 +2508,37 @@ mod tests { ); } + // The rewritten SQL is the plan assembled by + // `pipeline_advanced::finalize_materialize_query`, so what it contains is + // build-dependent: the public assembly runs the plan verbatim (tests only + // in the post-commit summary breakdown), the enterprise assembly adds the + // in-transaction write-audit-publish guard (whose shape/placement is + // tested next to its implementation in windmill-common's + // `pipeline_advanced_ee`). + #[test] + fn materialize_rewrite_carries_data_test_summary() { + let script = "-- materialize ducklake://main/orders\n\ + -- data_test not_null id\n\ + SELECT id FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!(rewritten.contains("AS data_tests")); + assert_eq!(meta.n_data_tests, 1); + #[cfg(not(feature = "private"))] + assert!( + !rewritten.contains("error("), + "public assembly is commit-then-test (no guard)" + ); + #[cfg(all(feature = "private", feature = "enterprise"))] + assert!( + rewritten.contains("error("), + "enterprise assembly places the WAP guard" + ); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 83c25f3eec..f0a8c38f45 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -354,10 +354,22 @@ load-bearing. repeats the natural key across closed versions, so an unscoped `unique()` would fail the run on the second change of any key. Custom tests see the raw history and scope themselves. -- **Commit-then-test.** Like dbt, the write commits before tests run; a failed - test fails the *run* (and records `Failed`, so downstream cascade stops) but - does not roll back the committed snapshot. Time-travel still lets you inspect - exactly what failed. +- **Commit-then-test (public) / write-audit-publish (enterprise).** In the + public build, like dbt, the write commits before tests run; a failed test + fails the *run* (and records `Failed`, so downstream cascade stops) but does + not roll back the committed snapshot — time-travel still lets you inspect + exactly what failed. Enterprise upgrades this to write-audit-publish: the + same checks also run *inside* the write transaction as a guard statement + that raises on any violation, aborting the run before `COMMIT` — a failing + slice is never published, readers keep the previous version, and no snapshot + is created (even a *first* run of a new asset rolls back to "no table"). + The whole mechanism lives in the enterprise repo: + `sql_materialize::build_wrap_blocks` produces a typed statement plan + (`MaterializePlan` — statements plus structural kinds and the compiled + checks), and `pipeline_advanced::finalize_materialize_query` assembles it — + verbatim on the public build, restructured into the guarded transaction on + EE. The public build thus carries only plan metadata, not the + write-audit-publish transform itself. - **Custom = DuckDB SQL, server worker.** The escape hatch fetches the deployed script's content (a single DuckDB `SELECT`/CTE returning the violating rows — it's embedded as a subquery, so a multi-statement body is rejected with a diff --git a/docs/enterprise.md b/docs/enterprise.md index 5ab869a651..229585ad25 100644 --- a/docs/enterprise.md +++ b/docs/enterprise.md @@ -47,3 +47,19 @@ cargo check --features enterprise,private # EE code that also requires license validation cargo check --features enterprise,private,license ``` + +## Troubleshooting + +**`E0583: file not found for module _ee` on `cargo check --features enterprise,private`**: +the EE worktree is behind the OSS code it must satisfy. Fast-forward it to EE +`origin/main` — EE worktrees are shallow clones, so run `git fetch --unshallow origin` +first or the merge fails with "refusing to merge unrelated histories". After the +fast-forward, any `*_ee.rs` file that is *new* since the worktree was created still +needs its OSS symlink made by hand (worktree setup only links files that existed then). +The EE repo has no `backend/` prefix — crates sit at its root. Use an absolute target +(a relative one would resolve against the link's directory, not your cwd): + +```bash +# from the windmill repo root +ln -s ~/windmill-ee-private//src/_ee.rs backend//src/_ee.rs +``` diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index 4ad8e65fb9..126a4ac215 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -75,6 +75,12 @@ the annotation→verifier pattern that column-lineage will reuse. The keyword is `data_test`, not `test`, to stay clear of the unrelated `// test:` CI-test annotation. +Enterprise goes one step *beyond* dbt here: write-audit-publish. dbt commits +the model then tests it, so a failing test leaves the bad table live for BI +readers; the enterprise build runs the same probes inside the write +transaction and rolls back on violation, so a failing slice is never +published (see `ducklake-materialization.md` §"Scoping decisions"). + ### 2. Incremental materializations See [Incremental deep-dive](#incremental-deep-dive) below. diff --git a/docs/validation.md b/docs/validation.md index ba775d01cf..3bfd1325ce 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -10,6 +10,7 @@ After making changes, run the appropriate checks and fix all errors before consi | Enterprise code (`*_ee.rs`) | `cargo check --features enterprise,private` | Also do EE PR workflow (see `docs/enterprise.md`) | | Enterprise + license-gated code | `cargo check --features enterprise,private,license` | When the feature requires a valid license key | | Kafka trigger code | `cargo check --features kafka` | | +| DuckDB executor code | `cargo check -p windmill-worker --features duckdb` | `duckdb_executor.rs` (and its `#[cfg(test)]` tests) only compile with this flag — a plain check/test silently skips them | | Native trigger code | `cargo check --features native_trigger` | | | Parquet code | `cargo check --features parquet` | | | Multiple gated modules | `cargo check --features enterprise,parquet` | Combine only the flags you need | From 33521505dbc34f22b575d21fda1cc76d698a8840 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 15:23:23 +0000 Subject: [PATCH 189/273] feat(ducklake): scheduled lake maintenance (expiry, compaction, orphan cleanup) (#9916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ducklake): scheduled lake maintenance (snapshot expiry, compaction, orphan cleanup) Co-Authored-By: Claude Fable 5 * fix(ducklake): review fixes — starts_with not LIKE, CE license-lapse escape Co-Authored-By: Claude Fable 5 * docs(ducklake): auth-contract docs + _unchecked rename per codex review Co-Authored-By: Claude Fable 5 * refactor(ducklake): move maintenance payload construction into EE module Co-Authored-By: Claude Fable 5 * fix(ducklake): fall through to script resolution for non-managed reserved-prefix schedules Co-Authored-By: Claude Fable 5 * docs(ducklake): document accepted pre-existing-schedule limitation on the reserved prefix Co-Authored-By: Claude Fable 5 * fix(ducklake): CE save-off clears the managed schedule row and queued occurrence Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 2fab310d4f50ed7c34857d69c9b854f4491bf217 This commit updates the EE repository reference after PR #645 was merged in windmill-ee-private. Previous ee-repo-ref: fff1fd830a36beba732486f05941ec243cf6b640 New ee-repo-ref: 2fab310d4f50ed7c34857d69c9b854f4491bf217 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- ...d76a4e5702c0324a27ef3a5740ef5d98d8946.json | 31 +++ ...838563f28e832e37e5c165329441d8471acdb.json | 16 ++ ...da5d913caa8d935b33ad8f21336dedefd1c23.json | 15 ++ ...efef25885a9bca64bb2ba4bb173bc19dcbdba.json | 23 +++ ...08c547742851975163550ec4f3823057e91fe.json | 20 ++ ...4e6219434de4d2f9ae5e484ed8ce5da426109.json | 16 ++ ...c8453d53c973dce61cc8ace7d278773c7506f.json | 31 +++ ...6afc85a8350706fe694935b25ef46a847409e.json | 22 +++ ...6e1a9307335b0e7a27613cd690e99812a99f3.json | 23 +++ ...548e2783c2de7e92ba0ff7e6f4637419fcf7d.json | 16 ++ ...a14576509147e51a86a95a2cc786bee3199d9.json | 16 ++ ...6f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json | 16 ++ backend/ee-repo-ref.txt | 2 +- backend/windmill-api-schedule/src/lib.rs | 39 +++- backend/windmill-api-workspaces/Cargo.toml | 6 +- .../windmill-api-workspaces/src/workspaces.rs | 59 ++++-- backend/windmill-api/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 21 ++ backend/windmill-api/src/workspaces_export.rs | 7 +- backend/windmill-common/src/workspaces.rs | 74 +++++++ .../src/ducklake_maintenance_oss.rs | 96 +++++++++ backend/windmill-queue/src/lib.rs | 7 + backend/windmill-queue/src/schedule.rs | 23 ++- backend/windmill-queue/tests/schedule_push.rs | 168 ++++++++++++++++ .../workspaceSettings/DucklakeSettings.svelte | 187 +++++++++++++++++- 25 files changed, 907 insertions(+), 29 deletions(-) create mode 100644 backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json create mode 100644 backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json create mode 100644 backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json create mode 100644 backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json create mode 100644 backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json create mode 100644 backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json create mode 100644 backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json create mode 100644 backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json create mode 100644 backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json create mode 100644 backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json create mode 100644 backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json create mode 100644 backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json create mode 100644 backend/windmill-queue/src/ducklake_maintenance_oss.rs diff --git a/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json new file mode 100644 index 0000000000..0dbcf04c6e --- /dev/null +++ b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4)\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946" +} diff --git a/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json b/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json new file mode 100644 index 0000000000..c19df789d4 --- /dev/null +++ b/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, edited_by, edited_at, schedule, enabled, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n )\n SELECT\n $1, path, edited_by, edited_at, schedule, FALSE, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n FROM schedule WHERE workspace_id = $2 AND path NOT LIKE $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb" +} diff --git a/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json b/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json new file mode 100644 index 0000000000..a1e3edbbe2 --- /dev/null +++ b/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23" +} diff --git a/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json b/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json new file mode 100644 index 0000000000..dfbc2b75e6 --- /dev/null +++ b/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM schedule WHERE workspace_id = $1 AND path LIKE $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba" +} diff --git a/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json b/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json new file mode 100644 index 0000000000..a8facb9d1b --- /dev/null +++ b/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, enabled, email, permissioned_as, summary, tag, cron_version\n ) VALUES ($1, $2, $3, 'Etc/UTC', $4, $2, false, true, $5, $6, $7, 'duckdb', 'v2')\n ON CONFLICT (workspace_id, path) DO UPDATE SET\n schedule = EXCLUDED.schedule,\n timezone = EXCLUDED.timezone,\n edited_by = EXCLUDED.edited_by,\n edited_at = now(),\n script_path = EXCLUDED.script_path,\n is_flow = false,\n enabled = true,\n email = EXCLUDED.email,\n permissioned_as = EXCLUDED.permissioned_as,\n summary = EXCLUDED.summary,\n tag = EXCLUDED.tag,\n cron_version = EXCLUDED.cron_version,\n error = NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe" +} diff --git a/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json b/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json new file mode 100644 index 0000000000..8bce508e75 --- /dev/null +++ b/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2) AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109" +} diff --git a/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json b/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json new file mode 100644 index 0000000000..5741a1aa5b --- /dev/null +++ b/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND schedule.path NOT LIKE $4\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f" +} diff --git a/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json b/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json new file mode 100644 index 0000000000..829083e0ec --- /dev/null +++ b/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.ducklake->'ducklakes' AS ducklake_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e" +} diff --git a/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json b/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json new file mode 100644 index 0000000000..9fc8cf1336 --- /dev/null +++ b/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM schedule WHERE workspace_id = $1 AND starts_with(path, $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3" +} diff --git a/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json b/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json new file mode 100644 index 0000000000..e3e8e3f2ad --- /dev/null +++ b/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path LIKE $2 AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d" +} diff --git a/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json b/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json new file mode 100644 index 0000000000..21e0f4613b --- /dev/null +++ b/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND starts_with(path, $2) AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9" +} diff --git a/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json b/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json new file mode 100644 index 0000000000..d316c16fff --- /dev/null +++ b/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, edited_by, edited_at, schedule, enabled, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n )\n SELECT\n $1, path, edited_by, edited_at, schedule, FALSE, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n FROM schedule WHERE workspace_id = $2 AND NOT starts_with(path, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e31f18cdd0..2c16f06c50 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c +2fab310d4f50ed7c34857d69c9b854f4491bf217 diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 7cc0111c28..573c47e355 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -182,6 +182,19 @@ fn to_json_raw_opt( value.map(|v| sqlx::types::Json(to_raw_value(&v))) } +/// Managed ducklake-maintenance schedules live under a reserved path prefix; +/// their lifecycle is owned by the workspace ducklake settings, so the +/// schedule API refuses to create/edit/delete/toggle them. +fn reject_reserved_schedule_path(path: &str) -> Result<()> { + if path.starts_with(windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX) { + return Err(Error::BadRequest(format!( + "Schedules under {} are managed by the workspace ducklake settings", + windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX + ))); + } + Ok(()) +} + /// Validate that a dynamic skip handler (script or flow) exists async fn validate_dynamic_skip<'c>( tx: &mut Transaction<'c, Postgres>, @@ -219,6 +232,7 @@ async fn create_schedule( Json(ns): Json, ) -> Result { check_scopes(&authed, || format!("schedules:write:{}", ns.path))?; + reject_reserved_schedule_path(&ns.path)?; let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await; @@ -468,6 +482,7 @@ async fn edit_schedule( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("schedules:write:{}", path))?; + reject_reserved_schedule_path(path)?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; let mut tx = user_db.begin(&authed).await?; @@ -750,6 +765,15 @@ async fn list_schedule( ) .order_by("edited_at", true) .and_where("workspace_id = ?".bind(&w_id)) + // managed ducklake-maintenance schedules are edited from the + // workspace ducklake settings, not the schedules UI/CLI. + // starts_with, not LIKE: the prefix contains `_` which LIKE treats as + // a wildcard, and a user folder like `ducklake-maintenance` must not + // be swept up. + .and_where( + "NOT starts_with(path, ?)" + .bind(&windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX), + ) .offset(offset) .limit(per_page) .clone(); @@ -923,12 +947,13 @@ async fn list_schedule_with_jobs( ORDER BY completed_at DESC LIMIT 20 ) AS jobs) t - WHERE workspace_id = $1 + WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4) ORDER BY edited_at DESC LIMIT $2 OFFSET $3", w_id, per_page as i64, - offset as i64 + offset as i64, + windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX ) .fetch_all(&mut *tx) .await?; @@ -1014,6 +1039,7 @@ pub async fn set_enabled( let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); check_scopes(&authed, || format!("schedules:write:{}", path))?; + reject_reserved_schedule_path(path)?; // Block enabling a schedule in a fork when the parent has the same path // (regardless of parent's enabled flag), unless force=true. Two enabled @@ -1190,6 +1216,7 @@ async fn delete_schedule( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("schedules:write:{}", path))?; + reject_reserved_schedule_path(path)?; let mut tx = user_db.begin(&authed).await?; clear_schedule(&mut tx, path, &w_id).await?; @@ -1399,6 +1426,14 @@ async fn set_default_error_handler( } } for updated_schedule_path in updated_schedules { + // managed ducklake-maintenance rows get the handler update (their + // failures should reach workspace handlers) but must not be + // pushed into git-sync as deployed schedules + if updated_schedule_path + .starts_with(windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX) + { + continue; + } handle_deployment_metadata( &authed.email, &authed.username, diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 4a9e04c417..fc5a1fe874 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -11,7 +11,11 @@ path = "src/lib.rs" [features] default = [] enterprise = ["windmill-common/enterprise"] -private = ["windmill-common/private"] +private = [ + "windmill-common/private", + "windmill-queue/private", + "windmill-audit/private", +] cloud = ["windmill-common/cloud"] no_auth = ["windmill-api-auth/no_auth"] parquet = ["windmill-object-store/parquet"] diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 43d4ef241b..aaf04972e1 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2295,6 +2295,17 @@ async fn edit_ducklake_config( require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &email).await.is_ok(); + // Lake names end up interpolated in `ATTACH 'ducklake://'`, + // generated maintenance SQL and the reserved maintenance schedule path + // (CHECK-constrained to [\w-]+ segments). + for name in new_config.settings.ducklakes.keys() { + if !windmill_common::workspaces::is_valid_ducklake_name(name) { + return Err(Error::BadRequest(format!( + "Invalid ducklake name '{name}': only letters, digits, '_' and '-' are allowed" + ))); + } + } + let mut tx = db.begin().await?; let args_for_audit = format!("{:?}", new_config.settings); @@ -2309,21 +2320,22 @@ async fn edit_ducklake_config( ) .await?; + let old_ducklakes = sqlx::query_scalar!( + r#" + SELECT ws.ducklake->'ducklakes' AS ducklake_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(serde_json::Value::Null); + let old_ducklakes: HashMap = + serde_json::from_value(old_ducklakes).unwrap_or_default(); + // Check that non-superadmins are not abusing Instance databases if !is_superadmin { - let old_ducklakes = sqlx::query_scalar!( - r#" - SELECT ws.ducklake->'ducklakes' AS ducklake_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(serde_json::Value::Null); - let old_ducklakes: HashMap = - serde_json::from_value(old_ducklakes).unwrap_or_default(); for (name, dl) in new_config.settings.ducklakes.iter() { if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { let old_dl = old_ducklakes.get(name); @@ -2341,7 +2353,7 @@ async fn edit_ducklake_config( } } - let config: serde_json::Value = serde_json::to_value(new_config.settings) + let config: serde_json::Value = serde_json::to_value(&new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( @@ -2352,6 +2364,19 @@ async fn edit_ducklake_config( .execute(&mut *tx) .await?; + // Same tx as the settings update: a failed schedule sync/push must fail + // the whole save — nothing reconciles a half-applied state later. + let tx = windmill_queue::ducklake_maintenance::sync_ducklake_maintenance_schedules( + &db, + tx, + &w_id, + &new_config.settings.ducklakes, + &old_ducklakes, + &username, + &email, + ) + .await?; + tx.commit().await?; Ok(format!("Edit ducklake config for workspace {}", &w_id)) @@ -4079,6 +4104,9 @@ async fn clone_triggers_and_schedules( source_workspace_id: &str, target_workspace_id: &str, ) -> Result<()> { + // Managed ducklake-maintenance schedules are excluded: the fork starts + // with no ducklake config, so cloned rows could never resolve a lake and + // the schedule API refuses mutations under the reserved prefix. sqlx::query!( r#"INSERT INTO schedule ( workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, @@ -4095,9 +4123,10 @@ async fn clone_triggers_and_schedules( on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry, summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args, cron_version, description, dynamic_skip, permissioned_as, labels - FROM schedule WHERE workspace_id = $2"#, + FROM schedule WHERE workspace_id = $2 AND NOT starts_with(path, $3)"#, target_workspace_id, source_workspace_id, + windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX, ) .execute(&mut **tx) .await?; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index ebbc98e83e..fa9ed3f2e5 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1f836cfb2b..fe7edb26b0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -28917,6 +28917,27 @@ components: - path extra_args: type: string + maintenance: + type: object + description: + Scheduled maintenance (enterprise) - snapshot expiry, adjacent-file + compaction and orphaned-file cleanup, run as a managed per-lake schedule + required: [enabled] + properties: + enabled: + type: boolean + schedule: + type: string + description: cron cadence (v2, UTC); defaults to daily at 03h with a per-lake minute offset + retention_days: + type: integer + description: snapshot retention window in days (default 7); time-travel older than this stops working + compaction: + type: boolean + description: merge adjacent small parquet files (default true) + orphan_cleanup: + type: boolean + description: delete orphaned files older than max(retention, 1 day) (default true) DataTableSettings: type: object diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 66bda77bac..fa112ef4e1 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -919,11 +919,16 @@ pub(crate) async fn tarball_workspace( } if include_schedules.unwrap_or(false) { + // Managed ducklake-maintenance schedules are excluded: they are + // derived from the workspace ducklake settings (and admins bypass the + // RLS that hides them), so exporting them would drag unsyncable rows + // into git. let schedules = sqlx::query_as::<_, Schedule>( "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule - WHERE workspace_id = $1", + WHERE workspace_id = $1 AND NOT starts_with(path, $2)", ) .bind(&w_id) + .bind(windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX) .fetch_all(&mut *tx) .await?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5593a565ef..293490e434 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -842,6 +842,80 @@ pub struct Ducklake { pub storage: DucklakeStorage, #[serde(skip_serializing_if = "Option::is_none")] pub extra_args: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub maintenance: Option, +} + +/// Scheduled maintenance for a ducklake (enterprise): snapshot expiry, +/// adjacent-file compaction and orphaned-file cleanup, run as a managed +/// per-lake schedule. Not mirrored in `instance_config::Ducklake`: +/// instance-level lakes have no workspace to schedule into. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct DucklakeMaintenance { + pub enabled: bool, + /// Cron (v2/croner, seconds optional). None → daily at 03:00 UTC with a + /// deterministic per-(workspace, lake) minute offset. + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule: Option, + /// Snapshot retention window in days (default 7). Snapshots older than + /// this are expired: time-travel reads (`AT (VERSION => n)`) older than + /// the window stop working. 0 keeps only the current snapshot. + #[serde(skip_serializing_if = "Option::is_none")] + pub retention_days: Option, + /// Merge adjacent small parquet files (default true). + #[serde(skip_serializing_if = "Option::is_none")] + pub compaction: Option, + /// Delete orphaned files older than max(retention, 1 day) (default true). + #[serde(skip_serializing_if = "Option::is_none")] + pub orphan_cleanup: Option, +} + +impl DucklakeMaintenance { + pub const DEFAULT_RETENTION_DAYS: u32 = 7; + + pub fn retention_days(&self) -> u32 { + self.retention_days.unwrap_or(Self::DEFAULT_RETENTION_DAYS) + } + pub fn compaction(&self) -> bool { + self.compaction.unwrap_or(true) + } + pub fn orphan_cleanup(&self) -> bool { + self.orphan_cleanup.unwrap_or(true) + } +} + +/// Reserved schedule path namespace for managed ducklake maintenance +/// schedules. Must satisfy the `schedule.path` CHECK constraint +/// (`^[ufg](\/[\w-]+){2,}$`), hence the `f/` prefix; the folder itself never +/// exists. The schedule API rejects user mutations under this prefix and the +/// list/export endpoints filter it out — the lifecycle is owned by the +/// workspace ducklake settings. +/// +/// Accepted limitation: a schedule that pre-dated this namespace under a real +/// `ducklake_maintenance` folder keeps running (tick dispatch falls through +/// to its script when its path's lake has no enabled maintenance config, and +/// the settings sync only touches rows derived from config) but stays hidden +/// from list/export and immutable via the schedule API until renamed out of +/// the namespace. Judged unlikely enough to not warrant a discriminator +/// column or a rename migration. +pub const DUCKLAKE_MAINTENANCE_PATH_PREFIX: &str = "f/ducklake_maintenance/"; + +pub fn ducklake_maintenance_schedule_path(lake: &str) -> String { + format!("{DUCKLAKE_MAINTENANCE_PATH_PREFIX}{lake}") +} + +pub fn lake_from_ducklake_maintenance_path(path: &str) -> Option<&str> { + path.strip_prefix(DUCKLAKE_MAINTENANCE_PATH_PREFIX) +} + +/// Lake names are interpolated into `ATTACH 'ducklake://'`, generated +/// maintenance SQL and the reserved schedule path (CHECK-constrained to +/// `[\w-]+` segments), so they must stay to this charset. +pub fn is_valid_ducklake_name(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-queue/src/ducklake_maintenance_oss.rs b/backend/windmill-queue/src/ducklake_maintenance_oss.rs new file mode 100644 index 0000000000..ab6e035847 --- /dev/null +++ b/backend/windmill-queue/src/ducklake_maintenance_oss.rs @@ -0,0 +1,96 @@ +//! OSS fallback: scheduled ducklake maintenance (snapshot expiry, adjacent-file +//! compaction, orphaned-file cleanup via managed per-lake schedules) is an +//! enterprise feature; the implementation lives in windmill-ee-private +//! (see `ducklake_maintenance_ee`). In the public build the entry points report +//! that the enterprise edition is required. + +use std::collections::HashMap; + +use sqlx::{Postgres, Transaction}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobPayload, + schedule::Schedule, + workspaces::Ducklake, + DB, +}; + +/// Reconcile the managed `f/ducklake_maintenance/` schedule rows with +/// the ducklake settings, inside the caller's transaction. +/// +/// Not an authorization boundary: it mutates schedule rows for `w_id` on +/// behalf of `edited_by`/`email`, so the caller MUST already have enforced +/// workspace-admin on `w_id` and that the identity is the authenticated +/// caller's (as `edit_ducklake_config` does via `require_admin`). +// Only newly-enabled maintenance is rejected: a config that already had it +// enabled (e.g. an enterprise license lapsed) must not make every unrelated +// ducklake settings save fail, and the admin must be able to save it off. +pub async fn sync_ducklake_maintenance_schedules<'c>( + _db: &DB, + mut tx: Transaction<'c, Postgres>, + w_id: &str, + ducklakes: &HashMap, + previous: &HashMap, + _edited_by: &str, + _email: &str, +) -> Result> { + let enabled = |dl: &Ducklake| -> bool { dl.maintenance.as_ref().is_some_and(|m| m.enabled) }; + if ducklakes + .iter() + .any(|(name, dl)| enabled(dl) && !previous.get(name).is_some_and(|prev| enabled(prev))) + { + return Err(Error::BadRequest( + "Ducklake scheduled maintenance is only available in the enterprise edition" + .to_string(), + )); + } + + // Saving maintenance off (e.g. after an enterprise license lapsed) must + // remove the managed row AND its already-queued occurrence here too — + // otherwise a maintenance job pushed under the enterprise edition still + // runs once after the admin disabled it. Like the enterprise + // implementation, the removed set is derived from config, never from the + // path prefix. + let removed = previous + .iter() + .filter(|(name, dl)| enabled(dl) && !ducklakes.get(*name).is_some_and(|cur| enabled(cur))) + .map(|(name, _)| windmill_common::workspaces::ducklake_maintenance_schedule_path(name)) + .collect::>(); + for path in removed.iter() { + crate::schedule::clear_schedule(&mut tx, path, w_id).await?; + } + sqlx::query!( + "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2)", + w_id, + &removed + ) + .execute(&mut *tx) + .await?; + + Ok(tx) +} + +/// Build the job payload for one occurrence of a managed maintenance schedule +/// (`push_scheduled_job` calls this for reserved-prefix schedule paths). +/// Returns `(payload, tag, timeout, on_behalf_of_email, created_by)`. +/// +/// Always `Ok(None)` in the public build: the caller falls through to normal +/// script resolution, so a pre-existing user schedule under a real +/// `ducklake_maintenance` folder keeps running, while a managed row left over +/// from an enterprise period fails script resolution with NotFound and is +/// auto-disabled with `schedule.error` recorded by the post-completion +/// scheduler. +pub async fn build_maintenance_schedule_payload<'c>( + _tx: &mut Transaction<'c, Postgres>, + _schedule: &Schedule, +) -> Result< + Option<( + JobPayload, + Option, + Option, + Option, + String, + )>, +> { + Ok(None) +} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 59ac7f1fa8..8d8534c926 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -15,6 +15,13 @@ pub use cascade_ee as cascade; #[cfg(not(feature = "private"))] pub use cascade_oss as cascade; #[cfg(feature = "private")] +pub mod ducklake_maintenance_ee; +pub mod ducklake_maintenance_oss; +#[cfg(feature = "private")] +pub use ducklake_maintenance_ee as ducklake_maintenance; +#[cfg(not(feature = "private"))] +pub use ducklake_maintenance_oss as ducklake_maintenance; +#[cfg(feature = "private")] pub mod freshness_watchdog_ee; pub mod freshness_watchdog_oss; #[cfg(feature = "private")] diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 8816eeae25..2b5aef96a5 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -227,11 +227,30 @@ pub async fn push_scheduled_job<'c>( } } + // Managed ducklake maintenance schedule (enterprise): the runnable is a + // generated DuckDB script, not a deployed one — built in the EE module. + // None (CE build, or no enabled maintenance config for the path's lake) + // falls through to normal script resolution, so a user schedule that + // pre-dates the reserved prefix keeps running its script and a stale + // managed row fails resolution with NotFound (auto-disabling it with + // schedule.error recorded). + let maintenance_payload = + if windmill_common::workspaces::lake_from_ducklake_maintenance_path(&schedule.path) + .is_some() + { + crate::ducklake_maintenance::build_maintenance_schedule_payload(&mut tx, schedule) + .await? + } else { + None + }; + // If schedule handler is defined, wrap the scheduled job in a synthetic flow // with the handler as the first step (with stop_after_if to skip if handler returns false) - let (payload, tag, timeout, on_behalf_of_email, created_by) = if let Some(handler_path) = - &schedule.dynamic_skip + let (payload, tag, timeout, on_behalf_of_email, created_by) = if let Some(maintenance_payload) = + maintenance_payload { + maintenance_payload + } else if let Some(handler_path) = &schedule.dynamic_skip { // Build skip handler args let mut skip_handler_args = HashMap::>::new(); skip_handler_args.insert( diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index 70a53763e0..745b2685e9 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -1592,4 +1592,172 @@ mod schedule_push { Ok(()) } + + // ----------------------------------------------------------------------- + // push_scheduled_job: reserved ducklake-maintenance prefix + // ----------------------------------------------------------------------- + + // A schedule that pre-dates the reserved prefix (a user schedule under a + // real `ducklake_maintenance` folder) must fall through to normal script + // resolution when its path's lake has no enabled maintenance config — + // never be hijacked into the maintenance payload builder and auto-disabled. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_push_reserved_prefix_no_config_falls_through_to_script( + db: Pool, + ) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.path = "f/ducklake_maintenance/legacy".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + let (ws, path, trigger, _) = get_queued_job(&db).await.unwrap(); + assert_eq!(ws, "test-workspace"); + assert_eq!( + path.as_deref(), + Some("f/system/test_script"), + "must resolve the schedule's script_path, not the maintenance builder" + ); + assert_eq!(trigger.as_deref(), Some("f/ducklake_maintenance/legacy")); + Ok(()) + } + + // With maintenance enabled for the path's lake, the occurrence is a + // raw-code duckdb job (kind preview, runnable_path = schedule path, + // duckdb tag pinned) — enterprise builds only. + #[cfg(feature = "private")] + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_push_reserved_prefix_with_config_builds_maintenance_job( + db: Pool, + ) -> anyhow::Result<()> { + sqlx::query( + r#"UPDATE workspace_settings SET ducklake = '{"ducklakes": {"legacy": { + "catalog": {"resource_type": "postgresql", "resource_path": "u/test/pg"}, + "storage": {"path": "legacy"}, + "maintenance": {"enabled": true, "retention_days": 3} + }}}'::jsonb WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/ducklake_maintenance/legacy".to_string(); + s.script_path = "f/ducklake_maintenance/legacy".to_string(); + s.tag = Some("duckdb".to_string()); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + let (kind, path, tag, raw_code) = sqlx::query_as::<_, (String, Option, String, Option)>( + "SELECT j.kind::text, j.runnable_path, j.tag, j.raw_code + FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(kind, "preview"); + assert_eq!(path.as_deref(), Some("f/ducklake_maintenance/legacy")); + assert_eq!(tag, "duckdb"); + let raw_code = raw_code.expect("maintenance job must carry generated SQL"); + assert!(raw_code.contains("ducklake_expire_snapshots")); + assert!(raw_code.contains("INTERVAL '3 days'")); + Ok(()) + } + + // Saving maintenance off must remove the managed row AND its queued + // occurrence in BOTH builds: the enterprise sync reconciles, and the + // public stub must not leave a job pushed under the enterprise edition + // to run after the admin disabled maintenance (EE-to-CE downgrade). + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_sync_disable_clears_managed_row_and_queued_occurrence( + db: Pool, + ) -> anyhow::Result<()> { + use std::collections::HashMap; + use windmill_common::workspaces::{ + Ducklake, DucklakeCatalog, DucklakeCatalogResourceType, DucklakeMaintenance, + DucklakeStorage, + }; + use windmill_queue::ducklake_maintenance::sync_ducklake_maintenance_schedules; + + fn lake(maintenance_enabled: bool) -> Ducklake { + Ducklake { + catalog: DucklakeCatalog { + resource_type: DucklakeCatalogResourceType::Postgresql, + resource_path: "u/test/pg".to_string(), + }, + storage: DucklakeStorage { storage: None, path: "legacy".to_string() }, + extra_args: None, + maintenance: Some(DucklakeMaintenance { + enabled: maintenance_enabled, + schedule: None, + retention_days: None, + compaction: None, + orphan_cleanup: None, + }), + } + } + + // a managed row with a queued occurrence (queued via fall-through: no + // lake config exists yet, so the push resolves the script path) + let schedule = make_schedule(|s| { + s.path = "f/ducklake_maintenance/legacy".to_string(); + }); + sqlx::query( + "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, + is_flow, enabled, email, permissioned_as, cron_version) + VALUES ($1, $2, $3, 'UTC', $4, $5, false, true, $6, $7, 'v2')", + ) + .bind(&schedule.workspace_id) + .bind(&schedule.path) + .bind(&schedule.schedule) + .bind(&schedule.edited_by) + .bind(&schedule.script_path) + .bind(&schedule.email) + .bind(&schedule.permissioned_as) + .execute(&db) + .await?; + let authed = make_authed(); + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + + // maintenance saved off + let previous = HashMap::from([("legacy".to_string(), lake(true))]); + let current = HashMap::from([("legacy".to_string(), lake(false))]); + let tx = db.begin().await?; + let tx = sync_ducklake_maintenance_schedules( + &db, + tx, + &schedule.workspace_id, + ¤t, + &previous, + "test-user", + "test@windmill.dev", + ) + .await?; + tx.commit().await?; + + assert_eq!( + count_queued_jobs(&db).await, + 0, + "queued occurrence must be cleared when maintenance is saved off" + ); + let row_exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM schedule WHERE workspace_id = $1 AND path = $2)", + ) + .bind(&schedule.workspace_id) + .bind(&schedule.path) + .fetch_one(&db) + .await?; + assert!(!row_exists, "managed schedule row must be deleted"); + Ok(()) + } } diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 4dd0ed0053..3ba4e9a213 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -1,6 +1,14 @@
    {#each tests as t (t.test)} -
  • - {#if t.violating > 0} - - failed: - {t.test} - — {t.violating} violating row{t.violating === 1 ? '' : 's'} - {:else} - - passed: - {t.test} +
  • +
    + {#if t.violating > 0} + + failed: + {t.test} + — {t.violating} violating row{t.violating === 1 ? '' : 's'} + {#if t.sample && t.sample.length > 0} + + {/if} + {:else} + + passed: + {t.test} + {/if} +
    + {#if t.violating > 0 && t.sample && t.sample.length > 0 && expanded.has(t.test)} +
    +
    + sample of the violating rows ({t.sample.length} of {t.violating}, unordered) +
    + +
    {/if}
    • {/each} diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index cbd81086a9..efb1dd2da1 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -574,24 +574,53 @@ // formats are produced by this repo's worker (see duckdb_executor.rs); the // derivation is inert (undefined) for every other DisplayResult use. let dataTests = $derived.by(() => { + // Both structured shapes carry `[{ test, violating, sample? }]`; the + // sample (bounded violating-row rows) may arrive as a JSON string (the + // worker keeps it string-typed through the summary row) and is optional + // by contract — anything malformed degrades to no sample, never to a + // dropped checklist. + const normalize = ( + dt: any + ): Array<{ test: string; violating: number; sample?: Record[] }> | undefined => { + if (typeof dt === 'string') { + try { + dt = JSON.parse(dt) + } catch { + return undefined + } + } + if ( + !Array.isArray(dt) || + dt.length === 0 || + !dt.every((x) => x && typeof x.test === 'string' && typeof x.violating === 'number') + ) { + return undefined + } + return dt.map((x) => { + let sample = x.sample + if (typeof sample === 'string') { + try { + sample = JSON.parse(sample) + } catch { + sample = undefined + } + } + if (!Array.isArray(sample) || !sample.every((r) => r && typeof r === 'object')) { + sample = undefined + } + return { test: x.test, violating: x.violating, sample } + }) + } // Success: structured column on the summary row. const row = Array.isArray(result) ? (result as any)?.[0] : (result as any) - let dt = row?.data_tests - if (typeof dt === 'string') { - try { - dt = JSON.parse(dt) - } catch { - dt = undefined - } - } - if ( - Array.isArray(dt) && - dt.length > 0 && - dt.every((x) => x && typeof x.test === 'string' && typeof x.violating === 'number') - ) { - return dt as Array<{ test: string; violating: number }> - } - // Failure: parse the worker's breakdown out of the error message. + const fromRow = normalize(row?.data_tests) + if (fromRow) return fromRow + // Failure: the worker attaches the same structured breakdown (plus + // per-failed-test samples) to the error payload. + const fromError = normalize((result as any)?.error?.data_tests) + if (fromError) return fromError + // Failure fallback for results predating the structured error payload: + // parse the worker's breakdown out of the error message. const msg = (result as any)?.error?.message if (typeof msg === 'string' && msg.includes('data tests failed on')) { const out: Array<{ test: string; violating: number }> = [] From 39eb9de1bce400109c130a081807e40e995ae068 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 09:58:06 +0200 Subject: [PATCH 192/273] feat(pipelines): fork data environments for ducklake materialization (dev data) (#9915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): fork-scoped ducklake namespaces with read-defer to parent Co-Authored-By: Claude Fable 5 * feat(pipelines): fork graph indicator + fork ducklake namespace cleanup endpoint Co-Authored-By: Claude Fable 5 * feat(pipelines): fork_views-keyed view transition, fork lineage clone, design doc Co-Authored-By: Claude Fable 5 * fix(pipelines): review hardening - fork DATA_PATH last-wins, registry cache TTL, defer tests Co-Authored-By: Claude Fable 5 * feat(pipelines): per-lake isolated/shared choice at fork creation Co-Authored-By: Claude Fable 5 * fix(pipelines): chain-aware defer discovery + per-location fork namespace registry Co-Authored-By: Claude Fable 5 * fix(pipelines): lake-scoped fork schemas, catalog identity in registry, chain-aware graph chips Co-Authored-By: Claude Fable 5 * fix(pipelines): cleanup deletes fork data from the registered storage identity Co-Authored-By: Claude Fable 5 * fix(pipelines): collapse fork data-path segment to one component (slash-safe ids) Co-Authored-By: Claude Fable 5 * fix(pipelines): per-catalog ancestor checks, ancestor extra_args passthrough, test compile fix Co-Authored-By: Claude Fable 5 * fix(pipelines): invalidate fork ancestor-chain cache on lineage mutations Co-Authored-By: Claude Fable 5 * fix(pipelines): sweep descendant ancestor-chain caches on delete/reparent Co-Authored-By: Claude Fable 5 * fix(pipelines): run fork ducklake cleanup inline in delete_workspace Co-Authored-By: Claude Fable 5 * fix(pipelines): resolve fork cleanup credentials pre-commit, destroy post-commit Co-Authored-By: Claude Fable 5 * fix(pipelines): shared dev-workspace authz gate for namespace drop, invalidatable registration cache, segment-boundary delete filter - extract require_prod_admin_for_dev_workspace, used by both delete_workspace and drop_forked_ducklake_namespaces so the gates cannot drift - key FORK_DUCKLAKE_REGISTERED per workspace and invalidate it in cleanup_fork_ducklake_namespaces so a same-id fork recreated within the TTL re-registers its namespaces - filter listed object locations to the segment boundary before deletion Co-Authored-By: Claude Fable 5 * fix(pipelines): keep orphaned wm-fork-* workspaces ducklake-isolated parent_workspace_id is ON DELETE SET NULL, so a fork can outlive its parent with an empty ancestor chain while its cloned config still points at the shared lake. Key the isolation gate on the wm-fork- prefix as well as the chain (mirroring workspace_is_fork): orphaned forks get the write redirect, registration and cleanup with zero ancestors (no defer), and keep their 'fork' graph chips. Co-Authored-By: Claude Fable 5 * fix(pipelines): attach orphaned wm-fork-* ancestors at their fork namespace Chain position alone classified the last ancestor as a root, but an orphaned wm-fork-* ancestor (its own parent deleted, SET NULL) ends the chain the same way while its data lives in its fork namespace — its descendants' defer views bound the dead root's lake instead. Key the root-vs-fork decision on the wm-fork- prefix too, matching the resolution gate. Co-Authored-By: Claude Fable 5 * fix(pipelines): never inherit shared lake opt-out; durable cleanup ledger for failed fork deletions - fork creation strips cloned fork_behavior stamps before applying the request's shared_ducklakes list: sharing is a per-creation choice, a fork of a shared fork defaults back to isolated - fork_ducklake_namespace loses its ON DELETE CASCADE FK: rows are the durable cleanup ledger and outlive the workspace when physical cleanup fails post-commit; fork creation retries leftover rows for the reused id and refuses to create while a metadata schema still cannot be dropped (data-file leftovers alone are inert once the schema is gone and are swept by the next successful same-prefix cleanup) Co-Authored-By: Claude Fable 5 * fix(pipelines): make orphaned-namespace cleanup retries independent of deleted fork resources - ledger rows gain a schema_dropped phase flag: set when the schema drop succeeded but data cleanup failed, so later retries skip the schema phase and need no catalog credentials at all; registration resets it on re-attach (ON CONFLICT DO UPDATE) since attaching recreates the schema - retry-path $res: resolution falls back to the workspace being forked (the deleted fork's resources were clones of a parent's); live paths (delete_workspace prepare, drop endpoint) pass no fallback Co-Authored-By: Claude Fable 5 * fix(pipelines): fork tables from failed-after-commit runs stay fork-owned in defer and graph A failed materialization must not disguise a physically existing fork table as deferred: CREATE VIEW IF NOT EXISTS silently yields to the table, so reads hit fork data while the graph claims parent defer. - record_mat upsert preserves the last committed snapshot_id on failure - defer discovery and graph chips treat fork rows with a committed snapshot as fork-owned even when status is failed - inspect_fork_catalog also lists live fork tables (same round trip) and the defer list is filtered against them — covers rows recorded before this fix and tables created by raw SQL - drop stale FK-cascade wording in the design doc and sidebar comment Co-Authored-By: Claude Fable 5 * feat(pipelines): fork-mode ducklake settings — per-lake isolated/shared chips + banner, fork_behavior round-trip The workspace-settings ducklake editor had no fork awareness: no reminder of each lake's isolated/shared choice and no warning about what edits mean in a fork. It also rebuilt each lake explicitly on save, silently dropping fork_behavior — any settings save in a shared fork flipped the lake back to isolated. - fork detection mirrors the backend gate (parent link or wm-fork- prefix) - info banner explaining isolated vs shared semantics in a fork - per-lake chip (emerald 'isolated' / amber 'shared with parent') with tooltips, matching the pipeline graph chip colors - fork_behavior added to DucklakeSettingsType and preserved through convertDucklakeSettingsToBackend Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- ...d15474599734bc46ede6930650e2cba5d8a46.json | 15 + ...950a1563bd516e730aaa9d0f1501ea8cea7f9.json | 58 ++ ...6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json | 36 + ...4a6ae5091c37ec901975c3d564fd817a405b7.json | 23 + ...d4e427342fb78c66980984aefa7b8c7592ecb.json | 22 + ...c20a9e310092ae787efe4b0657399c799424.json} | 4 +- ...01fcab837629fd3c1a09cf82b21641a890444.json | 29 + ...650fb861aa19a009d6dfe9937304057f2b91f.json | 22 + ...a48ebfed85ad8800e293945eadc874d7e42f3.json | 19 + ...88958b335f6a24d1e93641eab6aaac86301f7.json | 14 + ...cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json | 20 + ...92da8ee231c772f88dd9145f01d7cf3b8f8c1.json | 15 + ...b134780e55a521a5cee8eb657b67bbd4b2588.json | 19 + backend/Cargo.lock | 1 + ...703170745_fork_ducklake_namespace.down.sql | 1 + ...60703170745_fork_ducklake_namespace.up.sql | 51 ++ backend/windmill-api-assets/src/lib.rs | 89 +- backend/windmill-api-workspaces/Cargo.toml | 3 +- .../windmill-api-workspaces/src/workspaces.rs | 126 +++ .../src/workspaces_extra.rs | 602 ++++++++++++- backend/windmill-api/openapi.yaml | 33 + .../windmill-common/src/materialization.rs | 98 +- backend/windmill-common/src/workspaces.rs | 841 +++++++++++++++++- backend/windmill-queue/tests/schedule_push.rs | 12 +- .../windmill-worker/src/duckdb_executor.rs | 372 +++++++- docs/pipeline-env-isolation.md | 219 +++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 1 + .../AssetGraph/AssetGraphDetailsPane.svelte | 33 + .../assets/AssetGraph/AssetNode.svelte | 24 +- .../AssetGraph/PipelineGraphEditor.svelte | 4 + .../lib/components/assets/AssetGraph/types.ts | 5 + .../components/sidebar/SidebarContent.svelte | 19 + .../CreateWorkspaceInner.svelte | 4 + .../workspaceSettings/DucklakeSettings.svelte | 66 +- .../ForkDucklakeSection.svelte | 61 ++ .../(logged)/pipeline/[folder]/+page.svelte | 11 + 36 files changed, 2930 insertions(+), 42 deletions(-) create mode 100644 backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json create mode 100644 backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json create mode 100644 backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json create mode 100644 backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json create mode 100644 backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json rename backend/.sqlx/{query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json => query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json} (63%) create mode 100644 backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json create mode 100644 backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json create mode 100644 backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json create mode 100644 backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json create mode 100644 backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json create mode 100644 backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json create mode 100644 backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json create mode 100644 backend/migrations/20260703170745_fork_ducklake_namespace.down.sql create mode 100644 backend/migrations/20260703170745_fork_ducklake_namespace.up.sql create mode 100644 docs/pipeline-env-isolation.md create mode 100644 frontend/src/lib/components/workspaceSettings/ForkDucklakeSection.svelte diff --git a/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json b/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json new file mode 100644 index 0000000000..5f74d57491 --- /dev/null +++ b/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s)\n SELECT $2, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s\n FROM script_trigger WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46" +} diff --git a/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json b/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json new file mode 100644 index 0000000000..4599073796 --- /dev/null +++ b/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ducklake_name AS \"ducklake_name!\", metadata_schema AS \"metadata_schema!\",\n catalog AS \"catalog!\", storage AS \"storage!\",\n storage_ref AS \"storage_ref!\", data_path AS \"data_path!\",\n schema_dropped AS \"schema_dropped!\"\n FROM fork_ducklake_namespace WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "metadata_schema!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "catalog!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "storage!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "storage_ref!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "data_path!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "schema_dropped!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9" +} diff --git a/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json b/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json new file mode 100644 index 0000000000..1bcb7e6e53 --- /dev/null +++ b/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH ancestor AS (\n SELECT wid, ord FROM unnest($1::text[]) WITH ORDINALITY AS a(wid, ord)\n ), anc_mat AS (\n -- Per table, the nearest ancestor (lowest ord) that materialized it.\n SELECT DISTINCT ON (mp.asset_path) mp.asset_path, a.wid, a.ord\n FROM materialized_partition mp\n JOIN ancestor a ON a.wid = mp.workspace_id\n WHERE mp.asset_kind = 'ducklake' AND mp.status = 'materialized'\n AND split_part(mp.asset_path, '/', 1) = $3 AND mp.asset_path LIKE '%/%'\n ORDER BY mp.asset_path, a.ord\n ), fork_mat AS (\n -- Fork-OWNED assets: anything whose physical table exists in the fork\n -- namespace, not just clean materializations. A committed write whose data\n -- tests failed afterwards records status='failed' WITH a snapshot — its table\n -- is real, and a defer view emitted over it would silently yield to it\n -- (CREATE VIEW IF NOT EXISTS) while claiming the read defers to the parent.\n SELECT DISTINCT asset_path FROM materialized_partition\n WHERE workspace_id = $2 AND asset_kind = 'ducklake'\n AND (status = 'materialized' OR snapshot_id IS NOT NULL)\n ), latest_schema AS (\n SELECT DISTINCT ON (workspace_id, asset_path) workspace_id, asset_path, columns\n FROM materialized_asset_schema\n WHERE workspace_id = ANY($1) AND asset_kind = 'ducklake'\n ORDER BY workspace_id, asset_path, version DESC\n )\n SELECT am.asset_path AS \"asset_path!\",\n am.ord AS \"ord!\",\n COALESCE(EXISTS (\n SELECT 1 FROM jsonb_array_elements(ls.columns) e\n WHERE e->>'name' = 'is_current'\n ), false) AS \"has_current!\"\n FROM anc_mat am\n LEFT JOIN latest_schema ls\n ON ls.asset_path = am.asset_path AND ls.workspace_id = am.wid\n WHERE am.asset_path NOT IN (SELECT asset_path FROM fork_mat)\n ORDER BY am.asset_path\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "ord!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "has_current!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541" +} diff --git a/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json b/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json new file mode 100644 index 0000000000..9a0bbf46ef --- /dev/null +++ b/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET ducklake = jsonb_set(ducklake, ARRAY['ducklakes', $2, 'fork_behavior'], '\"shared\"')\n WHERE workspace_id = $1 AND ducklake->'ducklakes' ? $2\n RETURNING 1 AS \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7" +} diff --git a/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json b/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json new file mode 100644 index 0000000000..b6b0f06be5 --- /dev/null +++ b/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ducklake->'ducklakes' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb" +} diff --git a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json similarity index 63% rename from backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json rename to backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json index 17d372249b..5aacf1a295 100644 --- a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json +++ b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n snapshot_id = EXCLUDED.snapshot_id,\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", + "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n -- A failed run records no snapshot, but must not erase the last\n -- committed one: a physical table from an earlier commit (or from a\n -- committed write whose data tests then failed) still exists, and\n -- fork defer/graph state keys on that evidence.\n snapshot_id = COALESCE(EXCLUDED.snapshot_id, materialized_partition.snapshot_id),\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", "describe": { "columns": [], "parameters": { @@ -43,5 +43,5 @@ }, "nullable": [] }, - "hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19" + "hash": "853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424" } diff --git a/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json b/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json new file mode 100644 index 0000000000..efd12114f3 --- /dev/null +++ b/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n -- Fork rows also count when a snapshot ever committed (a failed run\n -- preserves it): the physical table exists, so reads hit the FORK's\n -- data — showing 'deferred' would misstate what a query returns.\n -- Ancestor rows still require a clean materialization.\n SELECT DISTINCT asset_path AS \"asset_path!\", workspace_id AS \"workspace_id!\"\n FROM materialized_partition\n WHERE (workspace_id = $1 OR workspace_id = ANY($2))\n AND asset_kind = 'ducklake'\n AND (status = 'materialized'\n OR (workspace_id = $1 AND snapshot_id IS NOT NULL))\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444" +} diff --git a/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json b/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json new file mode 100644 index 0000000000..0fac858572 --- /dev/null +++ b/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE depth > 0 ORDER BY depth\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f" +} diff --git a/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json b/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json new file mode 100644 index 0000000000..83f7394250 --- /dev/null +++ b/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE fork_ducklake_namespace SET schema_dropped = true\n WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3\n AND storage = $4 AND storage_ref = $5 AND data_path = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3" +} diff --git a/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json b/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json new file mode 100644 index 0000000000..f705dbc337 --- /dev/null +++ b/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET ducklake = jsonb_set(ducklake, '{ducklakes}', (\n SELECT COALESCE(jsonb_object_agg(key, value - 'fork_behavior'), '{}'::jsonb)\n FROM jsonb_each(ducklake->'ducklakes')\n ))\n WHERE workspace_id = $1 AND jsonb_typeof(ducklake->'ducklakes') = 'object'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7" +} diff --git a/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json b/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json new file mode 100644 index 0000000000..37bd25eb98 --- /dev/null +++ b/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO fork_ducklake_namespace\n (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path)\n DO UPDATE SET schema_dropped = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd" +} diff --git a/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json b/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json new file mode 100644 index 0000000000..7049fafe37 --- /dev/null +++ b/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n SELECT $2, path, kind, usage_access_type, usage_path, usage_kind, columns\n FROM asset WHERE workspace_id = $1 AND usage_kind IN ('script', 'flow')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1" +} diff --git a/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json b/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json new file mode 100644 index 0000000000..ad17a60330 --- /dev/null +++ b/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM fork_ducklake_namespace\n WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3\n AND storage = $4 AND storage_ref = $5 AND data_path = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aa66b845b3..1153916767 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14449,6 +14449,7 @@ version = "1.747.0" dependencies = [ "axum 0.8.9", "chrono", + "futures", "hex", "http 1.4.2", "hyper 1.10.1", diff --git a/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql b/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql new file mode 100644 index 0000000000..fd722bf414 --- /dev/null +++ b/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS fork_ducklake_namespace; diff --git a/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql b/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql new file mode 100644 index 0000000000..7762304199 --- /dev/null +++ b/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql @@ -0,0 +1,51 @@ +-- Registry of ducklake namespaces provisioned for fork/dev workspaces. One row per +-- (fork workspace, lake name): records the exact catalog metadata schema and data +-- sub-path the fork's jobs attach to, so fork deletion can drop the pg schema and +-- delete the S3 prefix deterministically (the row is written on first resolution, +-- before any physical state exists). +-- +-- Deliberately NO foreign key to workspace(id): rows are the durable cleanup ledger and +-- must OUTLIVE the workspace row when physical cleanup fails after the delete commits +-- (unreachable catalog, storage outage) — a CASCADE would erase the only record of the +-- orphaned namespace, letting a recreated same-id fork silently reattach stale tables. +-- Rows are deleted explicitly after each successful cleanup; leftover rows for a reused +-- id are retried at fork creation, which refuses to proceed while they cannot be cleaned. +CREATE TABLE fork_ducklake_namespace ( + workspace_id VARCHAR(50) NOT NULL, + ducklake_name VARCHAR(255) NOT NULL, + metadata_schema VARCHAR(63) NOT NULL, + -- Canonical identity of the catalog database the metadata schema lives in + -- (`:`, e.g. `instance:wm_ducklake` or + -- `postgres:u/admin/pg`). Cleanup connects to THIS catalog, not whatever the fork's + -- settings point at by then — a drifted catalog resource must not make cleanup drop a + -- schema in the wrong database and orphan the real one. + catalog TEXT NOT NULL, + -- Named workspace storage holding the fork's data files; '' = the default storage + -- (part of the PK, which cannot hold NULL). + storage TEXT NOT NULL DEFAULT '', + -- The storage's RESOLVED identity at registration time (`:`, e.g. `s3:u/admin/minio` or `filesystem:/data/lfs`; '' = unknown). Cleanup + -- deletes the fork prefix from THIS storage, not whatever the logical name points at by + -- then — repointing a storage after attach must not orphan the original fork data (or + -- delete a colliding prefix from the new one). + storage_ref TEXT NOT NULL DEFAULT '', + -- The fork namespace's data path within that storage (a bucket-root + -- `__wm_forks//…` prefix). + data_path TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Cleanup phase state: true once the metadata schema has been dropped but data files (or + -- the row delete) still failed. Later retries then skip the schema phase entirely — they + -- need NO catalog credentials, which may be gone for good with the deleted fork's + -- resources. Reset to false whenever a live fork re-registers the row (re-attaching + -- recreates the schema). + schema_dropped BOOLEAN NOT NULL DEFAULT false, + -- One row per physical location EVER attached: if the fork's lake settings drift + -- (catalog/storage/path change), later attaches add rows rather than replace them, so + -- cleanup covers every location the fork wrote, not just the first. + PRIMARY KEY (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path) +); + +-- Resolution runs under user_db transactions (SET LOCAL ROLE) in API contexts, so the +-- windmill roles need explicit grants (default privileges don't apply to app-created tables). +GRANT ALL ON fork_ducklake_namespace TO windmill_user; +GRANT ALL ON fork_ducklake_namespace TO windmill_admin; diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 587307afbb..2cebf9ed13 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -614,6 +614,12 @@ struct GraphQuery { struct GraphAssetNode { kind: AssetKind, path: String, + // Fork workspaces only: 'fork' when the fork has materialized this ducklake asset itself, + // 'deferred' when reads fall back to the parent workspace's current table (defer view). + // Absent outside forks, for non-ducklake assets, and for assets never materialized + // anywhere. Lockstep with TS `AssetGraphAssetNode.fork_materialization`. + #[serde(skip_serializing_if = "Option::is_none")] + fork_materialization: Option, } #[derive(Serialize, Debug)] @@ -805,6 +811,7 @@ async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, Extension(user_db): Extension, + Extension(db): Extension, Query(q): Query, ) -> JsonResult { let mut tx = user_db.begin(&authed).await?; @@ -1250,9 +1257,89 @@ async fn asset_graph( runnable_set.insert((AssetUsageKind::Script, e.consumer_path.clone())); } + // Fork data-environment state per ducklake asset. The parent's rows are read on the plain + // pool: fork membership does not imply parent membership, and defer already exposes the + // parent's data to fork jobs — surfacing its materialization status is strictly less. + let fork_materialization_by_path: std::collections::HashMap = { + // The WHOLE ancestor chain, matching defer discovery: a grandchild fork whose direct + // parent only defers a table still reads it (from the grandparent), so it must show + // as deferred, not unmarked. + let ancestors = windmill_common::workspaces::fork_ancestor_chain(&db, &w_id).await?; + match ancestors { + // An orphaned `wm-fork-*` (parent deleted, chain empty) is still isolated — its + // own materializations must keep their 'fork' chips (nothing can be 'deferred'). + a if a.is_empty() && !w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX) => { + std::collections::HashMap::new() + } + ancestors => { + // Lakes the fork chose to SHARE at creation have no fork namespace — their + // assets live in the parent's tables for real, so no chip applies. + let shared_lakes: std::collections::HashSet = sqlx::query_scalar!( + "SELECT ducklake->'ducklakes' FROM workspace_settings WHERE workspace_id = $1", + &w_id, + ) + .fetch_optional(&db) + .await? + .flatten() + .and_then(|v| v.as_object().cloned()) + .map(|lakes| { + lakes + .into_iter() + .filter(|(_, cfg)| { + cfg.get("fork_behavior").and_then(|b| b.as_str()) == Some("shared") + }) + .map(|(name, _)| name) + .collect() + }) + .unwrap_or_default(); + sqlx::query!( + r#" + -- Fork rows also count when a snapshot ever committed (a failed run + -- preserves it): the physical table exists, so reads hit the FORK's + -- data — showing 'deferred' would misstate what a query returns. + -- Ancestor rows still require a clean materialization. + SELECT DISTINCT asset_path AS "asset_path!", workspace_id AS "workspace_id!" + FROM materialized_partition + WHERE (workspace_id = $1 OR workspace_id = ANY($2)) + AND asset_kind = 'ducklake' + AND (status = 'materialized' + OR (workspace_id = $1 AND snapshot_id IS NOT NULL)) + "#, + &w_id, + &ancestors, + ) + .fetch_all(&db) + .await? + .into_iter() + .filter(|r| { + !shared_lakes.contains(r.asset_path.split('/').next().unwrap_or_default()) + }) + .fold(std::collections::HashMap::new(), |mut m, r| { + // A fork row wins over an inherited 'deferred' from any ancestor's row. + if r.workspace_id == w_id { + m.insert(r.asset_path, "fork"); + } else { + m.entry(r.asset_path).or_insert("deferred"); + } + m + }) + } + } + }; + let mut assets: Vec = asset_set .into_iter() - .map(|(kind, path)| GraphAssetNode { kind, path }) + .map(|(kind, path)| GraphAssetNode { + fork_materialization: (kind == AssetKind::Ducklake) + .then(|| { + fork_materialization_by_path + .get(&path) + .map(|s| s.to_string()) + }) + .flatten(), + kind, + path, + }) .collect(); assets.sort_by(|a, b| a.path.cmp(&b.path)); diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index fc5a1fe874..c25bb8f42b 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -18,7 +18,7 @@ private = [ ] cloud = ["windmill-common/cloud"] no_auth = ["windmill-api-auth/no_auth"] -parquet = ["windmill-object-store/parquet"] +parquet = ["windmill-object-store/parquet", "dep:futures"] [dependencies] windmill-common = { workspace = true, default-features = false } @@ -35,6 +35,7 @@ windmill-store.workspace = true axum.workspace = true chrono.workspace = true +futures = { workspace = true, optional = true } hex.workspace = true magic-crypt.workspace = true http.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index aaf04972e1..228b8d1831 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -183,6 +183,10 @@ pub fn workspaced_service() -> Router { "/drop_forked_datatable_databases", post(crate::workspaces_extra::drop_forked_datatable_databases), ) + .route( + "/drop_forked_ducklake_namespaces", + post(crate::workspaces_extra::drop_forked_ducklake_namespaces), + ) .route( "/get_datatable_full_schema", post(get_datatable_full_schema), @@ -449,6 +453,11 @@ struct CreateWorkspaceFork { /// forked workspace's datatable config to point to the new database. #[serde(default)] forked_datatables: Vec, + /// Lakes the user explicitly chose to SHARE with the parent (the fork then reads and + /// writes the parent's lake directly). Every lake not listed gets the default isolated + /// fork namespace + read-defer. + #[serde(default)] + shared_ducklakes: Vec, /// Create the fork as a persistent dev workspace: the id is not required to carry the /// `wm-fork-` prefix, and at most one dev workspace may exist per parent. #[serde(default)] @@ -4063,6 +4072,7 @@ async fn clone_workspace_data( // Clone CI test references clone_ci_test_references(tx, source_workspace_id, target_workspace_id).await?; clone_macro_registry(tx, source_workspace_id, target_workspace_id).await?; + clone_asset_usages_and_triggers(tx, source_workspace_id, target_workspace_id).await?; // Clone flows with new versions clone_flows(tx, source_workspace_id, target_workspace_id).await?; @@ -4659,6 +4669,39 @@ async fn clone_macro_registry( Ok(()) } +// Asset usage rows and `// on` subscriber triggers are deploy-derived like +// ci_test_reference / the macro registry: without cloning them the fork's +// pipeline graph has no asset nodes or lineage edges, and — worse — the asset +// dispatch cascade never fires in the fork (it reads `script_trigger`), so +// materializing an upstream node can't trigger its consumers until every +// script is manually redeployed. `job`-kind usage rows (runtime-detected, +// ephemeral) are skipped like the graph does. +async fn clone_asset_usages_and_triggers( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns) + SELECT $2, path, kind, usage_access_type, usage_path, usage_kind, columns + FROM asset WHERE workspace_id = $1 AND usage_kind IN ('script', 'flow')", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s) + SELECT $2, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s + FROM script_trigger WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_flows( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, @@ -5380,6 +5423,42 @@ async fn create_workspace_fork( // conflict instead of a misleading "maximum number of workspaces" error. check_fork_w_id_conflict(&db, &nw.id).await?; purge_stale_fork_diff_state(&db, &nw.id).await?; + // A previously deleted fork with this id may have left ducklake namespaces behind if its + // physical cleanup failed after the delete committed (registry rows are the durable + // ledger — no FK, they outlive the workspace). Retry that cleanup now, and refuse to + // proceed while any metadata schema still can't be dropped: creating the fork anyway + // would silently reattach the deterministic namespace and its stale tables. Data-file + // leftovers alone don't block — once the schema is gone they are inert (a deleted fork's + // `$res:` storage resource is gone forever, so they may never resolve again), and the + // surviving registry row has the next successful same-prefix cleanup sweep them. + // `$res:` fallback = the workspace being forked: the deleted fork's resource clones came + // from a parent, so the new parent is the natural donor for the retry's credentials. + let leftover_issues = crate::workspaces_extra::drop_forked_ducklake_namespaces_impl( + &db, + &nw.id, + Some(&parent_workspace_id), + ) + .await?; + let blocking: Vec<&str> = leftover_issues + .iter() + .filter(|i| i.blocking) + .map(|i| i.msg.as_str()) + .collect(); + if !blocking.is_empty() { + return Err(Error::BadRequest(format!( + "a previously deleted workspace with id '{}' left ducklake namespaces that could \ + not be cleaned up: {}; retry once the catalog is reachable again", + nw.id, + blocking.join("; ") + ))); + } + for i in &leftover_issues { + tracing::warn!( + "creating fork {}: leftover ducklake cleanup: {}", + nw.id, + i.msg + ); + } #[cfg(not(feature = "enterprise"))] _check_nb_of_workspaces(&db).await?; @@ -5477,6 +5556,42 @@ async fn create_workspace_fork( apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; } + // The settings clone copies the source's ducklake config verbatim — including a parent + // fork's own `fork_behavior` stamps. Sharing is a per-fork-creation choice, never + // inherited: reset any cloned stamps first, then apply this fork's requested list. + sqlx::query!( + r#"UPDATE workspace_settings + SET ducklake = jsonb_set(ducklake, '{ducklakes}', ( + SELECT COALESCE(jsonb_object_agg(key, value - 'fork_behavior'), '{}'::jsonb) + FROM jsonb_each(ducklake->'ducklakes') + )) + WHERE workspace_id = $1 AND jsonb_typeof(ducklake->'ducklakes') = 'object'"#, + &forked_id, + ) + .execute(&mut *tx) + .await?; + + // Stamp the per-lake ducklake fork choice into the fork's own settings. Only the `shared` + // opt-out needs stamping — absent `fork_behavior` already means isolated (the default), + // so unlisted lakes and API callers that omit the field stay safe. + for lake in &nw.shared_ducklakes { + let stamped = sqlx::query_scalar!( + r#"UPDATE workspace_settings + SET ducklake = jsonb_set(ducklake, ARRAY['ducklakes', $2, 'fork_behavior'], '"shared"') + WHERE workspace_id = $1 AND ducklake->'ducklakes' ? $2 + RETURNING 1 AS "one!""#, + &forked_id, + lake, + ) + .fetch_optional(&mut *tx) + .await?; + if stamped.is_none() { + return Err(Error::BadRequest(format!( + "cannot mark ducklake `{lake}` as shared: no such lake in the workspace settings" + ))); + } + } + // Lock the parent ("prod") so edits are funneled through this dev workspace. let locked_prod = nw.is_dev_workspace && (nw.lock_prod_deploy || nw.lock_prod_forking); if locked_prod { @@ -5501,6 +5616,10 @@ async fn create_workspace_fork( .await?; tx.commit().await?; + // A pre-creation lookup could have cached an EMPTY ancestor chain for this id, which + // would bypass ducklake fork isolation for the TTL. + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&forked_id); + if locked_prod { windmill_common::workspaces::invalidate_protection_rules_cache(&parent_workspace_id); } @@ -5674,6 +5793,13 @@ async fn attach_dev_workspace( // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping // so per-workspace job tags route to the prod family immediately rather than after the TTL. windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + // Drop the cached ancestor chains too — the workspace existed BEFORE the attach, so a + // cached empty chain reads as "not a fork" and its ducklake jobs would write the shared + // lake until the TTL. Descendants' chains also gained the new root. + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } // Same reparent invalidates the billing-workspace mapping so its usage meters to prod at once. The // candidate can bring its own fork subtree, whose descendants had resolved their (now-stale) root // to the candidate's old family; invalidate them too so they meter to prod without waiting out the diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index daf83171e4..9fbaad2bf3 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -749,6 +749,16 @@ pub(crate) async fn change_workspace_id( // (fork-of-fork) self-heal via the 60s billing-cache TTL. for child in &reparented_children { windmill_queue::tags::invalidate_fork_parent_cache(child); + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(child); + // Grandchildren's cached chains contain the old (renamed-away) ancestor id; a stale + // chain drops all defer ancestors in the ducklake resolver, so sweep the subtree + // rather than letting it wait out the TTL. + for id in windmill_common::workspaces::list_fork_descendants(&db, child) + .await + .unwrap_or_default() + { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } #[cfg(feature = "cloud")] windmill_common::workspaces::invalidate_billing_workspace_cache(child); } @@ -829,28 +839,19 @@ pub(crate) async fn delete_workspace( // Deleting an attached dev workspace removes the parent prod's dev_workspace_lock (below), so it // must be a prod-admin action, not just the dev's own owner (dev ownership can diverge from // prod's) — mirrors detach_dev_workspace, which is prod-admin gated. - if let Some(prod) = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", - &w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten() - { - let is_prod_admin = sqlx::query_scalar!( - "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", - &prod, - &authed.email - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(false); - if !is_prod_admin && !is_super_admin_email(&db, &authed.email).await? { - return Err(Error::PermissionDenied(format!( - "Deleting dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" - ))); - } - } + require_prod_admin_for_dev_workspace(&db, &authed, &w_id).await?; + + // Snapshot the fork's ducklake namespaces + RESOLVE their connection material NOW — the + // registry rows and the fork's `$res:` resources both CASCADE with the workspace row — + // but the destructive cleanup itself runs only after the commit below: a delete that + // fails mid-way must never leave a live workspace with its fork data destroyed and no + // registry row to retry from. Read-only: nothing is dropped here. + let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None) + .await + .unwrap_or_else(|e| { + tracing::warn!("deleting workspace {w_id}: preparing ducklake cleanup: {e:#}"); + vec![] + }); sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id) .execute(&mut *tx) @@ -1086,6 +1087,17 @@ pub(crate) async fn delete_workspace( .await?; tx.commit().await?; + // Physical ducklake-namespace cleanup, post-commit, from the pre-read snapshot: fork + // namespaces are deterministic from (id, lake), so an orphan would silently REATTACH to a + // recreated identical fork id. Runs inline so every delete path is covered (CLI, + // force-delete dialog, direct API — not just the sidebar flow, which still calls the + // endpoint first for per-lake error toasts; the rerun is an idempotent no-op). Best + // effort: failures are logged — the workspace row is already gone, and broken storage + // credentials must not have made it undeletable. + for e in cleanup_fork_ducklake_namespaces(&db, &w_id, fork_ducklake_cleanups).await { + tracing::warn!("deleted workspace {w_id}: ducklake namespace cleanup: {}", e.msg); + } + if let Some(parent) = dev_lock_parent { windmill_common::workspaces::invalidate_protection_rules_cache(&parent); } @@ -1099,12 +1111,26 @@ pub(crate) async fn delete_workspace( // billing TTL. for id in std::iter::once(&w_id).chain(orphaned_children.iter()) { windmill_queue::tags::invalidate_fork_parent_cache(id); + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(id); #[cfg(feature = "cloud")] { windmill_common::workspaces::invalidate_billing_workspace_cache(id); windmill_common::workspaces::invalidate_team_plan_cache(id); } } + // Deeper descendants' cached ancestor CHAINS still contain the deleted workspace; unlike + // the sibling caches (which self-heal harmlessly via TTL), a stale chain makes the + // ducklake resolver drop all defer ancestors (all-or-nothing on broken links) — a visible + // defer/chips outage for up to the TTL. Anchor at the orphaned children: the deleted row + // is gone, but their subtrees are intact. + for child in orphaned_children.iter() { + for id in windmill_common::workspaces::list_fork_descendants(&db, child) + .await + .unwrap_or_default() + { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } + } Ok(format!("Deleted workspace {}", &w_id)) } @@ -1266,6 +1292,538 @@ pub async fn drop_forked_datatable_databases( Ok(Json(errors)) } +/// Drop this fork workspace's ducklake namespaces: the `wm_fork_*` metadata schema in each +/// lake's catalog database, plus (best effort) the fork's `__wm_forks//…` data files in +/// the workspace storage. Driven by the `fork_ducklake_namespace` registry written at first +/// fork attach, so it works even after settings drift. Returns errors per lake that failed; +/// the registry row is only deleted once both cleanups succeeded, so a retry resumes. +/// Same permission as delete_workspace: fork owner or super admin. `delete_workspace` also +/// runs this inline (the UI calls this endpoint first for error visibility; the inline run +/// covers every other delete path — CLI, force-delete dialogs, direct API — whose row delete +/// would otherwise CASCADE the registry away while orphaning the physical namespaces, which a +/// recreated identical fork id would then silently reattach). +pub async fn drop_forked_ducklake_namespaces( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result>> { + let is_fork = workspace_is_fork(&db, &w_id).await?; + let mut tx = db.begin().await?; + if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?) + && !is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied( + "Dropping forked ducklake namespaces requires being the fork's owner or a superadmin" + .to_string(), + )); + } + tx.commit().await?; + require_prod_admin_for_dev_workspace(&db, &authed, &w_id).await?; + Ok(Json( + drop_forked_ducklake_namespaces_impl(&db, &w_id, None) + .await? + .into_iter() + .map(|i| i.msg) + .collect(), + )) +} + +/// The cleanup itself, shared by the endpoint and the post-commit `delete_workspace` run. No +/// authorization of its own — callers gate. +pub(crate) async fn drop_forked_ducklake_namespaces_impl( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, +) -> Result> { + let prepared = prepare_fork_ducklake_cleanups(db, w_id, res_fallback_w_id).await?; + Ok(cleanup_fork_ducklake_namespaces(db, w_id, prepared).await) +} + +struct ForkDucklakeNamespaceRow { + ducklake_name: String, + metadata_schema: String, + catalog: String, + storage: String, + storage_ref: String, + schema_dropped: bool, + data_path: String, +} + +/// A namespace row plus its RESOLVED connection material. `delete_workspace` prepares these +/// BEFORE its transaction commits — the registry rows AND the fork's `$res:` resources both +/// disappear with the workspace row — and runs the destructive cleanup only AFTER the commit: +/// a delete that fails mid-way must never leave a live workspace with its fork data +/// destroyed, and a committed delete must still be able to reach catalogs/storages whose +/// credentials lived in the (now gone) fork resources. Per-row resolution failures are +/// carried as strings so they surface with the other cleanup errors. +pub(crate) struct PreparedForkDucklakeCleanup { + ns: ForkDucklakeNamespaceRow, + catalog_pg: std::result::Result, + store: std::result::Result<(windmill_types::s3::LargeFileStorage, serde_json::Value), String>, +} + +/// One row per (lake, catalog, storage, data path) ever attached — settings drift adds rows, +/// so every location the fork wrote gets cleaned, not just the first. Read-only: resolves +/// credentials but destroys nothing. +/// `res_fallback_w_id`: second workspace to resolve `$res:` catalog/storage paths against +/// when the row's own workspace no longer has them — the retry path runs AFTER the fork and +/// its resources were deleted. Fork resources are clones of a parent's, so the workspace +/// being forked again is the natural donor. None on live-workspace paths. +pub(crate) async fn prepare_fork_ducklake_cleanups( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, +) -> Result> { + let rows = sqlx::query_as!( + ForkDucklakeNamespaceRow, + r#"SELECT ducklake_name AS "ducklake_name!", metadata_schema AS "metadata_schema!", + catalog AS "catalog!", storage AS "storage!", + storage_ref AS "storage_ref!", data_path AS "data_path!", + schema_dropped AS "schema_dropped!" + FROM fork_ducklake_namespace WHERE workspace_id = $1"#, + w_id + ) + .fetch_all(db) + .await?; + let mut prepared = Vec::with_capacity(rows.len()); + for ns in rows { + let catalog_pg = if ns.schema_dropped { + // Schema phase already done — the retry needs no catalog connection at all + // (its credentials may be unresolvable for good with the fork's resources gone). + Err("unused: metadata schema already dropped".to_string()) + } else { + resolve_fork_catalog_pg(db, w_id, res_fallback_w_id, &ns.ducklake_name, &ns.catalog) + .await + .map_err(|e| e.to_string()) + }; + let storage = Some(ns.storage.as_str()).filter(|s| !s.is_empty()); + let store = resolve_fork_storage(db, w_id, res_fallback_w_id, storage, &ns.storage_ref) + .await + .map_err(|e| e.to_string()); + prepared.push(PreparedForkDucklakeCleanup { ns, catalog_pg, store }); + } + Ok(prepared) +} + +/// Drop the prepared namespaces' metadata schemas + data files, deleting each registry row +/// only after both succeed (a no-op when the row already cascaded away with the workspace). +/// Returns per-namespace error strings; never fails as a whole. +/// One failed step of a fork ducklake cleanup. `blocking` = the metadata schema (or its +/// guards) failed, so the namespace is still attachable and a same-id fork must NOT be +/// created. Non-blocking = the schema is gone and only data files (or the registry-row +/// delete) failed — inert storage leftovers, tracked by the surviving row and swept by the +/// next successful cleanup of the same prefix. +pub(crate) struct ForkDucklakeCleanupIssue { + pub(crate) blocking: bool, + pub(crate) msg: String, +} + +pub(crate) async fn cleanup_fork_ducklake_namespaces( + db: &DB, + w_id: &str, + prepared: Vec, +) -> Vec { + // The registration once-cache must not outlive the rows it mirrors: a same-id fork + // recreated within the TTL would otherwise skip re-registration, and ITS eventual + // deletion would find no rows — orphaning the deterministic namespace for the next + // same-id fork to silently reattach. + windmill_common::workspaces::invalidate_fork_ducklake_registration_cache(w_id); + let mut errors: Vec = Vec::new(); + for PreparedForkDucklakeCleanup { ns, catalog_pg, store } in prepared { + // Hard guards mirroring the forked-datatable drop: never touch a schema outside the + // fork prefix, never delete outside the fork data dir — even if a registry row was + // somehow tampered with. + if !ns + .metadata_schema + .starts_with(windmill_common::workspaces::FORK_DUCKLAKE_SCHEMA_PREFIX) + { + errors.push(ForkDucklakeCleanupIssue { + blocking: true, + msg: format!( + "Refusing to drop schema '{}' for ducklake://{}: name does not start with '{}'", + ns.metadata_schema, + ns.ducklake_name, + windmill_common::workspaces::FORK_DUCKLAKE_SCHEMA_PREFIX + ), + }); + continue; + } + // The fork's directory segment, NOT the raw workspace id: ids are only + // git-branch-safe and may contain `/`, which raw would let one fork's prefix nest + // inside a sibling's (`wm-fork-a/b` under `wm-fork-a`) and be swept by its cleanup. + let expected_prefix = format!( + "{}/{}/", + windmill_common::workspaces::FORK_DUCKLAKE_DATA_DIR, + windmill_common::workspaces::fork_data_dir_segment(w_id) + ); + if !format!("{}/", ns.data_path.trim_end_matches('/')).starts_with(&expected_prefix) { + errors.push(ForkDucklakeCleanupIssue { + blocking: true, + msg: format!( + "Refusing to delete data path '{}' for ducklake://{}: not under '{}'", + ns.data_path, ns.ducklake_name, expected_prefix + ), + }); + continue; + } + + let drop_res = if ns.schema_dropped { + // Recorded as already dropped by a prior partial cleanup; skipping means no + // catalog credentials are needed. Registration resets the flag when a live fork + // re-attaches (recreating the schema). + Ok(()) + } else { + match catalog_pg { + Ok(pg) => drop_fork_ducklake_metadata_schema(db, pg, &ns.metadata_schema).await, + Err(e) => Err(Error::internal_err(e)), + } + }; + if let Err(e) = drop_res { + errors.push(ForkDucklakeCleanupIssue { + blocking: true, + msg: format!( + "Could not drop metadata schema '{}' for ducklake://{}: {e}", + ns.metadata_schema, ns.ducklake_name + ), + }); + continue; + } + let delete_res = match store { + Ok((lfs, resource_value)) => { + delete_fork_ducklake_data(lfs, resource_value, &ns.data_path).await + } + Err(e) => Err(Error::internal_err(e)), + }; + if let Err(e) = delete_res { + // Record the completed schema phase so retries never need catalog credentials + // again (best effort — a failed update just means the next retry re-drops an + // absent schema, which requires the catalog to be reachable). + sqlx::query!( + "UPDATE fork_ducklake_namespace SET schema_dropped = true + WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3 + AND storage = $4 AND storage_ref = $5 AND data_path = $6", + w_id, + &ns.ducklake_name, + &ns.catalog, + &ns.storage, + &ns.storage_ref, + &ns.data_path, + ) + .execute(db) + .await + .ok(); + errors.push(ForkDucklakeCleanupIssue { + blocking: false, + msg: format!( + "Dropped metadata schema but could not delete data files under '{}' for ducklake://{}: {e}", + ns.data_path, ns.ducklake_name + ), + }); + continue; + } + sqlx::query!( + "DELETE FROM fork_ducklake_namespace + WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3 + AND storage = $4 AND storage_ref = $5 AND data_path = $6", + w_id, + &ns.ducklake_name, + &ns.catalog, + &ns.storage, + &ns.storage_ref, + &ns.data_path, + ) + .execute(db) + .await + .map_err(|e| { + errors.push(ForkDucklakeCleanupIssue { + blocking: false, + msg: format!( + "Cleaned ducklake://{} but could not delete its registry row: {e}", + ns.ducklake_name + ), + }) + }) + .ok(); + } + errors +} + +/// Drop the fork's metadata schema in the catalog database recorded by the registry row — +/// NOT whatever the fork's settings point at by now: a drifted catalog resource must not make +/// cleanup drop a schema in the wrong database while orphaning the real one. The pg schema +/// holds only DuckLake metadata tables (auto-created at first fork attach), so a plain +/// `DROP SCHEMA … CASCADE` on the catalog connection removes the whole fork namespace. +/// Resolve the catalog connection recorded in the registry row (read-only): instance +/// identities rebuild instance creds; resource identities resolve their `$res:` in the fork's +/// workspace — which is why this must run BEFORE the workspace (and its resources) are +/// deleted. Mysql never registers (rejected at resolution). +async fn resolve_fork_catalog_pg( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, + ducklake_name: &str, + catalog: &str, +) -> Result { + let (resource_type, resource_path) = catalog.split_once(':').ok_or_else(|| { + Error::internal_err(format!( + "ducklake://{ducklake_name}: malformed registry catalog identity `{catalog}`" + )) + })?; + let catalog_resource = if resource_type == "instance" { + let mut pg_creds = windmill_common::PgDatabase::parse_uri( + &windmill_common::get_database_url().await?.as_str().await, + )?; + pg_creds.dbname = resource_path.to_string(); + pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.password = + Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); + serde_json::to_value(&pg_creds) + .map_err(|e| Error::internal_err(format!("serializing pg creds: {e}")))? + } else { + resolve_res_with_fallback(db, w_id, res_fallback_w_id, resource_path).await? + }; + serde_json::from_value(catalog_resource).map_err(|e| { + Error::internal_err(format!( + "ducklake://{ducklake_name}: catalog resource is not a postgres database: {e}" + )) + }) +} + +async fn drop_fork_ducklake_metadata_schema( + db: &DB, + pg: windmill_common::PgDatabase, + metadata_schema: &str, +) -> Result<()> { + let (client, connection) = pg.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let res = client + .execute( + &format!( + "DROP SCHEMA IF EXISTS \"{}\" CASCADE", + metadata_schema.replace('"', "\"\"") + ), + &[], + ) + .await; + drop(client); + let _ = join_handle.await; + res.map_err(|e| Error::internal_err(format!("{e:#}")))?; + Ok(()) +} + +/// Resolve the storage identified by the registry's `storage_ref` (read-only) — the storage +/// that was active when the fork's data was WRITTEN, not whatever the logical storage name +/// points at by deletion time (a repointed storage must not orphan the original fork data, +/// nor get a colliding prefix deleted). `storage_ref` = '' falls back to resolving the +/// logical name against current settings (registration couldn't identify the storage — best +/// effort). Resolves the `$res:` in the fork's workspace, which is why this must run BEFORE +/// the workspace is deleted. +async fn resolve_fork_storage( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, + storage: Option<&str>, + storage_ref: &str, +) -> Result<(windmill_types::s3::LargeFileStorage, serde_json::Value)> { + use windmill_types::s3::{ + AzureBlobStorage, FilesystemStorage, GoogleCloudStorage, LargeFileStorage, S3Storage, + }; + + let lfs: LargeFileStorage = if let Some((typ, path)) = storage_ref.split_once(':') { + // Rebuild the LFS entry from the registered identity; only the variant (which + // resource parser applies) and the path matter to `lfs_to_object_store_resource`. + let s3 = |p: &str| S3Storage { + s3_resource_path: p.to_string(), + public_resource: None, + advanced_permissions: None, + }; + match typ { + "S3Storage" => LargeFileStorage::S3Storage(s3(path)), + "S3AwsOidc" => LargeFileStorage::S3AwsOidc(s3(path)), + "AzureBlobStorage" => LargeFileStorage::AzureBlobStorage(AzureBlobStorage { + azure_blob_resource_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + "AzureWorkloadIdentity" => LargeFileStorage::AzureWorkloadIdentity(AzureBlobStorage { + azure_blob_resource_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + "GoogleCloudStorage" => LargeFileStorage::GoogleCloudStorage(GoogleCloudStorage { + gcs_resource_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + "FilesystemStorage" => LargeFileStorage::FilesystemStorage(FilesystemStorage { + root_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + other => { + return Err(Error::internal_err(format!( + "unknown registered storage type `{other}`" + ))) + } + } + } else { + let lfs_json = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() + .ok_or_else(|| Error::BadRequest("workspace has no storage configured".to_string()))?; + // Named storages live under `secondary_storage`; `None`/`_default_` is the primary. + match storage.filter(|s| *s != "_default_") { + None => serde_json::from_value(lfs_json.clone()) + .map_err(|e| Error::internal_err(format!("parsing large_file_storage: {e}")))?, + Some(name) => serde_json::from_value( + lfs_json + .get("secondary_storage") + .and_then(|s| s.get(name)) + .cloned() + .ok_or_else(|| { + Error::BadRequest(format!("workspace has no storage named {name}")) + })?, + ) + .map_err(|e| Error::internal_err(format!("parsing storage {name}: {e}")))?, + } + }; + // Filesystem storage stores a direct path (`lfs_to_object_store_resource` ignores the + // resource value); everything else references a resource whose stored path may or may not + // carry the `$res:` prefix — same normalization as `get_workspace_s3_resource_from_lfs`. + let resource_value = if matches!(lfs, LargeFileStorage::FilesystemStorage(_)) { + serde_json::Value::Null + } else { + let path = lfs.get_s3_resource_path(); + let path = path.strip_prefix("$res:").unwrap_or(path); + resolve_res_with_fallback(db, w_id, res_fallback_w_id, path).await? + }; + Ok((lfs, resource_value)) +} + +/// Resolve a `$res:` path in `w_id`, falling back to the same path in `res_fallback_w_id` +/// when the first lookup fails — retry-path cleanups run after the fork workspace (and its +/// cloned resource rows) were deleted, and the fork's resources were clones of a parent's. +async fn resolve_res_with_fallback( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, + resource_path: &str, +) -> Result { + let res = windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{resource_path}")), + w_id, + db, + ) + .await; + match (res, res_fallback_w_id) { + (Ok(v), _) => Ok(v), + (Err(e), None) => Err(e), + (Err(_), Some(fb)) => { + windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{resource_path}")), + fb, + db, + ) + .await + } + } +} + +/// Delete every object under the fork's data prefix in the pre-resolved storage. Requires the +/// `parquet` (object store) feature; without it the metadata schema is still dropped and the +/// unreachable data files are left for manual cleanup. +#[cfg(feature = "parquet")] +async fn delete_fork_ducklake_data( + lfs: windmill_types::s3::LargeFileStorage, + resource_value: serde_json::Value, + data_path: &str, +) -> Result<()> { + use futures::{StreamExt, TryStreamExt}; + + let store = windmill_object_store::build_object_store_client( + &windmill_object_store::lfs_to_object_store_resource(&lfs, resource_value)?, + ) + .await?; + + let prefix = windmill_object_store::object_store_reexports::Path::from( + data_path.trim_matches('/').to_string(), + ); + let locations: Vec<_> = store + .list(Some(&prefix)) + .map_ok(|m| m.location) + .try_collect() + .await + .map_err(windmill_object_store::object_store_error_to_error)?; + // The object_store crate evaluates list prefixes on a path-SEGMENT basis (`a/b` does not + // match `a/bc/…`), so sibling fork segments sharing a string prefix are already excluded. + // Filter anyway — deletion must not depend on a listing implementation detail. + let boundary = format!("{}/", prefix.as_ref()); + let locations: Vec<_> = locations + .into_iter() + .filter(|l| l.as_ref().starts_with(&boundary)) + .collect(); + // 1000-object chunks: S3 DeleteObjects caps a batch at 1000 keys. + for chunk in locations.chunks(1000) { + store + .delete_stream(futures::stream::iter(chunk.iter().cloned().map(Ok)).boxed()) + .try_collect::>() + .await + .map_err(windmill_object_store::object_store_error_to_error)?; + } + Ok(()) +} + +#[cfg(not(feature = "parquet"))] +async fn delete_fork_ducklake_data( + _lfs: windmill_types::s3::LargeFileStorage, + _resource_value: serde_json::Value, + _data_path: &str, +) -> Result<()> { + Err(Error::internal_err( + "object storage support (parquet feature) is not compiled in".to_string(), + )) +} + +/// Destroying an ATTACHED dev workspace (or its data environments) must be a prod-admin +/// action, not just the dev's own owner (dev ownership can diverge from prod's) — mirrors +/// detach_dev_workspace. Shared by `delete_workspace` and `drop_forked_ducklake_namespaces` +/// so the two gates cannot drift: the sidebar calls the drop endpoint BEFORE deleteWorkspace, +/// and a weaker gate on the drop would let a non-prod-admin dev owner destroy the live dev's +/// materializations and then have the deletion itself rejected. No-op for non-dev workspaces. +async fn require_prod_admin_for_dev_workspace( + db: &DB, + authed: &ApiAuthed, + w_id: &str, +) -> Result<()> { + if let Some(prod) = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + w_id + ) + .fetch_optional(db) + .await? + .flatten() + { + let is_prod_admin = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + &prod, + &authed.email + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if !is_prod_admin && !is_super_admin_email(db, &authed.email).await? { + return Err(Error::PermissionDenied(format!( + "Destroying dev workspace '{w_id}' or its data requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" + ))); + } + } + Ok(()) +} + async fn is_workspace_owner( authed: &ApiAuthed, w_id: &str, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fe7edb26b0..0c080b0a2a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4666,6 +4666,24 @@ paths: items: type: string + /w/{workspace}/workspaces/drop_forked_ducklake_namespaces: + post: + summary: drop this fork workspace's ducklake namespaces (catalog metadata schemas + data files) + operationId: dropForkedDucklakeNamespaces + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of errors (empty if all succeeded) + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/workspaces/import_pg_database: post: summary: import a PostgreSQL database from source to target via pg_dump @@ -21464,6 +21482,10 @@ paths: $ref: "#/components/schemas/AssetKind" path: type: string + fork_materialization: + type: string + enum: [fork, deferred] + description: Fork workspaces only — 'fork' when this ducklake asset was materialized in the fork itself, 'deferred' when reads fall back to the parent workspace's current table via a defer view. Omitted otherwise. runnables: type: array items: @@ -28178,6 +28200,11 @@ components: new_dbname: type: string description: "New database name for the fork" + shared_ducklakes: + type: array + items: + type: string + description: "Lake names the fork SHARES with the parent (reads and writes the parent's lake directly). Every lake not listed gets the default isolated fork namespace with read-defer to the parent." is_dev_workspace: type: boolean description: "Create the fork as a persistent dev workspace (id not required to carry the wm-fork- prefix; at most one per parent)" @@ -28917,6 +28944,12 @@ components: - path extra_args: type: string + fork_behavior: + type: string + enum: [isolated, shared] + description: + Fork workspaces only - how this lake behaves in the fork, stamped at fork + creation. Absent = isolated (fork-scoped namespace + read-defer to parent). maintenance: type: object description: diff --git a/backend/windmill-common/src/materialization.rs b/backend/windmill-common/src/materialization.rs index 8d9e31899f..236a71c855 100644 --- a/backend/windmill-common/src/materialization.rs +++ b/backend/windmill-common/src/materialization.rs @@ -94,7 +94,11 @@ pub async fn record_materialization<'e>( VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9) ON CONFLICT (workspace_id, asset_kind, asset_path, partition) DO UPDATE SET status = EXCLUDED.status, - snapshot_id = EXCLUDED.snapshot_id, + -- A failed run records no snapshot, but must not erase the last + -- committed one: a physical table from an earlier commit (or from a + -- committed write whose data tests then failed) still exists, and + -- fork defer/graph state keys on that evidence. + snapshot_id = COALESCE(EXCLUDED.snapshot_id, materialized_partition.snapshot_id), row_count = EXCLUDED.row_count, job_id = EXCLUDED.job_id, materialized_at = now(), @@ -255,6 +259,98 @@ pub async fn record_asset_schema( Ok(true) } +/// One table a fork workspace should read from an ancestor through a defer view. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ForkDeferTable { + /// Lake-internal table name: the `asset_path` minus its `/` prefix (may itself be + /// `schema.table`). + pub table: String, + /// The owning ancestor's latest captured schema carries the SCD2 marker column + /// (`is_current`), so that lake also holds a managed `
_current` companion view + /// that consumers read — defer it alongside the table. + #[serde(default)] + pub with_current_view: bool, + /// Index into `DucklakeForkDefer.ancestors` (nearest-first) of the NEAREST ancestor that + /// materialized this table — the defer view must target that ancestor's namespace. In a + /// `fork → parent → root` chain where only root materialized a table, the parent has no + /// physical copy (it defers too), so a view over the parent would not bind. Defaults to 0 + /// (the direct parent) for wire compatibility with agents that predate the field. + #[serde(default)] + pub ancestor_idx: u32, +} + +/// Tables of lake `lake_name` materialized somewhere in the fork's ancestor chain +/// (nearest-first) but not (yet) in the fork — the fork's read-defer set, each mapped to the +/// nearest ancestor that owns a physical copy. Only `Materialized` rows count on every side: a +/// deferred table must physically exist in the targeted ancestor (defer views bind at CREATE +/// and would otherwise fail the whole job), and any successful fork materialization makes the +/// fork's own table authoritative. +/// +/// **Authorization:** performs no access control; trusted server-side callers only. It reads +/// ancestor workspaces' rows on behalf of a fork — acceptable because defer itself exposes +/// the ancestors' table contents to fork members. +pub async fn list_fork_defer_tables<'e>( + executor: impl PgExecutor<'e>, + ancestor_workspace_ids: &[String], + fork_workspace_id: &str, + lake_name: &str, +) -> Result> { + let rows = sqlx::query!( + r#" + WITH ancestor AS ( + SELECT wid, ord FROM unnest($1::text[]) WITH ORDINALITY AS a(wid, ord) + ), anc_mat AS ( + -- Per table, the nearest ancestor (lowest ord) that materialized it. + SELECT DISTINCT ON (mp.asset_path) mp.asset_path, a.wid, a.ord + FROM materialized_partition mp + JOIN ancestor a ON a.wid = mp.workspace_id + WHERE mp.asset_kind = 'ducklake' AND mp.status = 'materialized' + AND split_part(mp.asset_path, '/', 1) = $3 AND mp.asset_path LIKE '%/%' + ORDER BY mp.asset_path, a.ord + ), fork_mat AS ( + -- Fork-OWNED assets: anything whose physical table exists in the fork + -- namespace, not just clean materializations. A committed write whose data + -- tests failed afterwards records status='failed' WITH a snapshot — its table + -- is real, and a defer view emitted over it would silently yield to it + -- (CREATE VIEW IF NOT EXISTS) while claiming the read defers to the parent. + SELECT DISTINCT asset_path FROM materialized_partition + WHERE workspace_id = $2 AND asset_kind = 'ducklake' + AND (status = 'materialized' OR snapshot_id IS NOT NULL) + ), latest_schema AS ( + SELECT DISTINCT ON (workspace_id, asset_path) workspace_id, asset_path, columns + FROM materialized_asset_schema + WHERE workspace_id = ANY($1) AND asset_kind = 'ducklake' + ORDER BY workspace_id, asset_path, version DESC + ) + SELECT am.asset_path AS "asset_path!", + am.ord AS "ord!", + COALESCE(EXISTS ( + SELECT 1 FROM jsonb_array_elements(ls.columns) e + WHERE e->>'name' = 'is_current' + ), false) AS "has_current!" + FROM anc_mat am + LEFT JOIN latest_schema ls + ON ls.asset_path = am.asset_path AND ls.workspace_id = am.wid + WHERE am.asset_path NOT IN (SELECT asset_path FROM fork_mat) + ORDER BY am.asset_path + "#, + ancestor_workspace_ids, + fork_workspace_id, + lake_name, + ) + .fetch_all(executor) + .await?; + Ok(rows + .into_iter() + .map(|r| ForkDeferTable { + table: r.asset_path[lake_name.len() + 1..].to_string(), + with_current_view: r.has_current, + // WITH ORDINALITY is 1-based; ancestors vec is 0-based. + ancestor_idx: (r.ord - 1).max(0) as u32, + }) + .collect()) +} + /// All captured schema versions for one asset, newest version first. /// /// **Authorization:** performs no access control (mirrors diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 293490e434..14bc5fc344 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -842,10 +842,27 @@ pub struct Ducklake { pub storage: DucklakeStorage, #[serde(skip_serializing_if = "Option::is_none")] pub extra_args: Option, + /// How this lake behaves when the workspace is a fork/dev workspace. Only meaningful in a + /// fork's own settings; stamped at fork creation from the user's per-lake choice. Absent = + /// `Isolated` — the safe default, so forks created before this field existed (and API + /// callers that omit it) never write the parent's lake. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fork_behavior: Option, #[serde(skip_serializing_if = "Option::is_none")] pub maintenance: Option, } +/// Per-lake fork data-environment choice, made at fork creation. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DucklakeForkBehavior { + /// Fork-scoped namespace + read-defer to the parent (default). + Isolated, + /// The fork reads AND WRITES the parent's lake directly — explicit opt-out of isolation + /// (e.g. a fork meant to run prod-equivalent backfills). + Shared, +} + /// Scheduled maintenance for a ducklake (enterprise): snapshot expiry, /// adjacent-file compaction and orphaned-file cleanup, run as a managed /// per-lake schedule. Not mirrored in `instance_config::Ducklake`: @@ -949,6 +966,213 @@ pub struct DucklakeWithConnData { pub storage: DucklakeStorage, #[serde(skip_serializing_if = "Option::is_none")] pub extra_args: Option, + /// Present when the resolved workspace is a fork/dev workspace. Carries the read-defer + /// context (ancestor namespaces + tables to expose as views over the direct parent). The + /// fork *write* redirect is folded into `storage.path`/`extra_args` above, so an agent + /// worker that predates this field still writes the fork namespace (it only misses the + /// defer views). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fork_defer: Option, +} + +/// Read-defer context for a fork workspace's ducklake: which ancestor namespaces to attach +/// read-only, and which tables to expose as views over the direct parent because the fork has +/// not materialized them yet. `ancestors` is empty when defer is unavailable (an ancestor no +/// longer defines the lake) — the fork namespace still isolates writes in that case. +#[derive(Deserialize, Serialize)] +pub struct DucklakeForkDefer { + /// Nearest-first (direct parent … root), each resolved from that workspace's own settings. + pub ancestors: Vec, + pub defer_tables: Vec, + /// Views currently live in the fork namespace (read from the catalog's `ducklake_view` at + /// resolution time, lake-internal names). The worker's view→table transition (DROP VIEW + /// before a managed materialize) keys on THIS, not on recorded materialization status: + /// after a failed run the status can't distinguish a defer view from a real table, and + /// `DROP VIEW` against a table (or `CREATE TABLE` against a view) errors — either guess + /// would wedge the asset until manual repair. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub fork_views: Vec, +} + +/// Connection data for one ancestor namespace of a fork's ducklake. +#[derive(Deserialize, Serialize)] +pub struct DucklakeAncestorAttach { + pub workspace_id: String, + /// DuckDB catalog alias this namespace must be attached under. Persisted defer-view SQL + /// references it, so it is a pure function of (lake name, ancestor workspace id) and every + /// session reading those views attaches the ancestor under this exact alias. + pub alias: String, + pub catalog: DucklakeCatalog, + pub catalog_resource: serde_json::Value, + pub storage: DucklakeStorage, + /// None = the lake's default metadata schema (the ancestor is a root/non-fork workspace). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata_schema: Option, + /// The ancestor config's own non-reserved ATTACH args (e.g. `ENCRYPTED true`), already + /// stripped of the fork-owned `METADATA_SCHEMA`/`DATA_PATH`/`OVERRIDE_DATA_PATH` — an + /// option-dependent lake would otherwise fail its read-only ancestor attach even though + /// the same lake attaches fine everywhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra_args: Option, +} + +/// Prefix of fork-scoped ducklake metadata schemas. Cleanup refuses to drop any pg schema not +/// carrying it, mirroring the `wm_fork_` guard on forked datatable databases. +pub const FORK_DUCKLAKE_SCHEMA_PREFIX: &str = "wm_fork_"; + +/// Bucket-root directory holding all fork namespaces' data files: each fork writes under +/// `{FORK_DUCKLAKE_DATA_DIR}//` where the segment is +/// [`fork_data_dir_segment`] (see [`fork_data_path`] for why it wraps the lake's path instead +/// of nesting under it). +pub const FORK_DUCKLAKE_DATA_DIR: &str = "__wm_forks"; + +fn mangle_identifier(s: &str, max: usize) -> String { + s.chars() + .take(max) + .map(|c| { + let c = c.to_ascii_lowercase(); + if c.is_ascii_alphanumeric() { + c + } else { + '_' + } + }) + .collect() +} + +/// Deterministic, injective pg-schema name for a fork workspace's namespace of ONE lake: +/// `wm_fork___<8-hex sha256>`, ≤58 chars (pg limit 63). +/// Lake-scoped, not just workspace-scoped: two lakes of one workspace may share a catalog +/// database, and a per-workspace schema would merge their namespaces in the fork (tables and +/// snapshots colliding across `ducklake://a/…` and `ducklake://b/…`). The hash keeps distinct +/// (workspace, lake) pairs distinct after mangling. The registry row records the computed name +/// for cleanup, but ATTACH recomputes it — so this function must stay stable across releases +/// or existing forks would silently lose their namespace. +pub fn fork_ducklake_metadata_schema(w_id: &str, lake_name: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = hex::encode(&Sha256::digest(format!("{w_id}\0{lake_name}").as_bytes())[..4]); + format!( + "{FORK_DUCKLAKE_SCHEMA_PREFIX}{}_{}_{hash}", + mangle_identifier(w_id, 24), + mangle_identifier(lake_name, 16) + ) +} + +/// The `METADATA_SCHEMA ''` value carried in a lake config's `extra_args`, if any — the +/// schema the lake's OWN catalog namespace lives in (how one catalog database hosts several +/// lakes). Ancestor read-only attaches must preserve it or they'd bind the wrong namespace. +pub fn extract_metadata_schema_arg(extra_args: &str) -> Option { + lazy_static::lazy_static! { + static ref MS: regex::Regex = regex::Regex::new( + r"(?i)\bMETADATA_SCHEMA\s*(?:'([^']*)'|([A-Za-z0-9_]+))" + ) + .unwrap(); + } + // Last occurrence wins, matching DuckDB's duplicate-option semantics. + MS.captures_iter(extra_args) + .last() + .and_then(|c| c.get(1).or_else(|| c.get(2))) + .map(|m| m.as_str().to_string()) +} + +/// Deterministic DuckDB attach alias for an ancestor namespace of a fork's lake. Persisted +/// defer-view SQL references it (same stability requirement as +/// [`fork_ducklake_metadata_schema`]). +pub fn fork_ducklake_ancestor_alias(lake_name: &str, ancestor_w_id: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = + hex::encode(&Sha256::digest(format!("{lake_name}\0{ancestor_w_id}").as_bytes())[..4]); + format!( + "__wm_dl_{}_{}_{hash}", + mangle_identifier(lake_name, 20), + mangle_identifier(ancestor_w_id, 20) + ) +} + +/// Strip `METADATA_SCHEMA` / `DATA_PATH` / `OVERRIDE_DATA_PATH` tokens from ducklake ATTACH +/// extra args. In a fork these options are injected by the fork resolution; a user- or +/// settings-supplied duplicate silently wins (DuckDB keeps the last occurrence) and would +/// escape the fork namespace back to the parent's, so they are removed rather than overridden. +pub fn strip_fork_reserved_attach_args(extra_args: &str) -> String { + lazy_static::lazy_static! { + static ref RESERVED: regex::Regex = regex::Regex::new( + r"(?i)\b(METADATA_SCHEMA|DATA_PATH|OVERRIDE_DATA_PATH)\s*('[^']*'|[A-Za-z0-9_]+)" + ) + .unwrap(); + } + RESERVED + .replace_all(extra_args, "") + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect::>() + .join(", ") +} + +lazy_static::lazy_static! { + /// fork workspace id -> (ancestor chain nearest-first, expiry ts). Empty chain = not a fork. + /// `parent_workspace_id` only changes on dev-workspace attach/detach, so a short TTL is safe. + static ref FORK_ANCESTOR_CHAIN_CACHE: Cache, i64)> = Cache::new(5000); + /// fork workspace id -> (locations recently upserted into `fork_ducklake_namespace`, + /// expiry ts), so the registry write doesn't run on every job of a fork. Keyed by + /// workspace id so cleanup can drop a fork's whole entry: a same-id fork recreated within + /// the TTL must re-register, or its materializations would carry no registry row and leak + /// at ITS deletion. TTL'd as a second line of defense for cleanup paths that bypass + /// `cleanup_fork_ducklake_namespaces` (e.g. manual registry edits). + static ref FORK_DUCKLAKE_REGISTERED: Cache, i64)> = + Cache::new(5000); +} + +/// Drop the "already registered" once-cache for a workspace. MUST be called whenever +/// `fork_ducklake_namespace` rows for that workspace are deleted (namespace cleanup, workspace +/// deletion): a surviving entry would make a same-id fork recreated within the TTL skip +/// re-registration, orphaning its namespace at deletion time. +pub fn invalidate_fork_ducklake_registration_cache(w_id: &str) { + FORK_DUCKLAKE_REGISTERED.remove(w_id); +} + +/// Drop the cached ancestor chain for a workspace. MUST be called wherever +/// `parent_workspace_id` lineage changes (fork creation, dev-workspace attach, reparenting +/// rename, deletion): a cached EMPTY chain reads as "not a fork" and would bypass ducklake +/// fork isolation for the TTL — the first jobs after a dev-workspace attach would write the +/// shared lake. +pub fn invalidate_fork_ancestor_chain_cache(w_id: &str) { + FORK_ANCESTOR_CHAIN_CACHE.remove(w_id); +} + +/// Ancestors of `w_id`, nearest-first (direct parent … root). Empty for a non-fork workspace or +/// an unknown id. The depth bound is a cycle-safety backstop, same convention as +/// [`fork_chain_depth`]. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_ancestor_chain(db: &crate::DB, w_id: &str) -> Result> { + let now = chrono::Utc::now().timestamp(); + if let Some((chain, expiry)) = FORK_ANCESTOR_CHAIN_CACHE.get(w_id) { + if expiry > now { + return Ok(chain); + } + } + let chain = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT id AS "id!" FROM chain WHERE depth > 0 ORDER BY depth + "#, + w_id + ) + .fetch_all(db) + .await + .map_err(|e| Error::internal_err(format!("resolving fork ancestors of {w_id}: {e:#}")))?; + FORK_ANCESTOR_CHAIN_CACHE.insert(w_id.to_string(), (chain.clone(), now + 60)); + Ok(chain) } pub async fn get_ducklake_from_db_unchecked( @@ -956,6 +1180,29 @@ pub async fn get_ducklake_from_db_unchecked( w_id: &str, db: &DB, ) -> Result { + let (base, fork_behavior) = ducklake_conn_data(name, w_id, db).await?; + let chain = fork_ancestor_chain(db, w_id).await?; + // `Shared` is the explicit fork-creation opt-out of isolation: the fork reads and writes + // the parent's lake through its own (cloned) config, exactly like a non-fork workspace. + // An empty chain alone does NOT mean "not a fork": `parent_workspace_id` is ON DELETE SET + // NULL, so a `wm-fork-*` workspace can outlive its parent — its cloned config still points + // at the shared lake, so the prefix keeps it isolated (mirrors `workspace_is_fork`). No + // ancestors ⇒ no defer, write redirect only. Prefix-less dev workspaces can't be orphaned + // (deletion of their prod is blocked while attached). + if (chain.is_empty() && !w_id.starts_with(WM_FORK_PREFIX)) + || fork_behavior == Some(DucklakeForkBehavior::Shared) + { + return Ok(base); + } + fork_scoped_ducklake(name, w_id, base, chain, db).await +} + +/// Resolve one workspace's own config for lake `name` — no fork awareness. +async fn ducklake_conn_data( + name: &str, + w_id: &str, + db: &DB, +) -> Result<(DucklakeWithConnData, Option)> { let ducklake = sqlx::query_scalar!( r#" SELECT ws.ducklake->'ducklakes'->$2 AS config @@ -988,13 +1235,447 @@ pub async fn get_ducklake_from_db_unchecked( ) .await? }; + let fork_behavior = ducklake.fork_behavior; let ducklake = DucklakeWithConnData { catalog_resource, catalog: ducklake.catalog, storage: ducklake.storage, extra_args: ducklake.extra_args, + fork_defer: None, }; - Ok(ducklake) + Ok((ducklake, fork_behavior)) +} + +/// The fork's directory segment under `__wm_forks/`: mangled + hashed into ONE path component +/// (same recipe and stability requirement as [`fork_ducklake_metadata_schema`]). Fork/dev +/// workspace ids are only git-branch-safe and may contain `/` (e.g. `wm-fork-a/b`) — used +/// raw, such an id would nest inside the sibling `wm-fork-a`'s prefix and be swept by ITS +/// cleanup. The hash keeps distinct ids distinct after mangling. +pub fn fork_data_dir_segment(fork_w_id: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = hex::encode(&Sha256::digest(fork_w_id.as_bytes())[..4]); + format!("{}_{hash}", mangle_identifier(fork_w_id, 40)) +} + +/// The fork namespace's data path within the same bucket: a bucket-root +/// `__wm_forks//` prefix wrapping the lake's own path, NOT a sub-path under it — +/// the parent lake's maintenance (snapshot expiry / `ducklake_delete_orphaned_files`) scans +/// everything under the parent's DATA_PATH and would treat live fork files nested there as +/// orphans and delete them. +pub fn fork_data_path(base_path: &str, fork_w_id: &str) -> String { + let segment = fork_data_dir_segment(fork_w_id); + let base = base_path.trim_matches('/'); + if base.is_empty() { + format!("{FORK_DUCKLAKE_DATA_DIR}/{segment}") + } else { + format!("{FORK_DUCKLAKE_DATA_DIR}/{segment}/{base}") + } +} + +/// Redirect a fork workspace's lake to its fork-scoped namespace (same catalog DB, fork +/// metadata schema + data sub-path) and assemble the read-defer context over its ancestors. +async fn fork_scoped_ducklake( + name: &str, + w_id: &str, + mut base: DucklakeWithConnData, + chain: Vec, + db: &DB, +) -> Result { + if base.catalog.resource_type == DucklakeCatalogResourceType::Mysql { + return Err(Error::BadRequest(format!( + "ducklake {name}: mysql-catalog lakes are not supported in fork workspaces — \ + running against the shared catalog would write the parent workspace's data" + ))); + } + let metadata_schema = fork_ducklake_metadata_schema(w_id, name); + let fork_path = fork_data_path(&base.storage.path, w_id); + let catalog_identity = ducklake_catalog_identity(&base.catalog); + register_fork_ducklake_namespace( + db, + w_id, + name, + &metadata_schema, + &catalog_identity, + base.storage.storage.as_deref(), + &fork_path, + ) + .await?; + + // Ancestor namespaces, nearest-first, each from its own settings so a fork-side settings + // edit can't silently repoint what "parent" means. All-or-nothing: a broken link anywhere + // in the chain disables defer entirely (a defer view over a missing ancestor attach fails + // at bind time and would kill unrelated jobs), but write isolation still applies. + let mut ancestors = Vec::with_capacity(chain.len()); + for (i, ancestor_id) in chain.iter().enumerate() { + match ducklake_conn_data(name, ancestor_id, db).await { + Ok((mut a, ancestor_behavior)) => { + // A fork ancestor lives in its own namespace UNLESS its lake is `Shared` — + // then it never redirected and its data sits at its config's default + // location, exactly like a root workspace. Chain position alone can't tell + // the two apart: an orphaned `wm-fork-*` ancestor (its own parent deleted, + // `parent_workspace_id` SET NULL) ends the chain like a root but its data + // lives in ITS fork namespace — key on the prefix too, as in resolution. + let is_isolated_fork = (i + 1 < chain.len() + || ancestor_id.starts_with(WM_FORK_PREFIX)) + && ancestor_behavior != Some(DucklakeForkBehavior::Shared); + let metadata_schema = if is_isolated_fork { + a.storage.path = fork_data_path(&a.storage.path, ancestor_id); + Some(fork_ducklake_metadata_schema(ancestor_id, name)) + } else { + // Root / shared ancestors live in their config's OWN catalog namespace — + // which may be a non-default schema when one catalog database hosts + // several lakes (`extra_args METADATA_SCHEMA '…'`). Preserve it, or the + // read-only attach would bind the wrong (or a nonexistent) lake. + a.extra_args + .as_deref() + .and_then(extract_metadata_schema_arg) + }; + let extra_args = a + .extra_args + .as_deref() + .map(strip_fork_reserved_attach_args) + .filter(|s| !s.is_empty()); + ancestors.push(DucklakeAncestorAttach { + workspace_id: ancestor_id.clone(), + alias: fork_ducklake_ancestor_alias(name, ancestor_id), + catalog: a.catalog, + catalog_resource: a.catalog_resource, + storage: a.storage, + metadata_schema, + extra_args, + }); + } + Err(e) => { + tracing::warn!( + "fork {w_id}: ducklake {name} not resolvable in ancestor {ancestor_id} \ + ({e:#}); read-defer disabled, fork namespace still isolated" + ); + ancestors.clear(); + break; + } + } + } + // Drop fork ancestors whose namespace was never bootstrapped (e.g. a fresh intermediate + // fork that never attached this lake): their READ_ONLY attach would fail the whole job + // ("DuckLake does not exist" + creation disabled), and a nonexistent namespace can't own + // tables or be referenced by any persisted view. Checked against the fork's catalog DB — + // must happen BEFORE defer discovery so `ancestor_idx` values index the filtered list. + let (existing_schemas, fork_views, fork_tables) = + inspect_fork_catalog(&base, &metadata_schema, &ancestors, db).await?; + ancestors.retain(|a| { + a.metadata_schema + .as_ref() + .map_or(true, |s| existing_schemas.contains(s)) + }); + let defer_tables = if ancestors.is_empty() { + vec![] + } else { + // Discovery walks the WHOLE chain: a table only a grandparent materialized has no + // physical copy in the direct parent (it defers too), so its view must target the + // nearest ancestor that owns one. + let ancestor_ids: Vec = ancestors.iter().map(|a| a.workspace_id.clone()).collect(); + let mut tables = + crate::materialization::list_fork_defer_tables(db, &ancestor_ids, w_id, name).await?; + // Catalog truth beats recorded status: any table live in the fork namespace is + // fork-owned, whatever its materialized_partition row says (failed-after-commit, + // raw SQL) — a defer view over it would silently yield to it. + tables.retain(|t| !fork_tables.contains(&t.table)); + tables + }; + + base.storage.path = fork_path; + base.extra_args = Some(match base.extra_args.take() { + Some(e) => { + let sanitized = strip_fork_reserved_attach_args(&e); + if sanitized.is_empty() { + format!("METADATA_SCHEMA '{metadata_schema}'") + } else { + format!("{sanitized}, METADATA_SCHEMA '{metadata_schema}'") + } + } + None => format!("METADATA_SCHEMA '{metadata_schema}'"), + }); + base.fork_defer = Some(DucklakeForkDefer { ancestors, defer_tables, fork_views }); + Ok(base) +} + +/// One round trip to the fork's catalog DB for both namespace introspections: which fork +/// ancestors' metadata schemas actually exist (a fresh intermediate fork may never have +/// bootstrapped its namespace), and the views currently live in THIS fork's namespace +/// (straight from DuckLake's `ducklake_view` metadata — missing metadata tables read as "no +/// views", correct since nothing can exist there). Each ancestor's schema is checked in the +/// ancestor's OWN catalog database — a fork whose catalog drifted away from an ancestor's +/// must not misread that ancestor's (existing, elsewhere) namespace as missing. Ancestors +/// whose catalog is unreachable read as missing — the safe direction: their defer is skipped +/// (loud absent-table reads) rather than emitting a READ_ONLY attach that would fail every +/// job in this fork. +async fn inspect_fork_catalog( + base: &DucklakeWithConnData, + metadata_schema: &str, + ancestors: &[DucklakeAncestorAttach], + db: &DB, +) -> Result<( + std::collections::HashSet, + Vec, + std::collections::HashSet, +)> { + // Ancestor fork-schemas grouped by which catalog database they live in. + let mut groups: std::collections::HashMap)> = + std::collections::HashMap::new(); + for a in ancestors { + if let Some(ms) = &a.metadata_schema { + groups + .entry(ducklake_catalog_identity(&a.catalog)) + .or_insert_with(|| (a.catalog_resource.clone(), vec![])) + .1 + .push(ms.clone()); + } + } + + async fn query_schemas( + client: &tokio_postgres::Client, + schemas: Vec, + ) -> std::result::Result, tokio_postgres::Error> { + client + .query( + "SELECT schema_name::text FROM information_schema.schemata + WHERE schema_name = ANY($1)", + &[&schemas], + ) + .await + .map(|rows| rows.iter().map(|r| r.get::<_, String>(0)).collect()) + } + + let mut existing_schemas: std::collections::HashSet = Default::default(); + + // The fork's own catalog: same-catalog ancestor schemas + this fork's live views. + let pg: crate::PgDatabase = + serde_json::from_value(base.catalog_resource.clone()).map_err(|e| { + Error::internal_err(format!("ducklake catalog resource is not postgres: {e}")) + })?; + let (client, connection) = pg.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let same_catalog = groups.remove(&ducklake_catalog_identity(&base.catalog)); + let same_catalog_res = match same_catalog { + Some((_, schemas)) => query_schemas(&client, schemas).await.map(Some), + None => Ok(None), + }; + // Identifier-quoted schema: the name is server-derived (mangled + hashed) but quote anyway. + let q = format!( + r#"SELECT CASE WHEN s.schema_name = 'main' THEN v.view_name + ELSE s.schema_name || '.' || v.view_name END AS name + FROM "{ms}".ducklake_view v + JOIN "{ms}".ducklake_schema s ON s.schema_id = v.schema_id AND s.end_snapshot IS NULL + WHERE v.end_snapshot IS NULL"#, + ms = metadata_schema.replace('"', "\"\"") + ); + let view_rows = client.query(&q, &[]).await; + // Live TABLES in the fork's namespace, same name shape as the views — the defer list is + // filtered against them: `CREATE VIEW IF NOT EXISTS` silently yields to an existing + // table, so emitting a defer view over one would leave reads on the fork table while + // claiming they defer (recorded status can't tell — a committed write whose data tests + // failed, or a table left by raw SQL, has no `materialized` row). + let qt = format!( + r#"SELECT CASE WHEN s.schema_name = 'main' THEN t.table_name + ELSE s.schema_name || '.' || t.table_name END AS name + FROM "{ms}".ducklake_table t + JOIN "{ms}".ducklake_schema s ON s.schema_id = t.schema_id AND s.end_snapshot IS NULL + WHERE t.end_snapshot IS NULL"#, + ms = metadata_schema.replace('"', "\"\"") + ); + let table_rows = client.query(&qt, &[]).await; + drop(client); + let _ = join_handle.await; + + existing_schemas.extend( + same_catalog_res + .map_err(|e| { + Error::internal_err(format!("checking fork ducklake ancestor schemas: {e}")) + })? + .unwrap_or_default(), + ); + let fork_views = match view_rows { + Ok(rows) => rows.iter().map(|r| r.get::<_, String>(0)).collect(), + // 42P01 undefined_table / 3F000 invalid_schema_name: namespace not bootstrapped yet. + Err(e) + if e.code().map_or(false, |c| { + c == &tokio_postgres::error::SqlState::UNDEFINED_TABLE + || c == &tokio_postgres::error::SqlState::INVALID_SCHEMA_NAME + }) => + { + vec![] + } + Err(e) => { + return Err(Error::internal_err(format!( + "listing fork ducklake views in {metadata_schema}: {e}" + ))) + } + }; + let fork_tables = match table_rows { + Ok(rows) => rows.iter().map(|r| r.get::<_, String>(0)).collect(), + Err(e) + if e.code().map_or(false, |c| { + c == &tokio_postgres::error::SqlState::UNDEFINED_TABLE + || c == &tokio_postgres::error::SqlState::INVALID_SCHEMA_NAME + }) => + { + Default::default() + } + Err(e) => { + return Err(Error::internal_err(format!( + "listing fork ducklake tables in {metadata_schema}: {e}" + ))) + } + }; + + // Ancestors living in OTHER catalog databases (this fork's catalog drifted after forking): + // one connection per distinct catalog, best-effort — an unreachable ancestor catalog only + // disables that ancestor's defer. + for (identity, (resource, schemas)) in groups { + let checked = async { + let pg: crate::PgDatabase = serde_json::from_value(resource) + .map_err(|e| Error::internal_err(format!("not a postgres resource: {e}")))?; + let (client, connection) = pg.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let res = query_schemas(&client, schemas).await; + drop(client); + let _ = join_handle.await; + res.map_err(|e| Error::internal_err(format!("{e}"))) + } + .await; + match checked { + Ok(found) => existing_schemas.extend(found), + Err(e) => { + tracing::warn!( + "fork ancestor catalog `{identity}` unreachable while checking ducklake \ + namespaces ({e:#}); its ancestors' defer is disabled for this resolution" + ); + } + } + } + Ok((existing_schemas, fork_views, fork_tables)) +} + +/// Canonical identity of a lake's catalog database (`:`) — used +/// for the cleanup registry and for grouping ancestors by which catalog their namespace lives +/// in. Identifies the database *pointer*, not its (live-resolved) credentials. +pub fn ducklake_catalog_identity(catalog: &DucklakeCatalog) -> String { + format!( + "{}:{}", + catalog.resource_type.as_ref(), + catalog.resource_path + ) +} + +/// Record that a fork attached this lake at this physical location, so fork deletion knows +/// exactly which pg metadata schema and which storage prefixes to clean up. One row per +/// (lake, storage, data path) EVER attached — if the fork's lake settings drift, later +/// attaches add rows rather than replace them, so cleanup covers every prefix the fork wrote. +/// The once-cache is keyed on the full location for the same reason. Re-registering an +/// existing row resets its `schema_dropped` cleanup phase: attaching recreates the metadata +/// schema, so a stale "already dropped" marker would make the eventual cleanup skip a live +/// schema. +async fn register_fork_ducklake_namespace( + db: &DB, + w_id: &str, + name: &str, + metadata_schema: &str, + catalog: &str, + storage: Option<&str>, + data_path: &str, +) -> Result<()> { + // '' = default storage (the column is part of the PK, which cannot hold NULL). + let storage = storage.unwrap_or(""); + // Resolve the logical storage name to its identity NOW: cleanup must delete from the + // storage that was active when the data was written, not whatever the name points at by + // deletion time. '' = unresolvable (no LFS configured — the write itself will fail at the + // proxy, so nothing lands anywhere). + let storage_ref = fork_storage_ref(db, w_id, storage) + .await + .unwrap_or_default(); + let location = format!("{name}\0{catalog}\0{storage}\0{storage_ref}\0{data_path}"); + let now = chrono::Utc::now().timestamp(); + if FORK_DUCKLAKE_REGISTERED + .get(w_id) + .is_some_and(|(locations, exp)| exp > now && locations.contains(&location)) + { + return Ok(()); + } + sqlx::query!( + "INSERT INTO fork_ducklake_namespace + (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path) + DO UPDATE SET schema_dropped = false", + w_id, + name, + metadata_schema, + catalog, + storage, + &storage_ref, + data_path, + ) + .execute(db) + .await + .map_err(|e| Error::internal_err(format!("registering fork ducklake namespace: {e:#}")))?; + let mut locations = FORK_DUCKLAKE_REGISTERED + .get(w_id) + .filter(|(_, exp)| *exp > now) + .map(|(locations, _)| locations) + .unwrap_or_default(); + locations.insert(location); + FORK_DUCKLAKE_REGISTERED.insert(w_id.to_string(), (locations, now + 60)); + Ok(()) +} + +/// Canonical identity of a workspace storage (`:`) as +/// configured RIGHT NOW — recorded in the fork namespace registry so cleanup targets the +/// storage the data was actually written to. `storage` = '' for the primary storage, else a +/// `secondary_storage` name. Returns None when no matching storage is configured. +async fn fork_storage_ref(db: &DB, w_id: &str, storage: &str) -> Option { + let lfs_json = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten()?; + let entry = if storage.is_empty() || storage == "_default_" { + lfs_json.clone() + } else { + lfs_json.get("secondary_storage")?.get(storage)?.clone() + }; + lfs_entry_storage_ref(&entry) +} + +/// The pure part of [`fork_storage_ref`]: one `LargeFileStorage` JSON entry → its canonical +/// `:` descriptor. The `$res:` prefix on resource paths is normalized away (stored +/// configs are inconsistent about it); cleanup re-adds it when resolving. +pub fn lfs_entry_storage_ref(entry: &serde_json::Value) -> Option { + let typ = entry.get("type")?.as_str()?; + let path_field = match typ { + "S3Storage" | "S3AwsOidc" => "s3_resource_path", + "AzureBlobStorage" | "AzureWorkloadIdentity" => "azure_blob_resource_path", + "GoogleCloudStorage" => "gcs_resource_path", + "FilesystemStorage" => "root_path", + _ => return None, + }; + let path = entry.get(path_field)?.as_str()?; + let path = path.strip_prefix("$res:").unwrap_or(path); + Some(format!("{typ}:{path}")) +} + +/// Resolve a `$res:`/`$var:` reference tree to its concrete value (recursively, secrets +/// decrypted). No permission checks — trusted server-side callers only; never echo the result +/// to a user. +pub async fn transform_json_value_unchecked( + value: &serde_json::Value, + w_id: &str, + db: &DB, +) -> Result { + transform_json_unchecked(value, w_id, db).await } // This does not check for any permission. Should never be displayed to a user. @@ -1140,4 +1821,162 @@ mod tests { let long_id = format!("wm-fork-{}", "a".repeat(43)); assert!(validate_fork_workspace_id(&long_id).is_err()); } + + #[test] + fn test_fork_ducklake_metadata_schema_shape() { + let s = fork_ducklake_metadata_schema("wm-fork-my-feature-42", "main"); + assert!(s.starts_with(FORK_DUCKLAKE_SCHEMA_PREFIX), "{s}"); + assert!(s.len() <= 63, "pg schema name limit: {s}"); + assert!( + s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'), + "{s}" + ); + // Deterministic (persisted view SQL / registry rows depend on it). + assert_eq!( + s, + fork_ducklake_metadata_schema("wm-fork-my-feature-42", "main") + ); + } + + #[test] + fn test_fork_ducklake_metadata_schema_injective_after_mangling() { + // `-` and `_` mangle to the same char; the hash suffix must keep them distinct. + let a = fork_ducklake_metadata_schema("wm-fork-a-b", "main"); + let b = fork_ducklake_metadata_schema("wm-fork-a_b", "main"); + assert_ne!(a, b); + // Lake-scoped: two lakes of one workspace may share a catalog database, so their fork + // namespaces must be distinct schemas. + assert_ne!( + fork_ducklake_metadata_schema("wm-fork-a-b", "lake_a"), + fork_ducklake_metadata_schema("wm-fork-a-b", "lake_b") + ); + // Long ids truncate to the same mangled prefix; hash must still differ. + let long_a = fork_ducklake_metadata_schema(&format!("wm-fork-{}x", "a".repeat(40)), "main"); + let long_b = fork_ducklake_metadata_schema(&format!("wm-fork-{}y", "a".repeat(40)), "main"); + assert_ne!(long_a, long_b); + let long_lake = + fork_ducklake_metadata_schema(&format!("wm-fork-{}", "a".repeat(42)), &"l".repeat(40)); + assert!(long_a.len() <= 63 && long_b.len() <= 63 && long_lake.len() <= 63); + } + + #[test] + fn test_fork_ducklake_ancestor_alias_valid_identifier() { + let a = fork_ducklake_ancestor_alias("analytics", "wm-fork-dev-1"); + assert!(a.starts_with("__wm_dl_"), "{a}"); + assert!( + a.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'), + "{a}" + ); + // Distinct per lake for the same ancestor (a script can attach several lakes). + assert_ne!(a, fork_ducklake_ancestor_alias("staging", "wm-fork-dev-1")); + assert_ne!( + a, + fork_ducklake_ancestor_alias("analytics", "wm-fork-dev-2") + ); + } + + #[test] + fn test_extract_metadata_schema_arg() { + assert_eq!( + extract_metadata_schema_arg("METADATA_SCHEMA 'lake_b_ns', ENCRYPTED true"), + Some("lake_b_ns".to_string()) + ); + assert_eq!( + extract_metadata_schema_arg("metadata_schema bare_ident"), + Some("bare_ident".to_string()) + ); + // Last occurrence wins (DuckDB duplicate-option semantics). + assert_eq!( + extract_metadata_schema_arg("METADATA_SCHEMA 'a', METADATA_SCHEMA 'b'"), + Some("b".to_string()) + ); + assert_eq!(extract_metadata_schema_arg("ENCRYPTED true"), None); + } + + #[test] + fn test_fork_data_path_prefix_isolation() { + // Fork/dev ids are git-branch-safe and may contain `/`: `wm-fork-a/b` is a valid id. + // Its data prefix must NOT nest inside `wm-fork-a`'s, or deleting `wm-fork-a` would + // sweep the sibling's files via the object-store prefix listing. + assert!(validate_fork_workspace_id("wm-fork-a/b").is_ok()); + let a = fork_data_path("lake", "wm-fork-a"); + let ab = fork_data_path("lake", "wm-fork-a/b"); + assert!( + !format!("{ab}/").starts_with(&format!( + "{FORK_DUCKLAKE_DATA_DIR}/{}/", + fork_data_dir_segment("wm-fork-a") + )), + "{ab} nests under {a}'s cleanup prefix" + ); + // Single path component: the segment itself contains no separator. + assert!(!fork_data_dir_segment("wm-fork-a/b").contains('/')); + // Injective after mangling (`/` and `_` both mangle to `_`). + assert_ne!( + fork_data_dir_segment("wm-fork-a/b"), + fork_data_dir_segment("wm-fork-a_b") + ); + // Deterministic (registry rows + cleanup guard recompute it). + assert_eq!(a, fork_data_path("lake", "wm-fork-a")); + // Empty base path still yields a well-formed prefix. + assert_eq!( + fork_data_path("", "wm-fork-a"), + format!( + "{FORK_DUCKLAKE_DATA_DIR}/{}", + fork_data_dir_segment("wm-fork-a") + ) + ); + } + + #[test] + fn test_lfs_entry_storage_ref() { + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({ + "type": "S3Storage", "s3_resource_path": "$res:u/admin/minio" + })), + Some("S3Storage:u/admin/minio".to_string()) + ); + // The `$res:` prefix is optional in stored configs; normalized either way. + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({ + "type": "AzureBlobStorage", "azure_blob_resource_path": "u/admin/az" + })), + Some("AzureBlobStorage:u/admin/az".to_string()) + ); + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({ + "type": "FilesystemStorage", "root_path": "/data/lfs" + })), + Some("FilesystemStorage:/data/lfs".to_string()) + ); + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({"type": "SomethingNew"})), + None + ); + assert_eq!(lfs_entry_storage_ref(&serde_json::json!({})), None); + } + + #[test] + fn test_strip_fork_reserved_attach_args() { + // Reserved options removed wherever they appear, others preserved. + assert_eq!( + strip_fork_reserved_attach_args("METADATA_SCHEMA 'main', ENCRYPTED true"), + "ENCRYPTED true" + ); + assert_eq!( + strip_fork_reserved_attach_args( + "ENCRYPTED true, DATA_PATH 's3://b/prod', OVERRIDE_DATA_PATH FALSE" + ), + "ENCRYPTED true" + ); + // Case-insensitive, and quoted values may contain commas/spaces. + assert_eq!( + strip_fork_reserved_attach_args("data_path 's3://b/x, y', SNAPSHOT_VERSION 3"), + "SNAPSHOT_VERSION 3" + ); + assert_eq!(strip_fork_reserved_attach_args(""), ""); + assert_eq!( + strip_fork_reserved_attach_args("METADATA_SCHEMA 'wm_fork_evil'"), + "" + ); + } } diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index 745b2685e9..e11ced23bc 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -1656,12 +1656,13 @@ mod schedule_push { tx.commit().await?; assert_eq!(count_queued_jobs(&db).await, 1); - let (kind, path, tag, raw_code) = sqlx::query_as::<_, (String, Option, String, Option)>( - "SELECT j.kind::text, j.runnable_path, j.tag, j.raw_code + let (kind, path, tag, raw_code) = + sqlx::query_as::<_, (String, Option, String, Option)>( + "SELECT j.kind::text, j.runnable_path, j.tag, j.raw_code FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", - ) - .fetch_one(&db) - .await?; + ) + .fetch_one(&db) + .await?; assert_eq!(kind, "preview"); assert_eq!(path.as_deref(), Some("f/ducklake_maintenance/legacy")); assert_eq!(tag, "duckdb"); @@ -1694,6 +1695,7 @@ mod schedule_push { }, storage: DucklakeStorage { storage: None, path: "legacy".to_string() }, extra_args: None, + fork_behavior: None, maintenance: Some(DucklakeMaintenance { enabled: maintenance_enabled, schedule: None, diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 31db95b91a..817c86281c 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -14,7 +14,7 @@ use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked, - DucklakeCatalogResourceType, + strip_fork_reserved_attach_args, DucklakeCatalogResourceType, }; use windmill_common::PgDatabase; use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE; @@ -1155,6 +1155,7 @@ pub async fn do_duckdb( conn, &mut hidden_passwords, &job.workspace_id, + materialize.as_ref().map(|(_, m)| m.asset_path.as_str()), ) .await? { @@ -1858,6 +1859,7 @@ async fn transform_attach_ducklake( conn: &Connection, hidden_passwords: &mut Arc>>, w_id: &str, + materialize_target: Option<&str>, ) -> Result>> { lazy_static::lazy_static! { static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'ducklake(://[^':]+)?'\s*AS\s+([^ ;]+)\s*(\([^)]*\))?").unwrap(); @@ -1867,15 +1869,28 @@ async fn transform_attach_ducklake( }; let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main"); let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or(""); - let extra_args = cap + let user_extra_args = cap .get(3) - .map(|m| format!(", {}", &m.as_str()[1..m.as_str().len() - 1])) - .unwrap_or("".to_string()); + .map(|m| m.as_str()[1..m.as_str().len() - 1].to_string()) + .unwrap_or_default(); let ducklake = match conn { Connection::Http(client) => get_ducklake_from_agent_http(client, name, w_id).await?, Connection::Sql(db) => get_ducklake_from_db_unchecked(name, w_id, db).await?, }; + // In a fork, METADATA_SCHEMA / DATA_PATH / OVERRIDE_DATA_PATH are owned by the fork + // resolution (DuckDB silently keeps the last occurrence of a duplicated option, so a + // user-supplied one would escape the fork namespace back to the parent's). + let user_extra_args = if ducklake.fork_defer.is_some() { + strip_fork_reserved_attach_args(&user_extra_args) + } else { + user_extra_args + }; + let extra_args = if user_extra_args.is_empty() { + String::new() + } else { + format!(", {}", user_extra_args) + }; let db_type = match ducklake.catalog.resource_type { DucklakeCatalogResourceType::Instance => "postgres", _ => ducklake.catalog.resource_type.as_ref(), @@ -1921,17 +1936,170 @@ async fn transform_attach_ducklake( } else { format!(", AUTOMATIC_MIGRATION TRUE{extra_args}") }; + // In a fork, re-emit the fork-owned DATA_PATH as the LAST option: DuckDB keeps the last + // occurrence of a duplicated option, so this wins over anything the arg stripping might + // not recognize (e.g. a dollar-quoted literal), regardless of literal syntax. The + // METADATA_SCHEMA injected by the fork resolution is already last within `extra_args`. + let extra_args = if ducklake.fork_defer.is_some() { + format!("{extra_args}, DATA_PATH 's3://{storage}/{data_path}'") + } else { + extra_args + }; let attach_str = format!( "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});", ); let install_db_ext_str = get_attach_db_install_str(db_type)?; - Ok(Some(vec![ + let mut statements = vec![ "INSTALL ducklake;".to_string(), install_db_ext_str.to_string(), attach_str, - ])) + ]; + if let Some(defer) = ducklake.fork_defer.as_ref() { + statements.extend(fork_defer_statements( + name, + alias_name, + defer, + materialize_target, + hidden_passwords, + )?); + } + Ok(Some(statements)) +} + +// Double-quote a possibly schema-qualified table reference, each dotted segment +// independently (local copy of sql_materialize's private helper). +fn quote_qualified_table(name: &str) -> String { + name.split('.') + .map(|id| format!("\"{}\"", id.replace('"', "\"\""))) + .collect::>() + .join(".") +} + +// `schema.table` → `schema.table_current`: the SCD2 companion view lives next to its table. +fn current_companion(table: &str) -> String { + match table.rsplit_once('.') { + Some((s, t)) => format!("{s}.{t}_current"), + None => format!("{table}_current"), + } +} + +/// Statements appended after a fork workspace's lake ATTACH: read-only attaches of the +/// ancestor namespaces plus `CREATE VIEW IF NOT EXISTS` defer views over the direct parent for +/// every table the fork has not materialized. When this job's managed materialize targets one +/// of the deferred tables, its defer view is dropped instead of created — the write must hit a +/// real fork table (`CREATE [OR REPLACE] TABLE` refuses to replace a view). +fn fork_defer_statements( + lake_name: &str, + alias_name: &str, + defer: &windmill_common::workspaces::DucklakeForkDefer, + materialize_target: Option<&str>, + hidden_passwords: &mut Arc>>, +) -> Result> { + let mut stmts = vec![]; + if defer.ancestors.is_empty() { + // Defer unavailable (an ancestor no longer defines this lake); the fork namespace + // still isolates writes. + return Ok(stmts); + } + for a in &defer.ancestors { + if let Some(pwd) = a.catalog_resource.get("password").and_then(|p| p.as_str()) { + hidden_passwords.lock().unwrap().push(pwd.to_string()); + } + let db_type = match a.catalog.resource_type { + DucklakeCatalogResourceType::Instance => "postgres", + _ => a.catalog.resource_type.as_ref(), + }; + stmts.push(get_attach_db_install_str(db_type)?.to_string()); + let conn_str = + format_attach_db_conn_str(a.catalog_resource.clone(), db_type)?.replace('\'', "''"); + let storage = a + .storage + .storage + .as_deref() + .unwrap_or(DEFAULT_STORAGE) + .replace('\'', "''"); + let data_path = a.storage.path.replace('\'', "''"); + let metadata_schema = a + .metadata_schema + .as_ref() + .map(|s| format!(", METADATA_SCHEMA '{}'", s.replace('\'', "''"))) + .unwrap_or_default(); + // The ancestor config's own non-reserved args (e.g. `ENCRYPTED true`) — an + // option-dependent lake wouldn't attach without them. Emitted FIRST: DuckDB keeps the + // last occurrence of a duplicated option, so the fork-owned DATA_PATH / READ_ONLY / + // METADATA_SCHEMA after them always win. + let extra_args = a + .extra_args + .as_ref() + .map(|e| format!("{e}, ")) + .unwrap_or_default(); + // READ_ONLY: a fork job must never write an ancestor namespace. No AUTOMATIC_MIGRATION + // / CREATE_IF_NOT_EXISTS: an ancestor lake that would need creating or migrating fails + // loudly rather than being mutated from a fork. IF NOT EXISTS: several ATTACH blocks in + // one script (e.g. the user's + the materialize synthetic one) emit the same ancestors. + stmts.push(format!( + "ATTACH IF NOT EXISTS 'ducklake:{db_type}:{conn_str}' AS {} ({extra_args}DATA_PATH 's3://{storage}/{data_path}', OVERRIDE_DATA_PATH TRUE, READ_ONLY{metadata_schema});", + a.alias + )); + } + let parent_alias = &defer.ancestors[0].alias; + let target_table = materialize_target.and_then(|ap| { + let (l, t) = ap.split_once('/')?; + (l == lake_name).then_some(t) + }); + let mut created_schemas = std::collections::HashSet::new(); + for dt in &defer.defer_tables { + if Some(dt.table.as_str()) == target_table { + continue; + } + // Each view targets the NEAREST ancestor that physically owns the table (a direct + // parent that itself defers has no copy to bind against). Out-of-range index (never + // produced by the resolver, but the field crosses the agent wire) falls back to the + // direct parent rather than panicking. + let owner_alias = defer + .ancestors + .get(dt.ancestor_idx as usize) + .map(|a| a.alias.as_str()) + .unwrap_or(parent_alias); + if let Some((schema, _)) = dt.table.rsplit_once('.') { + if created_schemas.insert(schema) { + stmts.push(format!( + "CREATE SCHEMA IF NOT EXISTS {alias_name}.{};", + quote_qualified_table(schema) + )); + } + } + let q = quote_qualified_table(&dt.table); + stmts.push(format!( + "CREATE VIEW IF NOT EXISTS {alias_name}.{q} AS SELECT * FROM {owner_alias}.{q};" + )); + if dt.with_current_view { + let qc = quote_qualified_table(¤t_companion(&dt.table)); + stmts.push(format!( + "CREATE VIEW IF NOT EXISTS {alias_name}.{qc} AS SELECT * FROM {owner_alias}.{qc};" + )); + } + } + if let Some(t) = target_table { + // View→table transition: replace the target's defer view(s) with the real table this + // job writes (`CREATE [OR REPLACE] TABLE` refuses to replace a view). Keyed on the + // catalog's ACTUAL live views — not on recorded materialization status, which after a + // failed run can't tell a defer view from a real table, and a mismatched DROP VIEW + // would wedge the asset. `_current` is dropped too when it is a view — SCD2 codegen + // recreates it with `IF NOT EXISTS`, which would otherwise silently keep a view over + // the parent. + for name in [t.to_string(), current_companion(t)] { + if defer.fork_views.iter().any(|v| v == &name) { + stmts.push(format!( + "DROP VIEW IF EXISTS {alias_name}.{};", + quote_qualified_table(&name) + )); + } + } + } + Ok(stmts) } async fn transform_attach_datatable( @@ -2036,6 +2204,198 @@ pub struct Arg { mod tests { use super::*; + fn test_ancestor(wid: &str) -> windmill_common::workspaces::DucklakeAncestorAttach { + windmill_common::workspaces::DucklakeAncestorAttach { + workspace_id: wid.to_string(), + alias: format!("__wm_dl_lake_{}_abcd1234", wid.replace('-', "_")), + catalog: windmill_common::workspaces::DucklakeCatalog { + resource_type: windmill_common::workspaces::DucklakeCatalogResourceType::Instance, + resource_path: "cat_db".to_string(), + }, + catalog_resource: serde_json::json!({ + "host": "h", "dbname": "cat_db", "user": "u", "password": "pw" + }), + storage: windmill_common::workspaces::DucklakeStorage { + storage: None, + path: "lake".to_string(), + }, + metadata_schema: None, + extra_args: None, + } + } + + fn test_fork_defer_chain( + ancestor_wids: Vec<&str>, + defer_tables: Vec<(&str, bool, u32)>, + fork_views: Vec<&str>, + ) -> windmill_common::workspaces::DucklakeForkDefer { + windmill_common::workspaces::DucklakeForkDefer { + ancestors: ancestor_wids.into_iter().map(test_ancestor).collect(), + defer_tables: defer_tables + .into_iter() + .map( + |(t, cur, idx)| windmill_common::materialization::ForkDeferTable { + table: t.to_string(), + with_current_view: cur, + ancestor_idx: idx, + }, + ) + .collect(), + fork_views: fork_views.into_iter().map(str::to_string).collect(), + } + } + + fn test_fork_defer( + defer_tables: Vec<(&str, bool)>, + fork_views: Vec<&str>, + ) -> windmill_common::workspaces::DucklakeForkDefer { + test_fork_defer_chain( + vec!["parent-ws"], + defer_tables.into_iter().map(|(t, c)| (t, c, 0)).collect(), + fork_views, + ) + } + + #[test] + fn test_fork_defer_statements_ancestor_extra_args() { + // The ancestor config's own args (e.g. ENCRYPTED) must ride on its read-only attach — + // BEFORE the fork-owned options, so DuckDB's last-occurrence-wins keeps DATA_PATH / + // READ_ONLY / METADATA_SCHEMA authoritative even if the args tried to override them. + let mut defer = test_fork_defer(vec![("orders", false)], vec![]); + defer.ancestors[0].extra_args = Some("ENCRYPTED true".to_string()); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let attach = stmts + .iter() + .find(|s| s.starts_with("ATTACH IF NOT EXISTS")) + .unwrap(); + assert!(attach.contains("(ENCRYPTED true, DATA_PATH "), "{attach}"); + assert!( + attach.find("ENCRYPTED true").unwrap() < attach.find("READ_ONLY").unwrap(), + "{attach}" + ); + } + + #[test] + fn test_fork_defer_statements_chained_ancestors() { + // fork → parent → root: `orders` was only materialized in the root (idx 1) — its view + // must target the ROOT alias (the parent has no physical copy); `daily` owned by the + // direct parent (idx 0) targets the parent alias. Out-of-range idx falls back to the + // direct parent instead of panicking. + let defer = test_fork_defer_chain( + vec!["parent-ws", "root-ws"], + vec![("orders", false, 1), ("daily", false, 0), ("oob", false, 9)], + vec![], + ); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let joined = stmts.join("\n"); + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"orders\" AS SELECT * FROM __wm_dl_lake_root_ws_abcd1234.\"orders\"" + ), + "{joined}" + ); + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"daily\" AS SELECT * FROM __wm_dl_lake_parent_ws_abcd1234.\"daily\"" + ), + "{joined}" + ); + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"oob\" AS SELECT * FROM __wm_dl_lake_parent_ws_abcd1234.\"oob\"" + ), + "{joined}" + ); + // Both ancestors attached read-only. + assert_eq!( + joined.matches("ATTACH IF NOT EXISTS").count(), + 2, + "{joined}" + ); + } + + #[test] + fn test_fork_defer_statements_shape() { + let defer = test_fork_defer(vec![("orders", false), ("dim", true)], vec![]); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let joined = stmts.join("\n"); + // Ancestor attach: read-only, idempotent, never auto-migrating or auto-creating. + assert!(joined.contains("ATTACH IF NOT EXISTS"), "{joined}"); + assert!(joined.contains("READ_ONLY"), "{joined}"); + assert!(!joined.contains("AUTOMATIC_MIGRATION"), "{joined}"); + assert!(!joined.contains("CREATE_IF_NOT_EXISTS"), "{joined}"); + // Ancestor catalog password is hidden from logs. + assert_eq!(hp.lock().unwrap().as_slice(), ["pw"]); + // Defer views over the parent alias, plus the SCD2 `_current` companion. + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"orders\" AS SELECT * FROM __wm_dl_lake_parent_ws_abcd1234.\"orders\"" + ), + "{joined}" + ); + assert!(joined.contains("dl.\"dim_current\""), "{joined}"); + // No target → no transition drops. + assert!(!joined.contains("DROP VIEW"), "{joined}"); + } + + #[test] + fn test_fork_defer_statements_target_transition() { + // Target currently a defer view → skip its CREATE, drop the view (+ companion). + let defer = test_fork_defer(vec![("orders", false)], vec!["orders", "orders_current"]); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = + fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp) + .unwrap(); + let joined = stmts.join("\n"); + assert!(!joined.contains("CREATE VIEW"), "{joined}"); + assert!( + joined.contains("DROP VIEW IF EXISTS _wm_target.\"orders\";"), + "{joined}" + ); + assert!( + joined.contains("DROP VIEW IF EXISTS _wm_target.\"orders_current\";"), + "{joined}" + ); + + // Target already a real table (NOT in fork_views, e.g. after a failed re-run whose + // status can't be trusted) → no DROP VIEW, or the job would wedge on a type mismatch. + let defer = test_fork_defer(vec![("orders", false)], vec![]); + let stmts = + fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp) + .unwrap(); + assert!(!stmts.join("\n").contains("DROP VIEW"), "{stmts:?}"); + + // Target in a different lake → this lake's defer views are untouched. + let defer = test_fork_defer(vec![("orders", false)], vec!["orders"]); + let stmts = + fork_defer_statements("lake", "dl", &defer, Some("other/orders"), &mut hp).unwrap(); + let joined = stmts.join("\n"); + assert!( + joined.contains("CREATE VIEW IF NOT EXISTS dl.\"orders\""), + "{joined}" + ); + assert!(!joined.contains("DROP VIEW"), "{joined}"); + } + + #[test] + fn test_fork_defer_statements_schema_qualified() { + let defer = test_fork_defer(vec![("staging.raw", false)], vec![]); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let joined = stmts.join("\n"); + assert!( + joined.contains("CREATE SCHEMA IF NOT EXISTS dl.\"staging\";"), + "{joined}" + ); + assert!( + joined.contains("CREATE VIEW IF NOT EXISTS dl.\"staging\".\"raw\""), + "{joined}" + ); + } + fn mrow(name: &str, params: &str, body: &str, is_table: bool, provider: &str) -> MacroRow { MacroRow { name: name.to_string(), diff --git a/docs/pipeline-env-isolation.md b/docs/pipeline-env-isolation.md new file mode 100644 index 0000000000..2b6b87ba68 --- /dev/null +++ b/docs/pipeline-env-isolation.md @@ -0,0 +1,219 @@ +# Fork data environments for pipeline materialization ("dev data") + +Status: **implemented** (branch `pipeline-env-isolation`). Companion to +[`ducklake-materialization.md`](./ducklake-materialization.md). + +## The problem + +Workspace forks fork the CODE, but `// materialize ducklake:///
` targets are +absolute pointers into shared physical storage. A fork clones `workspace_settings.ducklake` +verbatim, so `transform_attach_ducklake` resolved a fork's `ducklake://main` to the **parent's +catalog database and S3 data path** — deploying or running a materializing pipeline inside a +fork wrote the production tables. dbt answers this with target schemas + `--defer`; datatables +already have a fork answer (`forked_datatables` → `wm_fork_*` databases + a drop endpoint); +ducklake had none. `wmill pipeline run --local` only covers preview runs, and previews still +resolve targets against the real workspace. + +## The design + +Everything hangs off two DuckLake ATTACH capabilities (verified in a spike, DuckDB 1.5.4 + +current ducklake extension): `METADATA_SCHEMA` (the catalog's metadata tables live in a chosen +pg schema of the same catalog DB — auto-created on first attach; default is `public`) and +`READ_ONLY`, plus catalog-stored views that may reference other attached catalogs and rebind +per session. + +### Fork-time choice: isolated (default) vs shared, per lake + +The fork-creation dialog offers a per-lake data-environment choice (the section renders only +when the parent workspace has lakes configured, mirroring the datatable section): + +- **Isolated** (default — also what unlisted lakes and API callers that omit the field get) — + the full behavior described below. +- **Shared** — explicit opt-out: the fork reads and writes the parent's lake directly through + its cloned config, like a non-fork workspace (e.g. a fork meant to run prod-equivalent + backfills). Stamped as `fork_behavior: "shared"` on the lake entry in the fork's own + `workspace_settings.ducklake` at creation (`shared_ducklakes` on the create-fork request); + the resolver, the graph indicator, and (trivially — nothing registers) cleanup all key off + it. Shared also lifts the mysql-catalog restriction, since nothing needs namespacing. + +Absent-field-means-isolated keeps every pre-existing fork and API caller on the safe path. +Sharing is a per-fork-creation choice, never inherited: the settings clone copies the source's +config verbatim, so fork creation strips any cloned `fork_behavior` stamps before applying its +own `shared_ducklakes` list — a fork of a shared fork defaults back to isolated. A +shared fork ancestor in a fork chain resolves like a root (its data lives at its config's +default location). Flipping a fork's lake to `shared` later (settings edit) orphans any +namespace it already materialized until fork deletion cleans it — the registry row survives +the flip. + +### Write redirect — one seam, no grammar changes + +`get_ducklake_from_db_unchecked` (windmill-common/src/workspaces.rs) is fork-aware: when the +resolved workspace has a `parent_workspace_id` (throwaway `wm-fork-*` fork or standing dev +workspace), the returned `DucklakeWithConnData` is namespaced **in its existing fields**: + +- `extra_args` += `METADATA_SCHEMA 'wm_fork___'` — + injective, ≤63 chars, stable across releases (persisted state depends on it), and + **lake-scoped**: two lakes of one workspace may share a catalog database, and a + per-workspace schema would merge their namespaces in the fork; +- `storage.path` = `__wm_forks//` — the segment is the + mangled+hashed workspace id collapsed to ONE path component (fork ids are only + git-branch-safe and may contain `/`; used raw, `wm-fork-a/b` would nest inside sibling + `wm-fork-a`'s cleanup prefix) — a **bucket-root prefix**, not a + sub-path of the parent's DATA_PATH, because lake maintenance + (`ducklake_delete_orphaned_files`, snapshot expiry) scans everything under the parent's path + and would delete live fork files nested there. + +Folding the redirect into existing fields means the EE agent-worker endpoint (which just +serializes this struct) needs **no changes**, and an agent worker running an older binary still +writes the fork namespace — it only misses the defer views (reads fail loudly; never a prod +write). Every ducklake access funnels through the same ATTACH transform — the managed +materialize's synthetic `_wm_target` attach, `materialize manual`, raw user SQL, and all UI +preview / History / backfill surfaces (they are `WM_INTERNAL_DB_*` marker preview jobs) — so a +fork cannot reach the parent namespace through the managed path at all. + +Hardening: in fork mode, user- and settings-supplied `METADATA_SCHEMA` / `DATA_PATH` / +`OVERRIDE_DATA_PATH` ATTACH options are stripped before injection (DuckDB silently keeps the +*last* occurrence of a duplicated option, so a duplicate would escape the namespace). +MySQL-catalog lakes error loudly in forks (no pg schemas to namespace with) rather than ever +writing the shared catalog. + +### Read-defer — iterate on one node without rebuilding upstream + +In a fork, `transform_attach_ducklake` additionally emits: + +1. `ATTACH IF NOT EXISTS … AS __wm_dl___ ([, ] + DATA_PATH , OVERRIDE_DATA_PATH TRUE, READ_ONLY[, METADATA_SCHEMA + ])` per fork-chain ancestor, resolved from the **ancestor's own settings** + (fork-side settings drift can't repoint what "parent" means). The ancestor config's + non-reserved `extra_args` (e.g. `ENCRYPTED true`) ride along — FIRST, so the fork-owned + options win under DuckDB's last-occurrence-wins. No + `AUTOMATIC_MIGRATION`/`CREATE_IF_NOT_EXISTS` — a fork job must never mutate an ancestor + lake. `READ_ONLY` makes prod writes *physically impossible* through ducklake, including in + `materialize manual` and raw SQL. Fork ancestors whose namespace was never bootstrapped are + skipped, with existence checked in each ancestor's OWN catalog database (a fork whose + catalog drifted away from an ancestor's must not misread that ancestor's namespace as + missing); an unreachable ancestor catalog only disables that ancestor's defer. +2. `CREATE VIEW IF NOT EXISTS .
AS SELECT * FROM .
` for every **deferred table**: (`materialized_partition` rows with + `status = 'materialized'` anywhere in the ancestor chain) MINUS (the fork's own rows). Each + view targets the NEAREST ancestor that physically owns the table + (`ForkDeferTable.ancestor_idx`): in a `fork → parent → root` chain where only the root + materialized a table, the parent has no copy (it defers too), so a view over the parent + would not bind. SCD2 owners (detected by the `is_current` column in the owning ancestor's + latest captured `materialized_asset_schema`) also get the `
_current` companion view. + So a consumer reading `dl.orders` transparently reads the nearest owning ancestor's current + data until the fork materializes `orders` itself. +3. When the running job's managed materialize targets a table that currently exists **as a + view** in the fork namespace, the view (and its `_current` companion) is dropped after the + target attach — the first fork materialize replaces the defer view with a real fork table + (`CREATE [OR REPLACE] TABLE` refuses to replace a view — verified — so without the drop the + transition fails; with it, `sql_materialize.rs` codegen needs no fork awareness at all). + The is-it-a-view decision keys on the catalog's ACTUAL live views (`ducklake_view`, read at + resolution time into `fork_defer.fork_views`), **not** on recorded materialization status: + after a failed run the status cannot distinguish a defer view from a real table, and a + mismatched `DROP VIEW` (or skipped drop) would wedge the asset on every retry — found live + in e2e when a storage failure flipped a fork asset's status to `Failed`. + +The graph shows the state per ducklake asset in a fork: an amber ↗ chip = *deferred* (some ancestor in the chain +owns the table), +an emerald fork chip = *fork-materialized* (`fork_materialization` on the `/assets/graph` +response, computed from fork + parent `materialized_partition`; the parent's rows are read on +the plain pool since fork membership doesn't imply parent membership). The details pane shows +the equivalent banner. + +### Lifecycle & cleanup + +A sidecar registry `fork_ducklake_namespace(workspace_id, ducklake_name, +metadata_schema, catalog, storage, storage_ref, data_path)` is upserted (behind a TTL'd in-process cache, keyed +per workspace and invalidated whenever a workspace's registry rows are cleaned up — a same-id fork recreated +within the TTL must re-register or its namespace would be orphaned at its own deletion) on fork +resolution, **one row per physical location ever attached** — its PK includes +`(catalog, storage, storage_ref, data_path)`, so if the fork's lake settings drift, later attaches add rows rather +than replace them and cleanup covers every location the fork wrote, not just the first — connecting to the +REGISTERED catalog identity and deleting files from the REGISTERED storage identity +(`storage_ref`, resolved from the logical storage name at attach time), never whatever the +fork's settings point at by deletion time. Explicit +`GRANT`s ship in the same migration; the registry is not cloned into sub-forks. + +The registry has deliberately **no FK to `workspace`**: rows are the durable cleanup ledger and +outlive the workspace row when physical cleanup fails after the delete commits (unreachable +catalog, storage outage) — each row is deleted only after its schema + data cleanup succeed. +Fork **creation** retries any leftover rows for the reused id and refuses to proceed while a +metadata schema still cannot be dropped, so a recreated same-id fork can never silently +reattach stale tables. Data-file leftovers alone don't block: a deleted fork's `$res:` storage +resource is gone forever, but once the schema is dropped the files are inert — the surviving +row keeps them tracked and the next successful cleanup of the same prefix (typically the +recreated fork's own deletion, with live credentials) sweeps them. + +Retries after the fork is gone get two aids: the row's `schema_dropped` phase flag (set when +the schema dropped but data cleanup failed; retries then skip the schema phase and need no +catalog credentials — registration resets it on re-attach, which recreates the schema), and +`$res:` fallback resolution against the workspace being forked (the deleted fork's resources +were clones of a parent's, so the new parent is the natural credential donor). + +`POST /w/{fork}/workspaces/drop_forked_ducklake_namespaces` (same permissions as +`delete_workspace` via a shared gate: fork owner or superadmin, plus parent-prod-admin when the +fork is an attached dev workspace) drops each registered pg metadata schema +(`DROP SCHEMA … CASCADE` on the catalog connection, hard-guarded on the `wm_fork_` prefix) and +deletes the `__wm_forks//…` objects in the workspace storage (hard-guarded on that +prefix; `parquet` feature). The fork-deletion UI calls it next to +`drop_forked_datatable_databases`, for the fork and for deleted child forks. Registry rows are +deleted per-lake only after both cleanups succeed, so a partial failure is retryable. + +## Semantics & caveats (by design) + +- **Orphaned forks stay isolated.** `parent_workspace_id` is `ON DELETE SET NULL`, so a + `wm-fork-*` workspace can outlive its parent. Its ancestor chain is then empty, but its cloned + config still points at the shared lake — so fork resolution keys on the `wm-fork-` prefix as + well as the chain (mirroring `workspace_is_fork`). An orphaned fork gets the write redirect, + registration and cleanup but no defer views; pre-orphaning defer views fail loudly at bind and + flip to real tables on first materialize. The same rule applies to a fork's *ancestors*: a + last-in-chain `wm-fork-*` ancestor is an orphaned fork (not a root) and its READ_ONLY attach + targets its fork namespace, not its cloned base config. Prefix-less dev workspaces cannot be + orphaned (deleting their prod is blocked while attached). +- **Defer is live, not snapshot-pinned.** A deferred read sees the parent's *current* table — + the same trade dbt `--defer` makes. Snapshot-pinned defer could later ride on the recorded + `snapshot_id`s. +- **Defer granularity is per-table.** Materializing one partition in the fork makes the fork's + table authoritative for *all* partitions (the rest are empty until backfilled in the fork). +- **Incremental strategies start empty in a fork.** `merge`/`append`/SCD2 create a fresh table + on first fork run — no parent rows are inherited, and fork SCD2 history diverges from the + parent's. Copy-on-fork seeding would be a follow-up opt-in. +- **Only tracked tables defer.** A parent table with no `materialized_partition` row (raw SQL + writes outside `// materialize`) or whose latest run `Failed` is not deferred — defer views + bind at CREATE against the parent catalog, and a view on a missing table would fail every + unrelated fork job. Such tables read as absent in the fork. +- **Transient race.** Between a fork materialize's commit and its state recording, a concurrent + fork job may still emit `CREATE VIEW IF NOT EXISTS` for that table — which silently yields to + an existing table (verified), so the race is a no-op. +- **Accident prevention, not a security boundary.** A fork holds the parent's cloned + credentials by design; a determined fork user can still `ATTACH 'postgres:…'` with the cloned + catalog resource or `COPY TO 's3://…'` directly. The isolation guarantees are scoped to the + managed ducklake path (where they are physical: READ_ONLY parent attach + separate metadata + schema + separate data prefix). +- **Fork chains** compose: each ancestor namespace is attached under a deterministic alias, and + a parent's own defer views (referencing *its* parent's alias) rebind correctly in the + grandchild's session because the whole chain is attached. +- **Datatables are out of scope** — the existing `forked_datatables` chooser at fork creation + covers them (`keep_original` remains their default, which shares the parent DB; picking + `schema_only` there is the datatable analog of this feature). + +## Key files + +- `backend/windmill-common/src/workspaces.rs` — fork resolution (`fork_scoped_ducklake`), + `fork_ancestor_chain`, naming (`fork_ducklake_metadata_schema`, + `fork_ducklake_ancestor_alias`, `fork_data_path`), option stripping, registry. +- `backend/windmill-common/src/materialization.rs` — `list_fork_defer_tables`. +- `backend/windmill-worker/src/duckdb_executor.rs` — `fork_defer_statements` (ancestor + attaches, defer views, target transition), user-arg stripping. +- `backend/windmill-api-assets/src/lib.rs` — `fork_materialization` on the graph. +- `backend/windmill-api-workspaces/src/workspaces_extra.rs` — + `drop_forked_ducklake_namespaces`. +- `backend/windmill-api-workspaces/src/workspaces.rs` — `clone_asset_usages_and_triggers`: + `asset` + `script_trigger` rows are deploy-derived and were never cloned into forks, so a + fork's pipeline graph had no asset nodes/edges and its dispatch cascade never fired (fixed + here because this feature is unusable without it). +- `backend/migrations/20260703170745_fork_ducklake_namespace.*` — registry + GRANTs. +- Frontend: `AssetGraph/{types.ts,AssetNode.svelte,AssetGraphCanvas.svelte, + AssetGraphDetailsPane.svelte}`, `pipeline/[folder]/+page.svelte`, + `sidebar/SidebarContent.svelte` (fork-delete wiring). diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 6352e6adc9..41f5ddc3f5 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -330,6 +330,7 @@ data: { asset_kind: a.kind, path: a.path, + fork_materialization: a.fork_materialization, onAddScript: onAddScriptForAsset, pathPrefix, defaultPathSuffix, diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index c9b921be2e..095169e3b7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -8,9 +8,11 @@ import { formatAssetKind, type AssetWithAltAccessType } from '$lib/components/assets/lib' import { AlertTriangle, + ArrowUpRight, Code2, ExternalLink, GitBranch, + GitFork, Loader2, PanelRightClose, Save, @@ -147,6 +149,11 @@ // `replace` producer). Forwarded to the Schema tab: version history when // true, a single fixed-schema view when false. Defaults to true (unknown). schemaCanEvolve?: boolean + // Fork workspaces: data-environment state of the selected ducklake asset + // ('fork' = materialized here, 'deferred' = reads the parent's current + // data). Pulled from the graph node by the parent page, like + // `schemaCanEvolve`. Undefined outside forks. + selectionForkMaterialization?: 'fork' | 'deferred' // Producer-side facts for the editor's live schema-contract diagnostics // (ignore suppression + scd2 `_current` fallback), built by the page from // the resolved graph and forwarded to ScriptEditor. @@ -259,6 +266,7 @@ selectionProducers = [], selectionColumnGraph, schemaCanEvolve = true, + selectionForkMaterialization = undefined, schemaContractContext = undefined, runsRefreshKey, runsPendingJobId, @@ -1065,6 +1073,31 @@ selected snapshot / tab instead of carrying state across. --> {#key selection.path}
+ {#if selectionForkMaterialization === 'deferred'} + +
+ + + Deferred to parent workspace — reads its current data (not + snapshot-pinned). Materialize this node in the fork to iterate on it. + +
+ {:else if selectionForkMaterialization === 'fork'} +
+ + + Materialized in this fork — reads and writes use the fork's isolated + namespace; the parent workspace's table is untouched. + +
+ {/if} {#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0}
{formatShortAssetPath(asset)} + + {#if data.fork_materialization === 'deferred'} + + + + {:else if data.fork_materialization === 'fork'} + + + + {/if}
{#if showActions} +{#if allDucklakes.current && allDucklakes.current.length > 0} +
` statements. + target_table: String, } // Fetch and validate a custom data-test script's body. v1 custom tests are @@ -148,11 +169,14 @@ fn build_materialized_query( .target_path .split_once('/') .unwrap_or((m.target_path.as_str(), "")); - let meta = MaterializeExec { + let mut meta = MaterializeExec { asset_kind: windmill_common::assets::AssetKind::Ducklake, asset_path: m.target_path.clone(), partition: partition.clone(), n_data_tests: ann.data_tests.len(), + on_schema_change: m.on_schema_change, + // Set below once the strategy is known (managed persist-and-mutate only). + sync_prepass: None, }; // `{partition}` → escaped SQL literal substitution, applied to the managed @@ -242,6 +266,36 @@ fn build_materialized_query( } } + let cg_probe = windmill_parser::sql_materialize::MaterializeCodegen { + target_qualified: "", + select_sql: "", + partition_col: "_wm_partition", + partition_value_sql: "", + partitioned, + strategy: strategy.clone(), + on_schema_change: m.on_schema_change, + }; + // `sync` needs a host round-trip: a pre-pass probe reads the SELECT's and the + // table's columns so the ALTER DDL can be computed, then spliced into the + // plan at the sentinel slot. Build the probe query here (setup + target + // ATTACH + the two column reads) while we still have the substituted SELECT + + // setup; the executor runs it through the same interpolation/ATTACH transform + // as the main query. Only for the positional persist-and-mutate strategies + // (scd2 / whole-table replace emit no sentinel). + if m.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Sync + && cg_probe.is_persist_and_mutate() + { + meta.sync_prepass = Some(SyncPrepass { + probe_query: build_sync_probe_query( + &plan.setup, + &synthetic_attach, + &plan.output, + table, + ), + target_table: table.to_string(), + }); + } + let mat_plan = build_wrap_blocks( &plan, &synthetic_attach, @@ -251,6 +305,7 @@ fn build_materialized_query( &pval, partitioned, strategy, + m.on_schema_change, &resolved, ) .map_err(Error::ExecutionErr)?; @@ -263,6 +318,191 @@ fn build_materialized_query( Ok(Some((Some(blocks.join("\n")), meta))) } +// Build the `on_schema_change=sync` pre-pass probe: the substituted setup + the +// synthetic target ATTACH, then two column reads unioned — the SELECT's columns +// (via `DESCRIBE`) tagged `sel`, and the target table's (via information_schema, +// which yields zero rows when the table doesn't exist yet, so a first +// materialize is a no-drift skip rather than an error) tagged `tbl`. The managed +// `_wm_partition` column is filtered out of the table side. Carries +// `$args`/`{partition}` verbatim; the caller interpolates + ATTACH-transforms it +// like the main query so the DESCRIBE binds run-time args. +fn build_sync_probe_query( + setup: &[String], + synthetic_attach: &str, + select_sql: &str, + table: &str, +) -> String { + let (schema, tname) = match table.split_once('.') { + Some((s, t)) => (Some(s), t), + None => (None, table), + }; + let esc = |s: &str| s.replace('\'', "''"); + let schema_filter = schema + .map(|s| format!(" AND table_schema = '{}'", esc(s))) + .unwrap_or_default(); + let mut out = String::new(); + let mut push_stmt = |s: &str| { + let t = s.trim_end(); + if t.is_empty() { + return; + } + out.push_str(t); + if !t.ends_with(';') { + out.push(';'); + } + out.push('\n'); + }; + for s in setup { + push_stmt(s); + } + push_stmt(synthetic_attach); + // Read the target's columns from `information_schema.columns` scoped to the + // attached target catalog via `table_catalog` (DuckDB's catalog-qualified + // `.information_schema` does NOT exist; the unqualified view spans + // attached catalogs and distinguishes them by `table_catalog`). It yields + // zero rows for a not-yet-created table — a no-drift skip, not an error + // (unlike `pragma_table_info`/`DESCRIBE`). + out.push_str(&format!( + "SELECT 'sel' AS _wm_which, column_name AS _wm_name, column_type AS _wm_type \ + FROM (DESCRIBE SELECT * FROM ({select_sql})) \ + UNION ALL \ + SELECT 'tbl' AS _wm_which, column_name AS _wm_name, data_type AS _wm_type \ + FROM information_schema.columns \ + WHERE table_catalog = '{alias}' AND table_name = '{tname}'{schema_filter} \ + AND column_name <> '_wm_partition';", + alias = windmill_parser::sql_materialize::TARGET_ALIAS, + tname = esc(tname), + )); + out +} + +// Run the sync pre-pass probe (already interpolated + ATTACH-transformed by the +// caller) and turn its column diff into the `ALTER TABLE … ADD/DROP COLUMN` DDL +// to splice at the sentinel. Empty string when the table is fresh (no rows) or +// the SELECT already matches. The probe failing is fatal (no silent fallback). +async fn compute_sync_alter_ddl( + probe_blocks: Vec, + job_args: Vec, + target_table: String, + token: String, + base_internal_url: String, + w_id: String, + job_dir: String, +) -> Result { + let n = probe_blocks.len(); + let (result, _) = tokio::task::spawn_blocking(move || { + run_duckdb_ffi_safe( + probe_blocks.iter().map(String::as_str), + n, + job_args, + &token, + &base_internal_url, + &w_id, + &job_dir, + SqlResultCollectionStrategy::LastStatementAllRows, + ) + }) + .await + .map_err(|e| Error::from(to_anyhow(e))) + .and_then(|r| r) + .map_err(|e| { + Error::ExecutionErr(format!( + "on_schema_change=sync: the schema pre-pass probe failed ({e}); refusing to write" + )) + })?; + + let (added, removed) = parse_sync_drift(&result)?; + let tq = format!( + "{}.{}", + windmill_parser::sql_materialize::TARGET_ALIAS, + quote_qualified_table(&target_table) + ); + let mut stmts = Vec::new(); + for (name, typ) in &added { + if typ.is_empty() { + return Err(Error::ExecutionErr(format!( + "on_schema_change=sync: could not resolve a type for new column `{name}`" + ))); + } + // `typ` is DuckDB's own DESCRIBE type name (e.g. BIGINT, VARCHAR, + // DECIMAL(10,2)) — a valid type expression, injected verbatim. + stmts.push(format!( + "ALTER TABLE {tq} ADD COLUMN \"{}\" {typ};", + name.replace('"', "\"\"") + )); + } + for name in &removed { + stmts.push(format!( + "ALTER TABLE {tq} DROP COLUMN \"{}\";", + name.replace('"', "\"\"") + )); + } + Ok(stmts.join("\n")) +} + +// Parse the sync probe result rows (`_wm_which`/`_wm_name`/`_wm_type`) into the +// column diff: `added` = SELECT columns (with their types) absent from the +// table, `removed` = table columns absent from the SELECT. `_wm_partition` and +// empty names are ignored. An empty table side means the table doesn't exist +// yet — a no-op first materialize, so both lists are empty. +fn parse_sync_drift(result: &RawValue) -> Result<(Vec<(String, String)>, Vec)> { + let root: Value = serde_json::from_str(result.get()).map_err(|e| { + Error::ExecutionErr(format!("on_schema_change=sync: bad probe result json: {e}")) + })?; + // The FFI may surface the rows as an array, a single object, or a JSON + // string — normalize to a list of row objects. + let owned; + let rows: Vec<&Value> = match &root { + Value::Array(a) => a.iter().collect(), + Value::Object(_) => vec![&root], + Value::String(s) => { + owned = serde_json::from_str::(s).unwrap_or(Value::Null); + match &owned { + Value::Array(a) => a.iter().collect(), + Value::Object(_) => vec![&owned], + _ => vec![], + } + } + _ => vec![], + }; + let mut sel: Vec<(String, String)> = Vec::new(); + let mut tbl: std::collections::HashSet = std::collections::HashSet::new(); + for row in rows { + let Some(o) = row.as_object() else { continue }; + let name = o + .get("_wm_name") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + if name.is_empty() || name == "_wm_partition" { + continue; + } + let typ = o + .get("_wm_type") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + match o.get("_wm_which").and_then(|v| v.as_str()).unwrap_or("") { + "sel" => sel.push((name, typ)), + "tbl" => { + tbl.insert(name); + } + _ => {} + } + } + if tbl.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + let sel_names: std::collections::HashSet<&String> = sel.iter().map(|(n, _)| n).collect(); + let added = sel + .iter() + .filter(|(n, _)| !tbl.contains(n)) + .cloned() + .collect(); + let removed = tbl.into_iter().filter(|n| !sel_names.contains(n)).collect(); + Ok((added, removed)) +} + // Fetch the deployed body of every `// data_test ` custom test declared in // `query`, keyed by path, so the sync `build_materialized_query` can splice them // in. The DB read is the only part of materialize codegen that needs a @@ -878,6 +1118,50 @@ fn extract_schema( } } +// Pull the `on_schema_change=warn` drift out of the materialize summary's +// `schema_drift` column: a `{added: [..], removed: [..]}` struct (or NULL when +// the SELECT matched the table). Like `output_schema`, the FFI may surface it +// nested or as a JSON string — accept both. `None` when there is no drift. +fn extract_schema_drift(result: &RawValue) -> Option<(Vec, Vec)> { + fn str_list(v: Option<&Value>) -> Vec { + match v { + Some(Value::Array(a)) => a + .iter() + .filter_map(|x| x.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + } + } + fn find_field(v: &Value) -> Option<&Value> { + match v { + Value::Object(o) => o.get("schema_drift"), + Value::Array(a) => a.iter().find_map(find_field), + _ => None, + } + } + let root = serde_json::from_str::(result.get()).ok()?; + let field = find_field(&root)?; + let owned; + let obj = match field { + Value::Object(_) => field, + // FFI serialized the struct as a JSON string — parse it. + Value::String(s) => { + owned = serde_json::from_str::(s).ok()?; + &owned + } + // NULL / absent ⇒ no drift. + _ => return None, + }; + let o = obj.as_object()?; + let added = str_list(o.get("added")); + let removed = str_list(o.get("removed")); + if added.is_empty() && removed.is_empty() { + None + } else { + Some((added, removed)) + } +} + // Render the full pass/fail breakdown for a failed data-test run — every test, // not just the first failure, so the user sees the whole picture in one place. fn format_data_test_breakdown(asset_path: &str, tests: &[DataTestOutcome]) -> String { @@ -1092,7 +1376,27 @@ pub async fn do_duckdb( let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; - let query = transform_s3_uris(query).await?; + let mut query = transform_s3_uris(query).await?; + + // `on_schema_change=sync`: interpolate the pre-pass probe with the same + // sig / args / reserved vars as the main query (before `sig` is consumed + // and the arg map drained just below) so its `DESCRIBE` of the SELECT + // binds `$args`/`$partition` the same way. + let sync_probe_interpolated = match materialize + .as_ref() + .and_then(|(_, m)| m.sync_prepass.as_ref()) + { + Some(pp) => { + let (interp, _) = sanitize_and_interpolate_unsafe_sql_args( + &pp.probe_query, + &sig, + &job_args, + &reserved_variables, + )?; + Some(transform_s3_uris(&interp).await?) + } + None => None, + }; let job_args = { let mut m = Vec::new(); @@ -1127,6 +1431,96 @@ pub async fn do_duckdb( m }; + // `on_schema_change=sync` pre-pass: run the probe (through the same + // ATTACH transform as the main query), diff the SELECT's columns against + // the table's, and splice the `ALTER TABLE … ADD/DROP COLUMN` DDL into + // the plan at the sentinel slot (removing the sentinel when there is no + // drift). Runs before the main query so the ALTERs are in place when the + // BY NAME insert executes. + if let (Some(probe_sql), Some(pp)) = ( + &sync_probe_interpolated, + materialize + .as_ref() + .and_then(|(_, m)| m.sync_prepass.as_ref()), + ) { + let mut probe_blocks = vec![]; + // Held for the whole probe (through compute_sync_alter_ddl): a + // `TYPE bigquery` ATTACH needs GOOGLE_APPLICATION_CREDENTIALS + the + // temp creds file set before DuckDB binds the extension, exactly as + // the main query path does — otherwise the probe DESCRIBE fails. + let mut probe_bigquery_credentials = None; + for query_block in parse_sql_blocks(probe_sql, true).iter() { + let query_block = remove_comments(query_block); + if let Some(parsed) = parse_attach_db_resource(query_block) { + probe_blocks.extend( + transform_attach_db_resource_query( + &parsed, + &job.id, + client, + &mut hidden_passwords, + ) + .await?, + ); + if parsed.db_type == "bigquery" { + probe_bigquery_credentials = Some(UseBigQueryCredentialsFile::new( + job.id, + parsed.resource_path, + )?); + } + } else if let Some(q) = transform_attach_ducklake( + &query_block, + conn, + &mut hidden_passwords, + &job.workspace_id, + materialize.as_ref().map(|(_, m)| m.asset_path.as_str()), + ) + .await? + { + probe_blocks.extend(q); + } else if let Some(q) = transform_attach_datatable( + &query_block, + conn, + &mut hidden_passwords, + &job.workspace_id, + ) + .await? + { + probe_blocks.extend(q); + } else { + probe_blocks.push(query_block.to_string()); + } + } + // The probe `DESCRIBE`s the same SELECT as the main query, so it + // needs the same workspace-macro / `// use` library definitions + // injected (post-ATTACH), or a macro-calling SELECT fails to bind + // here and the fatal probe makes `sync` unusable with macros. + let probe_blocks = inject_workspace_macros( + conn, + &job.workspace_id, + macro_ann.macros, + ¯o_ann.use_libs, + probe_blocks, + ) + .await?; + let alter_ddl = compute_sync_alter_ddl( + probe_blocks, + job_args.clone(), + pp.target_table.clone(), + token.clone(), + client.base_internal_url.clone(), + job.workspace_id.clone(), + job_dir.to_string(), + ) + .await?; + query = query.replace( + windmill_parser::sql_materialize::SYNC_ALTER_SENTINEL, + &alter_ddl, + ); + // Kept alive until the probe finished; the main query path recreates + // its own credentials for its own ATTACH pass below. + drop(probe_bigquery_credentials); + } + let query_block_list = parse_sql_blocks(&query, true); // Replace custom ATTACH statements with the real instructions @@ -1283,6 +1677,37 @@ pub async fn do_duckdb( } else { None }; + // `on_schema_change=warn` (default): the summary carries a + // `schema_drift` column when the SELECT's columns diverged from the + // fixed table schema. The write already happened positionally (data + // may have landed in the wrong/old columns) — log it LOUDLY so the + // drift isn't silent. The drift also rides back in the job result + // (it's a column of the returned summary row). + if is_managed + && meta.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Warn + { + if let Some((added, removed)) = extract_schema_drift(&result) { + let fmt = |cols: &[String]| { + if cols.is_empty() { + "(none)".to_string() + } else { + cols.join(", ") + } + }; + let warning = format!( + "\n⚠️ SCHEMA DRIFT on {asset}: the SELECT's columns no longer match the \ + existing table.\n added (in SELECT, not in table): {added}\n removed \ + (in table, not in SELECT): {removed}\n The write proceeded POSITIONALLY \ + against the existing table schema — data may have landed in the wrong \ + columns. Set `on_schema_change=fail` to block this, or \ + `on_schema_change=sync` to migrate the table automatically.\n", + asset = meta.asset_path, + added = fmt(&added), + removed = fmt(&removed), + ); + append_logs(&job.id, &job.workspace_id, warning, conn).await; + } + } // Defense-in-depth: codegen embedded `n_data_tests` checks, so the // summary row must carry that many outcomes. Recovering fewer means // the `data_tests` column was dropped/reshaped before we read it — @@ -2984,6 +3409,134 @@ mod tests { ); } + // on_schema_change=fail on a persist-and-mutate strategy (merge) emits the + // drift guard inside the write, and no sync artifacts. + #[test] + fn materialize_fail_emits_drift_guard() { + let script = "-- materialize ducklake://main/dim key=id on_schema_change=fail\n\ + SELECT id, name FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("CAST(error(") && rewritten.contains("on_schema_change=fail"), + "fail emits the drift guard: {rewritten}" + ); + assert!(!rewritten.contains("BY NAME"), "fail writes positionally"); + assert!(meta.sync_prepass.is_none(), "fail needs no pre-pass probe"); + } + + // on_schema_change=sync writes BY NAME, emits the ALTER sentinel, and builds + // the pre-pass probe (DESCRIBE of the SELECT + information_schema read). + #[test] + fn materialize_sync_by_name_and_prepass() { + let script = "-- materialize ducklake://main/dim key=id on_schema_change=sync\n\ + SELECT id, name FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("INSERT INTO _wm_target.dim BY NAME"), + "sync insert is name-mapped: {rewritten}" + ); + assert!( + rewritten.contains(windmill_parser::sql_materialize::SYNC_ALTER_SENTINEL), + "sync emits the ALTER injection sentinel" + ); + let pp = meta.sync_prepass.expect("sync builds a pre-pass probe"); + assert_eq!(pp.target_table, "dim"); + assert!(pp + .probe_query + .contains("DESCRIBE SELECT * FROM (SELECT id, name FROM dl.src)")); + assert!(pp.probe_query.contains("FROM information_schema.columns")); + assert!(pp + .probe_query + .contains("table_catalog = '_wm_target' AND table_name = 'dim'")); + assert!(pp + .probe_query + .contains("ATTACH 'ducklake://main' AS _wm_target;")); + } + + // on_schema_change=warn (default) on a persist-and-mutate strategy folds the + // drift into the summary; no guard, no sync artifacts. + #[test] + fn materialize_warn_summary_carries_schema_drift() { + let script = "-- materialize ducklake://main/dim key=id\n\ + SELECT id, name FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("AS schema_drift"), + "warn folds drift into the summary: {rewritten}" + ); + assert!(!rewritten.contains("CAST(error(")); + assert!(!rewritten.contains("BY NAME")); + assert!(meta.sync_prepass.is_none()); + } + + // Whole-table replace (unpartitioned, no key/append) is not persist-and- + // mutate: even with on_schema_change=fail there is no guard / sentinel — the + // CREATE OR REPLACE already rebuilds the schema each run. + #[test] + fn materialize_whole_table_replace_ignores_guardrail() { + let script = "-- materialize ducklake://main/t on_schema_change=fail\n\ + SELECT a, b FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!(rewritten.contains("CREATE OR REPLACE TABLE _wm_target.t")); + assert!(!rewritten.contains("on_schema_change=fail")); + assert!(!rewritten.contains(windmill_parser::sql_materialize::SYNC_ALTER_SENTINEL)); + assert!(meta.sync_prepass.is_none()); + } + + #[test] + fn extract_schema_drift_parses_struct_and_string_and_null() { + // nested struct form + let r = raw( + r#"{"materialized":"ducklake://main/dim","schema_drift":{"added":["c"],"removed":["b"]}}"#, + ); + let (added, removed) = extract_schema_drift(&r).expect("drift present"); + assert_eq!(added, vec!["c".to_string()]); + assert_eq!(removed, vec!["b".to_string()]); + // FFI string-encoded form + let r = raw(r#"{"schema_drift":"{\"added\":[\"x\"],\"removed\":[]}"}"#); + let (added, removed) = extract_schema_drift(&r).expect("drift present"); + assert_eq!(added, vec!["x".to_string()]); + assert!(removed.is_empty()); + // NULL / both-empty ⇒ no drift + assert!(extract_schema_drift(&raw(r#"{"schema_drift":null}"#)).is_none()); + assert!( + extract_schema_drift(&raw(r#"{"schema_drift":{"added":[],"removed":[]}}"#)).is_none() + ); + assert!(extract_schema_drift(&raw(r#"{"rows":3}"#)).is_none()); + } + + #[test] + fn parse_sync_drift_computes_added_removed_and_skips_fresh_table() { + // table (a, b) vs SELECT (a, c): add c, drop b; _wm_partition ignored + let r = raw(r#"[{"_wm_which":"sel","_wm_name":"a","_wm_type":"BIGINT"}, + {"_wm_which":"sel","_wm_name":"c","_wm_type":"VARCHAR"}, + {"_wm_which":"tbl","_wm_name":"a","_wm_type":"BIGINT"}, + {"_wm_which":"tbl","_wm_name":"b","_wm_type":"BIGINT"}]"#); + let (added, removed) = parse_sync_drift(&r).unwrap(); + assert_eq!(added, vec![("c".to_string(), "VARCHAR".to_string())]); + assert_eq!(removed, vec!["b".to_string()]); + // no `tbl` rows ⇒ table doesn't exist yet ⇒ no migration + let r = raw(r#"[{"_wm_which":"sel","_wm_name":"a","_wm_type":"BIGINT"}]"#); + let (added, removed) = parse_sync_drift(&r).unwrap(); + assert!(added.is_empty() && removed.is_empty()); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { From 5769b6036cf14b0cb424c5b3d9d878c600a5652e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 17:10:06 +0200 Subject: [PATCH 196/273] fix(pipelines): activity-axis label clarity + select failed node on cascade failure (#9931) * fix(pipelines): clarify activity-window axis label + select failed node on cascade failure Co-Authored-By: Claude Fable 5 * fix(pipelines): clear active draft so failed-node focus is not masked PipelineGraphEditor gives an open draft (activeDraftPath) priority over selection via openScriptPath, so the cascade-failure focus set only selection and stayed masked while a draft pane was open. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Fable 5 --- .../assets/AssetGraph/ActivityHistogram.svelte | 5 +++-- .../(logged)/pipeline/[folder]/+page.svelte | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte b/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte index fcfd95a5d3..cd9548de59 100644 --- a/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte @@ -147,9 +147,10 @@ > {/if} - +
- {fmtTime(from)} + from {fmtTime(from)} now
diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 0889d53a01..167aaa8f25 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -1480,6 +1480,14 @@ } else { const failed = [...res.statuses.entries()].filter(([, s]) => s.status === 'failure') const skipped = [...res.statuses.values()].filter((s) => s.status === 'skipped').length + // Surface the failed node's error in the details pane. Clear the + // active draft too — it has pane priority over `selection` + // (openScriptPath), so a bare selection would stay masked while a + // draft is open. + if (failed.length > 0) { + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: failed[0][0] } + } sendUserToast( `Chain run failed at ${failed.map(([p]) => p).join(', ')}` + (skipped > 0 ? ` — ${skipped} downstream skipped` : ''), @@ -1616,6 +1624,14 @@ } else { const failed = [...res.statuses.entries()].filter(([, s]) => s.status === 'failure') const skipped = [...res.statuses.values()].filter((s) => s.status === 'skipped').length + // Surface the failed node's error in the details pane. Clear the + // active draft too — it has pane priority over `selection` + // (openScriptPath), so a bare selection would stay masked while a + // draft is open. + if (failed.length > 0) { + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: failed[0][0] } + } sendUserToast( `Bounded run failed at ${failed.map(([p]) => p).join(', ')}` + (skipped > 0 ? ` — ${skipped} downstream skipped` : ''), From 182b10b2ad084cce28be3f25fdded44b9bf51f9e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 18:01:20 +0200 Subject: [PATCH 197/273] render FFI errors with real newlines and no stray quoting (#9932) * fix(duckdb): render FFI errors with real newlines and no stray quoting The DuckDB FFI returns errors as ERROR , so the executor was surfacing the serde_json-escaped form (wrapping quotes, literal \\n). Multi-line errors like the write-audit-publish data-test breakdown were unreadable. Decode the JSON string back to the raw message at both FFI error sites. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: tighten decode_ffi_error comment to the invariant Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-worker/src/duckdb_executor.rs | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index d7436c579c..e51c59cb98 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2004,6 +2004,15 @@ fn cgroup_bytes_to_duckdb_memory_limit(bytes: i64) -> Option { } // Read backend/windmill-duckdb-ffi-internal/README_DEV.md for details about why we use FFI +// The FFI returns errors as `ERROR ` (the message is +// serde_json::to_string'd). Decode the JSON string back so multi-line errors +// render with real newlines, not escaped `\n` inside wrapping quotes; fall back +// to the raw slice if it is somehow not a JSON string. +fn decode_ffi_error(result_str: &str) -> String { + let raw = result_str.strip_prefix("ERROR ").unwrap_or(result_str); + serde_json::from_str::(raw).unwrap_or_else(|_| raw.to_string()) +} + fn run_duckdb_ffi_safe<'a>( query_block_list: impl Iterator, query_block_list_count: usize, @@ -2069,7 +2078,7 @@ fn run_duckdb_ffi_safe<'a>( }; if result_str.starts_with("ERROR") { - Err(Error::ExecutionErr(result_str[6..].to_string())) + Err(Error::ExecutionErr(decode_ffi_error(&result_str))) } else { let result = if collection_strategy == SqlResultCollectionStrategy::AllStatementsAllRows { // Avoid parsing JSON @@ -2133,7 +2142,7 @@ fn prepare_duckdb_ffi_safe<'a>( }; if result_str.starts_with("ERROR") { - Err(Error::ExecutionErr(result_str[6..].to_string())) + Err(Error::ExecutionErr(decode_ffi_error(&result_str))) } else { Ok(serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?) } @@ -2629,6 +2638,31 @@ pub struct Arg { mod tests { use super::*; + #[test] + fn decode_ffi_error_unescapes_multiline_and_strips_quotes() { + // Mirror the FFI: JSON-encode the raw DuckDB message, prefix "ERROR ". + let raw_msg = "Invalid Input Error: data tests failed on main/raw_orders \ + (1/3 failed) — write rolled back, previous version left live:\n \ + ✓ not_null(order_id)\n ✗ accepted_values(status) — 12 violating row(s)"; + let ffi = format!("ERROR {}", serde_json::to_string(raw_msg).unwrap()); + let decoded = decode_ffi_error(&ffi); + assert_eq!(decoded, raw_msg); + // real newlines, no literal `\n`, no wrapping quotes + assert!(decoded.contains('\n') && !decoded.contains("\\n")); + assert!(!decoded.starts_with('"')); + // single-line error with inner quotes round-trips unescaped + let single = format!( + "ERROR {}", + serde_json::to_string("Binder Error: column \"x\" not found").unwrap() + ); + assert_eq!( + decode_ffi_error(&single), + "Binder Error: column \"x\" not found" + ); + // non-JSON payload falls back to the raw slice + assert_eq!(decode_ffi_error("ERROR not json"), "not json"); + } + fn test_ancestor(wid: &str) -> windmill_common::workspaces::DucklakeAncestorAttach { windmill_common::workspaces::DucklakeAncestorAttach { workspace_id: wid.to_string(), From 799b9e3b7c1998c1264c17cafba858bf192b8cb8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 18:23:51 +0200 Subject: [PATCH 198/273] chore(main): release 1.748.0 (#9914) * chore(main): release 1.748.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 23 ++ backend/Cargo.lock | 211 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 174 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5342e3024..ec10dc8990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [1.748.0](https://github.com/windmill-labs/windmill/compare/v1.747.0...v1.748.0) (2026-07-05) + + +### Features + +* **ai-agent:** support reasoning effort in AI agent workflow steps ([#9886](https://github.com/windmill-labs/windmill/issues/9886)) ([a368d49](https://github.com/windmill-labs/windmill/commit/a368d49bd8786a2dca6771f2051f1d44d1b2363d)) +* **ducklake:** scheduled lake maintenance (expiry, compaction, orphan cleanup) ([#9916](https://github.com/windmill-labs/windmill/issues/9916)) ([3352150](https://github.com/windmill-labs/windmill/commit/33521505dbc34f22b575d21fda1cc76d698a8840)) +* **pipelines:** asset freshness — fresh/stale badge (CE) + watchdog (EE) ([#9909](https://github.com/windmill-labs/windmill/issues/9909)) ([5d7fb6d](https://github.com/windmill-labs/windmill/commit/5d7fb6deca3e02e89d77e5d3856483beb8b8bfeb)) +* **pipelines:** capture violating-row samples for data tests ([#9919](https://github.com/windmill-labs/windmill/issues/9919)) ([d4b4374](https://github.com/windmill-labs/windmill/commit/d4b4374de8f8a7875b050c16d1236fcd0355812b)) +* **pipelines:** fork data environments for ducklake materialization (dev data) ([#9915](https://github.com/windmill-labs/windmill/issues/9915)) ([39eb9de](https://github.com/windmill-labs/windmill/commit/39eb9de1bce400109c130a081807e40e995ae068)) +* **pipelines:** on_schema_change write guardrails + data_test deploy validation ([#9930](https://github.com/windmill-labs/windmill/issues/9930)) ([377c02e](https://github.com/windmill-labs/windmill/commit/377c02ec47389e64b7ef5cbbae0de05df648266d)) +* **pipelines:** record upstream snapshot ids on cascade-dispatched jobs ([#9910](https://github.com/windmill-labs/windmill/issues/9910)) ([af36498](https://github.com/windmill-labs/windmill/commit/af36498432e643108308e1c03b5d986d0f0f8888)) +* **pipelines:** schema contracts — save-time consumer checks vs captured schemas ([#9917](https://github.com/windmill-labs/windmill/issues/9917)) ([42e11c6](https://github.com/windmill-labs/windmill/commit/42e11c6570b62ffaa86598438fa8ddf462c4035f)) +* **pipeline:** write-audit-publish for materialization data tests ([#9911](https://github.com/windmill-labs/windmill/issues/9911)) ([dce247c](https://github.com/windmill-labs/windmill/commit/dce247c6d2678a2c95bd728027e17ae3965638e2)) +* **sdk:** enforce s3:// URIs for string S3 params + ingestion (EL) docs ([#9912](https://github.com/windmill-labs/windmill/issues/9912)) ([5ad2de9](https://github.com/windmill-labs/windmill/commit/5ad2de91a26b312bf27124ceca16ef331621bde8)) + + +### Bug Fixes + +* **cli:** pipeline + workspace UX batch (init/bind stub, run errors, macro libs, lock-job report, upgrade errors) ([#9929](https://github.com/windmill-labs/windmill/issues/9929)) ([28a6b08](https://github.com/windmill-labs/windmill/commit/28a6b086c842105298f236baa0a61868f71a5eb1)) +* **cli:** publish all windmill-parser-wasm-* deps so local pipeline graph keeps write edges ([#9926](https://github.com/windmill-labs/windmill/issues/9926)) ([744a759](https://github.com/windmill-labs/windmill/commit/744a7597edaf3ca9a7fd2b21a34fb33457913a64)) +* **pipelines:** activity-axis label clarity + select failed node on cascade failure ([#9931](https://github.com/windmill-labs/windmill/issues/9931)) ([5769b60](https://github.com/windmill-labs/windmill/commit/5769b6036cf14b0cb424c5b3d9d878c600a5652e)) + ## [1.747.0](https://github.com/windmill-labs/windmill/compare/v1.746.0...v1.747.0) (2026-07-03) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1153916767..cd780be48c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2057,9 +2057,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "jobserver", @@ -5072,9 +5072,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -7563,9 +7565,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -8361,9 +8363,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" dependencies = [ "memchr", "ucd-trie", @@ -8371,9 +8373,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" dependencies = [ "pest", "pest_generator", @@ -8381,9 +8383,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" dependencies = [ "pest", "pest_meta", @@ -8394,12 +8396,11 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" dependencies = [ "pest", - "sha2 0.10.9", ] [[package]] @@ -8995,15 +8996,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.0", + "rand 0.10.2", + "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", "rustls 0.23.35", @@ -9017,16 +9019,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9184,6 +9186,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -13322,9 +13333,9 @@ checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" [[package]] name = "utf8-width" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" +checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a" [[package]] name = "utf8_iter" @@ -13735,7 +13746,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-nats", @@ -13817,7 +13828,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.747.0" +version = "1.748.0" dependencies = [ "async-stream", "async-trait", @@ -13850,7 +13861,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13863,7 +13874,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "argon2", @@ -14001,7 +14012,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14024,7 +14035,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14039,7 +14050,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14065,7 +14076,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.747.0" +version = "1.748.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14075,7 +14086,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14092,7 +14103,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14114,7 +14125,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14137,7 +14148,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14153,7 +14164,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14174,7 +14185,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14195,7 +14206,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14209,7 +14220,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-nats", @@ -14244,7 +14255,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14269,7 +14280,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14287,7 +14298,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14309,7 +14320,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14329,7 +14340,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14366,7 +14377,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14394,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.747.0" +version = "1.748.0" dependencies = [ "lazy_static", "serde", @@ -14406,7 +14417,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.747.0" +version = "1.748.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14431,7 +14442,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14445,7 +14456,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14479,7 +14490,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.747.0" +version = "1.748.0" dependencies = [ "chrono", "lazy_static", @@ -14493,7 +14504,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14512,7 +14523,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.747.0" +version = "1.748.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14614,7 +14625,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.747.0" +version = "1.748.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14633,7 +14644,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.747.0" +version = "1.748.0" dependencies = [ "regex", "serde", @@ -14648,7 +14659,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14672,7 +14683,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "futures", @@ -14689,7 +14700,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.747.0" +version = "1.748.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14705,7 +14716,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -14726,7 +14737,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -14757,7 +14768,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "arc-swap", @@ -14782,7 +14793,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-stream", @@ -14816,7 +14827,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "futures", @@ -14834,7 +14845,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.747.0" +version = "1.748.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14843,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -14855,7 +14866,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -14867,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "gosyn", @@ -14879,7 +14890,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -14891,7 +14902,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -14903,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "nu-parser", @@ -14914,7 +14925,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14925,7 +14936,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14937,7 +14948,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14948,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -14970,7 +14981,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -14982,7 +14993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -14996,7 +15007,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15013,7 +15024,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -15026,7 +15037,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -15038,7 +15049,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -15056,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15072,7 +15083,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15088,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -15099,7 +15110,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -15138,7 +15149,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "const_format", @@ -15177,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.747.0" +version = "1.748.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15188,7 +15199,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -15222,7 +15233,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15246,7 +15257,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15279,7 +15290,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15312,7 +15323,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15332,7 +15343,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15366,7 +15377,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15402,7 +15413,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15425,7 +15436,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15449,7 +15460,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-nats", @@ -15473,7 +15484,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15508,7 +15519,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15536,7 +15547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15561,7 +15572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15580,7 +15591,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-once-cell", @@ -15690,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.747.0" +version = "1.748.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a91518dc38..e80fbc89db 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.747.0" +version = "1.748.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.747.0" +version = "1.748.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 815d62644f..a9caa82ede 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.747.0" +version = "1.748.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.747.0" +version = "1.748.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.747.0" +version = "1.748.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 9111ca8ccb..521fb7a132 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.747.0" +version = "1.748.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0c080b0a2a..afe8cbc0b6 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.747.0 + version: 1.748.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 4b342d93bd..37132a3b9e 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.747.0"; +export const VERSION = "v1.748.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 9e2110814d..f01ea9c68b 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.747.0"; +export const VERSION = "1.748.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3b9c1b8134..f0fd64f6fc 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.747.0", + "version": "1.748.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.747.0", + "version": "1.748.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 7f0ed5e1d9..adfd48cad4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.747.0", + "version": "1.748.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 482c95db43..d2b78d3e8f 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.747.0" +wmill = ">=1.748.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index b0508b2719..32f13a4f0c 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.747.0 + version: 1.748.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 0b3006323a..f99fb3832d 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.747.0' + ModuleVersion = '1.748.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index dda29ba2b4..3219756afa 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.747.0" +version = "1.748.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index afcb96626d..191bab2bf6 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.747.0", + "version": "1.748.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index d8058153be..02a1bbb060 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.747.0", + "version": "1.748.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 8eff60cc0e..a16d7ab553 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.747.0 +1.748.0 From 22452ce54034a9bea8f7d48946818fd148b938c0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:20:55 +0200 Subject: [PATCH 199/273] fix(object-storage): remove 20-file bucket-browser listing cap in CE (#9935) * fix(object-storage): remove 20-file bucket-browser cap in CE The Community Edition build rejected the object-storage `list_stored_files` endpoint with an error once a workspace bucket held more than 20 objects, making the bucket browser unusable on larger buckets. The listing already collects up to `max_keys` objects, so the hard cap was purely a gate. Drops the CE listing cap (in the EE-symlinked `job_helpers_ee.rs`, tracked in the companion windmill-ee-private PR) and removes the now-inaccurate sentence from the workspace object-storage settings banner. The 10 GiB total-storage write quota remains as the intentional CE limit. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 4bd7b73dcef1b77dc2866bc58e0c86962559495c This commit updates the EE repository reference after PR #648 was merged in windmill-ee-private. Previous ee-repo-ref: aa14d0724216030948c2f575bcc19c0e6e0476a7 New ee-repo-ref: 4bd7b73dcef1b77dc2866bc58e0c86962559495c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../lib/components/workspaceSettings/StorageSettings.svelte | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c3ae39c767..cc9c36f5a5 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -fa7ac11c1e0ab39e84a0c18973ba427a240933ca +4bd7b73dcef1b77dc2866bc58e0c86962559495c diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index be49e3c564..1e6c990819 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -190,8 +190,7 @@ {#if !$enterpriseLicense} Total workspace storage is capped at {quotaDisplay} in the Community Edition: writes that would exceed - the quota are rejected. The bucket browser will also not work for buckets containing more than 20 - files. Consider upgrading to Windmill EE for unlimited workspace storage. + the quota are rejected. Consider upgrading to Windmill EE for unlimited workspace storage. {:else} From 52ce805f619747af4f998cde7819a164c754205a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:24:41 +0200 Subject: [PATCH 200/273] fix(pipelines): dedup guard for keyed merge + deploy-time SCD2 validation (#9936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two correctness/validation improvements to managed materialization: 1. A keyed `merge` (`key=`) is delete-by-key + insert-all and does NOT deduplicate its source, so two incoming rows sharing a key both landed under that key — silently breaking the one-row-per-key contract. Codegen now emits an in-transaction guard (same `error(...)` shape as the schema -drift guard) that fails the run when the SELECT returns more than one row for a non-NULL key, naming the key. Authors deduplicate in the SELECT or switch to `append`. NULL keys are exempt, matching the delete's `IN (...)` scope. 2. The two SCD2 misconfigurations that were only caught at run time — `history` without `key=`, and `history` + `// partitioned` — now fail fast at deploy via a shared `MaterializeSpec::validate`, called from `create_script_internal`. The DuckDB executor keeps the same check as a safety net for preview/test runs that never deploy (shared message, no drift). Adds unit tests for the merge guard codegen and for `validate` (all four cases), and updates docs/ducklake-materialization.md and docs/pipelines-vs-dbt.md. Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-parser/src/asset_parser.rs | 79 +++++++++++++++++++ .../windmill-parser/src/sql_materialize.rs | 64 +++++++++++++++ backend/windmill-api-scripts/src/scripts.rs | 7 ++ .../windmill-worker/src/duckdb_executor.rs | 18 +++-- docs/ducklake-materialization.md | 19 ++++- docs/pipelines-vs-dbt.md | 11 ++- 6 files changed, 185 insertions(+), 13 deletions(-) diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 4ef30246a6..d95234a297 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -333,6 +333,40 @@ impl OnSchemaChange { } } +impl MaterializeSpec { + /// Deploy-time validation of the option combination against the script's + /// partitioning, returning a human-facing error for combinations the + /// runtime cannot honor. Called at save (`create_script_internal`) so a + /// misconfigured script is rejected up front, and again in the DuckDB + /// executor as a safety net for preview/test runs that never deploy. Both + /// checks are SCD2-specific and inert for `manual` mode (which owns its DDL + /// and ignores the reconciliation strategy). `partitioned` is whether the + /// script declares `// partitioned`. + pub fn validate(&self, partitioned: bool) -> Result<(), String> { + if self.manual || !self.scd2 { + return Ok(()); + } + // SCD2 needs a natural key to identify an entity across versions. + if self.unique_key.as_deref().map_or(true, str::is_empty) { + return Err( + "materialize scd2: requires a natural key — add `key=` (e.g. \ + `// materialize ducklake:///
key=id history`)" + .to_string(), + ); + } + // SCD2's diff/close/open shape has no partition-scoped form in v1. + if partitioned { + return Err( + "materialize scd2: `// partitioned` is not supported with scd2 in v1 — remove \ + `// partitioned`, or drop `history`/`scd2` to materialize the partition without \ + history" + .to_string(), + ); + } + Ok(()) + } +} + // `// data_test …` — a data-quality assertion run against the // freshly-materialized asset (post DELETE+INSERT), failing the run on // violation. The first extensible annotation family: the parser turns a @@ -1721,6 +1755,51 @@ mod pipeline_annotation_tests { assert!(!out.materialize.expect("materialize").close_deleted); } + #[test] + fn materialize_validate_scd2_requires_key() { + // scd2 without `key=` is rejected at deploy (was a run-time error). + let m = parse_pipeline_annotations("// materialize ducklake://a/dim history") + .materialize + .expect("materialize"); + assert!(m.scd2 && m.unique_key.is_none()); + let err = m.validate(false).expect_err("scd2 without key must fail"); + assert!(err.contains("requires a natural key")); + // with a key it validates + let m = parse_pipeline_annotations("// materialize ducklake://a/dim key=id history") + .materialize + .expect("materialize"); + assert!(m.validate(false).is_ok()); + } + + #[test] + fn materialize_validate_scd2_rejects_partitioned() { + // scd2 + `// partitioned` has no v1 form — rejected at deploy. + let m = parse_pipeline_annotations("// materialize ducklake://a/dim key=id history") + .materialize + .expect("materialize"); + let err = m.validate(true).expect_err("scd2 + partitioned must fail"); + assert!(err.contains("`// partitioned` is not supported with scd2")); + // unpartitioned scd2 is fine + assert!(m.validate(false).is_ok()); + } + + #[test] + fn materialize_validate_non_scd2_and_manual_are_inert() { + // Non-scd2 strategies are unconstrained by these checks, partitioned or not. + let m = parse_pipeline_annotations("// materialize ducklake://a/orders key=id") + .materialize + .expect("materialize"); + assert!(m.validate(true).is_ok()); + assert!(m.validate(false).is_ok()); + // `manual` owns its DDL and ignores the strategy — never rejected here, + // even with a partitioned scd2-looking combo. + let m = parse_pipeline_annotations("// materialize manual ducklake://a/dim history") + .materialize + .expect("materialize"); + assert!(m.manual && m.scd2); + assert!(m.validate(true).is_ok()); + } + #[test] fn materialize_on_schema_change_opt() { let out = parse_pipeline_annotations( diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index fff036b22c..dccf8ec313 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -514,6 +514,11 @@ impl<'a> MaterializeCodegen<'a> { // same write shape as `replace`, which is reliable. The DELETE is // scoped to the current partition when partitioned so it stays // slice-local (a key present in another partition is untouched). + // + // Guard first: the DELETE+INSERT does not dedup the source, so + // two incoming rows sharing a key would both persist under it. + // Raise instead of silently double-writing (see the helper). + out.push(duplicate_source_key_guard_sql(sel, unique_key)); let scope = if self.partitioned { format!("{pcol} = {pval} AND ") } else { @@ -746,6 +751,29 @@ fn schema_drift_guard_sql(sel_sql: &str, target_qualified: &str, partition_col: ) } +/// In-transaction guard for the keyed `merge` strategy: raises via DuckDB +/// `error(...)` when the source SELECT holds more than one row for the same +/// non-NULL `unique_key`. A keyed merge is delete-by-key + insert-all (it does +/// NOT deduplicate the source), so duplicate source keys would land every +/// duplicate row under one key — the exact silent double-write this guards +/// against. Erroring keeps the semantics explicit: the author must deduplicate +/// in the SELECT (or use `append`). NULL keys are excluded to match the delete's +/// `key IN (...)` scope, which never matches NULL. `unique_key` is embedded raw +/// in identifier position (matching the merge's own DELETE/IN) and doubled-quote +/// escaped where it lands inside the error string literal. +fn duplicate_source_key_guard_sql(sel_sql: &str, unique_key: &str) -> String { + let key_lit = unique_key.replace('\'', "''"); + format!( + "SELECT CASE WHEN _wm_dup_keys > 0 THEN CAST(error('managed materialize: keyed merge on \ + `{key_lit}` blocked — the source has ' || _wm_dup_keys || ' key value(s) with more than \ + one row. A keyed merge keeps one row per key and does not deduplicate; deduplicate in the \ + SELECT (e.g. QUALIFY row_number() OVER (PARTITION BY {key_lit} ORDER BY …) = 1) or use \ + `append`.') AS VARCHAR) ELSE 'ok' END FROM (SELECT count(*) AS _wm_dup_keys FROM (SELECT \ + {unique_key} FROM ({sel_sql}) WHERE {unique_key} IS NOT NULL GROUP BY {unique_key} HAVING \ + count(*) > 1));" + ) +} + /// The `warn`-mode summary column: a `schema_drift` struct `{added, removed}` /// when the SELECT drifted from the table, else NULL. Appended to the /// materialize summary row so the executor can log it and fold it into the job @@ -1555,6 +1583,42 @@ mod tests { .any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'"))); } + #[test] + fn codegen_merge_emits_duplicate_source_key_guard() { + // A keyed merge does not dedup its source, so codegen must emit a guard + // that fails the write when the SELECT has >1 row per key — otherwise + // duplicate source keys silently double-write. + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT order_id, amount FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: false, + strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() }, + on_schema_change: OnSchemaChange::Warn, + }; + let st = cg.statements(); + let guard = st + .iter() + .find(|s| s.contains("_wm_dup_keys")) + .expect("duplicate-key guard stmt"); + // raises via error(), counts non-NULL keys appearing more than once + assert!(guard.contains("error('managed materialize: keyed merge on `order_id` blocked")); + assert!(guard.contains("GROUP BY order_id HAVING count(*) > 1")); + assert!(guard.contains("WHERE order_id IS NOT NULL")); + // must run before the mutations so a violation aborts before any write + let guard_pos = st.iter().position(|s| s.contains("_wm_dup_keys")).unwrap(); + let del_pos = st + .iter() + .position(|s| s.starts_with("DELETE FROM")) + .unwrap(); + let ins_pos = st + .iter() + .position(|s| s.starts_with("INSERT INTO")) + .unwrap(); + assert!(guard_pos < del_pos && guard_pos < ins_pos); + } + #[test] fn codegen_append_inserts_only() { let cg = MaterializeCodegen { diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index c3d01dad13..0ae16b9d51 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1307,6 +1307,13 @@ async fn create_script_internal<'c>( ns.path ); } + // SCD2 (`key=history` / `scd2`) option-combination checks the + // runtime cannot honor: reject at deploy with a clear message rather + // than letting the script deploy and fail on its first run. Mirrors the + // executor's safety-net `MaterializeSpec::validate` (duckdb_executor). + if let Err(e) = m.validate(pipeline_annotations.partition.is_some()) { + return Err(Error::BadRequest(e)); + } // `manual` materialize never captures a schema (no wrap codegen), so // there is no contract for `on_schema_change` to mute downstream. if m.manual && m.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Ignore { diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e51c59cb98..051aa77488 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -219,19 +219,18 @@ fn build_materialized_query( for s in plan.setup.iter_mut() { *s = substitute(s); } + // Deploy (`create_script_internal`) already rejects the invalid SCD2 combos, + // but preview/test runs reach the executor without a deploy — re-check so a + // bad combo fails with the same clear message instead of a raw DuckDB error. + m.validate(partitioned).map_err(Error::ExecutionErr)?; let strategy = if m.scd2 { - // SCD2 needs a natural key to identify an entity across versions, and its - // diff/close/open shape has no partition-scoped form in v1. + // SCD2 needs a natural key to identify an entity across versions (checked + // by `validate` above, which guarantees a non-empty key here). let key = m.unique_key.clone().ok_or_else(|| { Error::ExecutionErr( "materialize scd2: requires a natural key — add `key=`".to_string(), ) })?; - if partitioned { - return Err(Error::ExecutionErr( - "materialize scd2: `// partitioned` is not supported with scd2 in v1".to_string(), - )); - } MaterializeStrategy::Scd2 { key, track: m.track.clone(), close_deleted: m.close_deleted } } else if m.append { MaterializeStrategy::Append @@ -3510,7 +3509,10 @@ mod tests { rewritten.contains("AS schema_drift"), "warn folds drift into the summary: {rewritten}" ); - assert!(!rewritten.contains("CAST(error(")); + // No schema-drift *fail* guard in warn mode (the keyed merge still emits + // its own duplicate-source-key guard, which is a different `error(...)`). + assert!(!rewritten.contains("on_schema_change=fail blocked")); + assert!(rewritten.contains("keyed merge on `id` blocked")); assert!(!rewritten.contains("BY NAME")); assert!(meta.sync_prepass.is_none()); } diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 8f99b4c2c9..38c38a01f2 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -62,9 +62,18 @@ stays separate because it is cross-cutting (cascade + scheduling + materialize). error pointing to the `wmll.ducklake` helpers). - **`manual`** — escape hatch: the script writes its own DDL; Windmill only records state (no snapshot capture, no idempotency guarantee). Rare; explicit. -- **`key=`** → MERGE (dedup within slice, SCD type 1 — overwrites history); - **`append`** → INSERT-only; neither → DELETE-by-partition + INSERT (replace). - `append` wins over `key` if both are given (deploy warning). +- **`key=`** → MERGE (upsert-by-key within the slice, SCD type 1 — + overwrites history); **`append`** → INSERT-only; neither → DELETE-by-partition + + INSERT (replace). `append` wins over `key` if both are given (deploy warning). + - **Source must have unique keys.** The merge is delete-by-key + insert-all: it + reconciles the incoming rows against the *target* (rows whose key is in the + SELECT are replaced), but it does **not** deduplicate the *source*. Two + incoming rows sharing a key would therefore both land under that key, silently + breaking the "one row per key" contract. The managed write guards against this + — the run **fails** with a clear error when the SELECT returns more than one + row for a non-NULL key (`NULL` keys are exempt, matching the insert-only path). + Deduplicate in the SELECT (e.g. `QUALIFY row_number() OVER (PARTITION BY + ORDER BY DESC) = 1`) or use `append` if duplicates are intended. - **`key=history [track=] [deletes=close]`** → upgrades the keyed merge to managed SCD **type 2** history (the leading keyword `scd2` is an alias). The SELECT is the *current snapshot* (one row per `key`), and the runtime adds @@ -73,7 +82,9 @@ stays separate because it is cross-cutting (cascade + scheduling + materialize). history. Diff → close-old (`UPDATE`) → open-new (`INSERT`) in one transaction; the effective timestamp is the transaction clock (`now()`), so a run is self-consistent. v1 is **non-partitioned only** (`// partitioned` + history is - rejected). Unlike `manual`, it is managed, so `// data_test` and schema capture + rejected **at deploy**), and it **requires `key=`** (also rejected at deploy). + Both misconfigurations fail fast at save with a clear message instead of on the + first run. Unlike `manual`, it is managed, so `// data_test` and schema capture work. - **Deletes.** By default a key that disappears from the snapshot stays current (soft delete — dbt's `hard_deletes=ignore`). `deletes=close` opts into diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index 126a4ac215..d00a78aa7f 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -137,7 +137,9 @@ schema capture, and each run (re)creates a `_current` view; consumers get effective-dated joins via native `ASOF JOIN … >= valid_from`. Deletes follow dbt: soft by default (absent keys stay current), or `deletes=close` for `hard_deletes=close`. v1 is non-partitioned; partitioned history is a follow-up. -See `ducklake-materialization.md`. +Both misconfigurations — `history` without `key=`, and `history` + `// +partitioned` — are rejected **at deploy** (fail-fast at save, not on the first +run). See `ducklake-materialization.md`. ### 5. Selective execution grammar @@ -434,6 +436,13 @@ DELETE-by-partition + INSERT works only if you reprocess from a source-of-truth source. If you're consuming amendments and need dedup *within* the slice, you need MERGE on `order_id`, not DELETE on partition. +Note the boundary: the keyed merge reconciles the incoming rows against the +*target* (delete-by-key + insert), but it does not deduplicate the *source*. +"Latest version per `order_id`" is still your SELECT's job — pick one row per +key (e.g. `QUALIFY row_number() OVER (PARTITION BY order_id ORDER BY updated_at +DESC) = 1`). A source that still has two rows for one key is rejected at run +time rather than double-written (see `ducklake-materialization.md`). + This is what dbt's `unique_key` does. Orthogonal to partitioning: `partitioned` answers "which slice?"; `unique_key` answers "how do I dedup inside the slice?" From 0ad174fa490e17eeb26280b5bdfd62956dfed9ff Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:29:35 +0200 Subject: [PATCH 201/273] feat(pipelines): self-teaching custom data_test errors + scaffold (#9937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custom `// data_test ` scripts must be a single SELECT reading the freshly-materialized target via the internal `_wm_target.
` alias — neither was documented or scaffolded. Make the codegen errors name the exact violation (multi-statement, non-SELECT, wrong alias, empty) and append a copyable `SELECT * FROM _wm_target.
WHERE ` example. Add a DuckDB-only 'Data test' pipeline output kind that scaffolds that starter body. Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-parser/src/sql_materialize.rs | 136 ++++++++++++++++-- .../pipelineTemplates.dataTest.test.ts | 56 ++++++++ .../assets/AssetGraph/pipelineTemplates.ts | 48 ++++++- 3 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index dccf8ec313..1b24a5bccd 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -1248,6 +1248,28 @@ fn sample_star(ctx: &DataTestCtx, qualifier: Option<&str>) -> String { } } +// Self-teaching tail appended to every malformed-custom-test error. It states +// the two rules that aren't documented or scaffolded anywhere else — the body +// is a single SELECT, and it reads the freshly-materialized target through the +// internal `_wm_target.
` alias — and doubles that alias into a copyable +// one-line example. `target_qualified` is already `_wm_target.
`. +fn custom_test_hint(target_qualified: &str) -> String { + format!( + "Write a single SELECT against `{target_qualified}` returning the offending rows, e.g. \ + `SELECT * FROM {target_qualified} WHERE ` — an empty result means the test \ + passes." + ) +} + +// Whether a custom-test statement reads the materialized target through the +// reserved `_wm_target` alias (the only handle the runtime attaches it under). +// SQL identifiers are case-insensitive, so match case-insensitively; +// `split_statements` has already stripped comments, so a match here is a real +// reference, not one buried in a comment. `TARGET_ALIAS` is lowercase. +fn references_target(stmt: &str) -> bool { + stmt.to_lowercase().contains(TARGET_ALIAS) +} + /// Compile resolved data tests into ATTACH statements + per-test checks for /// `ctx`'s target. Pure: returns SQL text, executes nothing. Errors carry an /// actionable message (e.g. a relationships target that isn't an attachable @@ -1379,24 +1401,44 @@ pub fn build_data_test_checks( } DataTestResolved::Custom { path, body } => { // dbt singular-test convention: the body is a *single* SELECT - // (or CTE) returning the violating rows. It is embedded as a - // subquery (`FROM ()`), so a multi-statement body would - // produce invalid SQL — validate up front with an actionable - // error. It runs in the target's connection (can read - // `_wm_target` + the user's attaches); partition substitution is - // already applied by the worker. + // (or CTE) returning the violating rows, reading the + // freshly-materialized target through the internal `_wm_target` + // schema. It is embedded as a subquery (`FROM ()`), so a + // multi-statement or non-SELECT body would produce invalid SQL. + // Neither rule is documented or scaffolded elsewhere, so the + // errors are self-teaching: they name the exact violation and + // append a correct one-line example. It runs in the target's + // connection (can read `_wm_target` + the user's attaches); + // partition substitution is already applied by the worker. + let hint = custom_test_hint(t); let stmts = split_statements(body); if stmts.is_empty() { - return Err(format!("data_test custom `{path}`: empty test body")); + return Err(format!( + "data_test custom `{path}`: empty test body. {hint}" + )); } if stmts.len() > 1 { return Err(format!( - "data_test custom `{path}`: must be a single SELECT returning the \ - violating rows (found {} statements)", + "data_test custom `{path}`: a custom data test must be a single SELECT, \ + but found {} statements. {hint}", stmts.len() )); } - push_check(&mut out, format!("custom({path})"), stmts[0].to_string()); + let stmt = &stmts[0]; + if classify_block(stmt) != BlockClass::Output { + return Err(format!( + "data_test custom `{path}`: a custom data test must be a single SELECT, \ + not a write or DDL statement. {hint}" + )); + } + if !references_target(stmt) { + return Err(format!( + "data_test custom `{path}`: the test never reads the freshly-materialized \ + target — reference it through the internal `{TARGET_ALIAS}` schema (as \ + `{t}`), not the table name on its own. {hint}" + )); + } + push_check(&mut out, format!("custom({path})"), stmt.to_string()); } } } @@ -2343,13 +2385,85 @@ mod tests { #[test] fn data_test_custom_rejects_multi_statement_body() { // The body is embedded as a subquery, so a setup-then-SELECT body would - // produce invalid SQL — reject it up front with an actionable error. + // produce invalid SQL — reject it up front with a self-teaching error + // that names the violation and shows the correct single-SELECT shape. let tests = vec![DataTestResolved::Custom { path: "f/tests/amount".into(), body: "SET threads = 1; SELECT * FROM _wm_target.orders WHERE amount < 0".into(), }]; let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); assert!(err.contains("single SELECT"), "unexpected error: {err}"); + assert!( + err.contains("found 2 statements"), + "unexpected error: {err}" + ); + // the copyable example points at the internal target alias. + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_rejects_non_select_body() { + // A write/DDL body can't be embedded as `FROM ()`; the error must + // say so and teach the single-SELECT convention. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "DELETE FROM _wm_target.orders WHERE amount < 0".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("single SELECT"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_rejects_wrong_target_alias() { + // Referencing the target by its bare table name (not `_wm_target.
`) + // is the most common custom-test mistake — the runtime only attaches the + // freshly-materialized target under `_wm_target`, so the query would fail + // at runtime. Catch it at codegen with a self-teaching error. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "SELECT * FROM orders WHERE amount < 0".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("_wm_target"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_accepts_from_first_and_uppercased_alias() { + // DuckDB's FROM-first syntax is a valid Output, and the alias match is + // case-insensitive (SQL identifiers are), so this passes. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "FROM _WM_TARGET.orders WHERE amount < 0".into(), + }]; + let sql = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap(); + assert!(sql.checks[0] + .probe + .contains("FROM (FROM _WM_TARGET.orders WHERE amount < 0) _wm_v")); + } + + #[test] + fn data_test_custom_empty_body_teaches_shape() { + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: " \n-- just a comment\n".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("empty test body"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); } #[test] diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts new file mode 100644 index 0000000000..2acdf2abfb --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest' +import { + autoOutputAsset, + compatibleOutputKinds, + generatePipelineDraft, + PIPELINE_OUTPUT_KINDS +} from './pipelineTemplates' + +// The `data_test` output kind scaffolds a *custom* (singular) data test: a +// standalone DuckDB script referenced from a materialize script's +// `-- data_test ` line. It must be a single SELECT that reads the +// freshly-materialized target through the internal `_wm_target` schema — the +// two rules the backend's self-teaching errors enforce. +describe('data_test scaffold', () => { + it('is a DuckDB-only output kind exposed in the picker', () => { + expect(compatibleOutputKinds('duckdb')).toContain('data_test') + expect(compatibleOutputKinds('python3')).not.toContain('data_test') + expect(PIPELINE_OUTPUT_KINDS.map((k) => k.id)).toContain('data_test') + }) + + it('produces no output asset (it asserts against an existing target)', () => { + expect(autoOutputAsset('data_test', 'folder', 'duckdb')).toBeUndefined() + }) + + it('scaffolds a single SELECT against `_wm_target.
`', () => { + const src = generatePipelineDraft({ + language: 'duckdb', + outputKind: 'data_test', + triggers: [] + }) + // starter body is a single SELECT against the internal target alias. + expect(src).toContain('SELECT * FROM _wm_target.your_table WHERE your_condition;') + // exactly one SQL statement (single SELECT) — count statement lines, not + // the word "SELECT" that also appears in the guidance comment. + const stmtLines = src.split('\n').filter((l) => /^\s*SELECT\b/i.test(l)) + expect(stmtLines).toHaveLength(1) + // no `-- materialize ` output annotation — a data test declares no + // asset (the word still appears in the guidance comment, which is fine). + expect(src).not.toMatch(/^--\s*materialize\s/m) + // teaches how to wire it up + the offending-rows convention. + expect(src).toContain('-- data_test ') + expect(src).toContain('offending rows') + }) + + it('seeds the table name from an upstream ducklake asset when present', () => { + const src = generatePipelineDraft({ + language: 'duckdb', + outputKind: 'data_test', + input: { kind: 'ducklake', path: 'analytics/orders' }, + triggers: [] + }) + expect(src).toContain('SELECT * FROM _wm_target.orders WHERE your_condition;') + // no ATTACH of the input — the runtime attaches the target as `_wm_target`. + expect(src).not.toContain('ATTACH') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index 11077d3609..f81d1a330d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -17,6 +17,7 @@ export type PipelineOutputKind = | 'datatable' | 'ducklake' | 'materialize' + | 'data_test' | 's3_parquet' | 's3_object' | 'macros' @@ -37,6 +38,11 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [ label: 'Materialized table', description: 'Managed DuckLake table — idempotent, versioned, tracked' }, + { + id: 'data_test', + label: 'Data test', + description: 'Custom assertion — a single SELECT returning offending rows (empty = pass)' + }, { id: 'datatable', label: 'Data table', @@ -83,7 +89,16 @@ const LANG_COMPATIBILITY: Record = { // single SELECT. The Python/TS `wmll.ducklake` helper currently takes a SQL // SELECT (not in-memory rows), so a polyglot managed materialize is a // separate follow-up — those langs keep the `ducklake` raw-write kind. - duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'macros', 'none'], + duckdb: [ + 'materialize', + 'data_test', + 'datatable', + 'ducklake', + 's3_parquet', + 's3_object', + 'macros', + 'none' + ], postgresql: ['datatable', 'none'], mysql: ['none'], mssql: ['none'], @@ -183,8 +198,10 @@ export function autoOutputAsset( } } // A macro library produces no asset — its "output" is the registry - // entries the deploy records. + // entries the deploy records. A custom data test produces no asset + // either — it asserts against an existing materialized target. case 'macros': + case 'data_test': case 'none': return undefined } @@ -316,6 +333,17 @@ export type TemplateContext = { function header(ctx: TemplateContext): string { const { language, triggers, output, outputKind } = ctx const p = commentPrefix(language) + // A custom data test is a standalone script, not a graph node that produces + // an asset — so it gets no `// pipeline` / output annotation. Instead, tell + // the author how to wire it up (the `data_test` reference) and the two rules + // that aren't obvious: single SELECT, returning the offending rows. + if (outputKind === 'data_test') { + return [ + `${p} Custom data test — reference it from a materialize script with \`${p} data_test \`.`, + `${p} It must be a single SELECT returning the offending rows; the run fails if any row comes back.`, + '' + ].join('\n') + } const lines = triggers.map((t) => { switch (t.kind) { case 'asset': @@ -544,6 +572,22 @@ function bodyPython(ctx: TemplateContext): string { function bodyDuckdb(ctx: TemplateContext): string { const { input, output, outputKind } = ctx const dataUpload = isDataUpload(ctx.triggers) + if (outputKind === 'data_test') { + // Standalone custom data test: it reads ONLY the freshly-materialized + // target, which the runtime attaches under the internal `_wm_target` + // schema — so it emits no ATTACH / input load of its own. When the test + // was created off a ducklake asset, seed its table name; otherwise a + // clear placeholder. This exact shape (single SELECT vs `_wm_target`) is + // what the backend's self-teaching errors ask for. + const testTable = input?.kind === 'ducklake' ? catalogTableRef(input.path) : 'your_table' + return [ + '', + `-- Return the rows that VIOLATE your assertion; an empty result means the test passes.`, + `-- \`_wm_target\` is the freshly-materialized target, attached by the runtime.`, + `SELECT * FROM _wm_target.${testTable} WHERE your_condition;`, + '' + ].join('\n') + } const lines: string[] = [] if (dataUpload) { // `(s3object)` param declaration → the run form renders the S3 picker From 46be39dfb7fbfb2b70e61819d6065b45810c41c9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:34:58 +0200 Subject: [PATCH 202/273] fix(pipelines): order data_test relationships refs before the tested script in a cascade (#9934) * fix(pipelines): order data_test relationships refs before the tested script in a cascade Co-Authored-By: Claude Opus 4.8 (1M context) * fix(pipelines): key custom-test reads by (usage_kind, path) to avoid same-path flow collisions Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-api-assets/src/lib.rs | 118 ++++++++++++++++++ backend/windmill-api/openapi.yaml | 32 +++++ docs/ducklake-materialization.md | 34 +++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 27 ++++ .../assets/AssetGraph/boundedCascade.test.ts | 71 ++++++++++- .../assets/AssetGraph/boundedCascade.ts | 14 +++ .../lib/components/assets/AssetGraph/types.ts | 17 +++ 7 files changed, 311 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 2cebf9ed13..c0a93c039b 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -797,6 +797,25 @@ enum TriggerEdge { }, } +// Ordering-only "must-run-after" edge: `runnable_path`'s data test reads +// `asset` (a `// data_test relationships` ref, or a custom test whose body +// reads a known pipeline asset), so the asset's in-pipeline producer must +// materialize before `runnable_path` runs. NOT a data-consumption edge — the +// tested script doesn't ingest the asset's rows, it only needs the table to +// exist at test time. Rendered dashed (like macro edges) and fed into the +// cascade topo-sort so a cold cascade orders the referenced dimension first. +// Only emitted when the referenced asset has a producer in the graph; an +// external table (no producer) adds no edge — the runtime error stands. +#[derive(Serialize, Debug)] +struct TestEdge { + producer_kind: AssetUsageKind, + producer_path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + asset_kind: AssetKind, + asset_path: String, +} + #[derive(Serialize, Debug)] struct AssetGraphResponse { assets: Vec, @@ -805,6 +824,8 @@ struct AssetGraphResponse { triggers: Vec, #[serde(skip_serializing_if = "Vec::is_empty", default)] macro_edges: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + test_edges: Vec, } async fn asset_graph( @@ -1257,6 +1278,102 @@ async fn asset_graph( runnable_set.insert((AssetUsageKind::Script, e.consumer_path.clone())); } + // Data-test ordering edges. A `// data_test relationships -> ` + // (and, best-effort, a custom `// data_test + {#if data.fork_materialization === 'deferred'} - + + parent {:else if data.fork_materialization === 'fork'} - + + fork {/if} +
+ +
+ {/if} {#if showActions} + + {/if} {#if mode === 'edit' && saveErrors.size > 0} + {#if calendarPicker && materializeTarget?.kind === 'ducklake' && value} + {#if selectedMaterialized} + + + {value} is already materialized — running replaces it. + + {:else} + + + {value} not materialized yet — running creates it. + + {/if} + {/if} + + + {#if upstreamMissing} + + + + No upstream data for {value} yet — this run may materialize an + empty partition. + + + {/if} + + + {#if recentlyMissing.length > 0} +
+ + {recentlyMissing.length} of the last {recentWindow(spec.kind)} + {spec.kind} partitions are not materialized: + +
+ {#each recentlyMissing.slice(0, MAX_MISSING_CHIPS) as b (b)} + + {/each} + {#if recentlyMissing.length > MAX_MISSING_CHIPS} + + +{recentlyMissing.length - MAX_MISSING_CHIPS} more + + {/if} +
+
+ {/if} + diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte index fc5e7ad945..4384f125ba 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte @@ -1,6 +1,9 @@ - +
+ {#if partitionSpec} + args.partition, (v) => (args.partition = v)} + {workspace} + {materializeTarget} + {upstreamAssets} + /> + {/if} + +
diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte index cadd3c6a94..64e24e0ad2 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte @@ -1,11 +1,13 @@ + +{#if show} +
+
+ +
+

Finish setting up pipelines

+

+ Pipelines materialize data into DuckLake tables backed by object storage. Configure the + following before your first pipeline can run. +

+
+
+ +
    + {#each steps as step (step.title)} + {@const Icon = step.icon} +
  • + {#if step.done === true} + + {:else if step.done === false} + + {:else} + + {/if} +
    + {step.title} + {step.description} +
    + {#if step.done !== true} + + {step.cta} + + + {:else} + Configured + {/if} +
    • + {/each} +
+
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts new file mode 100644 index 0000000000..a8a47af259 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from 'vitest' +import { + bucketFor, + bucketFromInputValue, + defaultBucket, + inputValueFromBucket, + isBeforeStart, + isValidStart, + isValidTimeZone, + partitionInputType, + partitionMetadataError, + recentBuckets, + startBucketOf, + usesCalendarPicker +} from './partitionBuckets' +import type { PartitionSpec } from './parsePipelineAnnotations' + +const spec = (kind: PartitionSpec['kind'], extra: Partial = {}): PartitionSpec => + ({ kind, ...extra }) as PartitionSpec + +// A fixed UTC instant: 2026-07-05 14:37Z (a Sunday — ISO week 27 of 2026). +// Explicit UTC so the assertions are independent of the test runner's TZ, and +// they exercise the same UTC default the backend uses when `spec.tz` is absent. +const at = new Date(Date.UTC(2026, 6, 5, 14, 37, 0)) + +describe('bucketFor mirrors backend default_format (UTC)', () => { + it('daily -> %Y-%m-%d', () => { + expect(bucketFor(spec('daily'), at)).toBe('2026-07-05') + }) + it('hourly -> %Y-%m-%dT%H', () => { + expect(bucketFor(spec('hourly'), at)).toBe('2026-07-05T14') + }) + it('monthly -> %Y-%m', () => { + expect(bucketFor(spec('monthly'), at)).toBe('2026-07') + }) + it('weekly -> ISO %G-W%V', () => { + expect(bucketFor(spec('weekly'), at)).toBe('2026-W27') + }) +}) + +describe('bucketFor honours spec.tz', () => { + // 2026-07-05 02:30Z is still 2026-07-04 in America/New_York (UTC-4 in July). + const nearMidnight = new Date(Date.UTC(2026, 6, 5, 2, 30, 0)) + it('shifts the day boundary by the producer tz', () => { + expect(bucketFor(spec('daily'), nearMidnight)).toBe('2026-07-05') // UTC default + expect(bucketFor(spec('daily', { tz: 'America/New_York' }), nearMidnight)).toBe('2026-07-04') + }) + it('shifts the hour bucket by the producer tz', () => { + // 02:30Z -> 22 (previous day) in New York. + expect(bucketFor(spec('hourly', { tz: 'America/New_York' }), nearMidnight)).toBe( + '2026-07-04T22' + ) + }) +}) + +describe('defaultBucket honours the start= anchor', () => { + // `at` is 2026-07-05. + it('seeds the current bucket when at or after start', () => { + expect(defaultBucket(spec('daily'), at)).toBe('2026-07-05') + expect(defaultBucket(spec('daily', { start: '2026-01-01' }), at)).toBe('2026-07-05') + expect(defaultBucket(spec('daily', { start: '2026-07-05' }), at)).toBe('2026-07-05') // == start, not before + }) + it('seeds the start bucket (never a pre-start one) when before start', () => { + expect(defaultBucket(spec('daily', { start: '2026-08-01' }), at)).toBe('2026-08-01') + expect(defaultBucket(spec('monthly', { start: '2026-08-01' }), at)).toBe('2026-08') + expect(defaultBucket(spec('hourly', { start: '2026-08-01' }), at)).toBe('2026-08-01T00') + }) + it('isBeforeStart mirrors the backend date comparison', () => { + expect(isBeforeStart(spec('daily', { start: '2026-08-01' }), at)).toBe(true) + expect(isBeforeStart(spec('daily', { start: '2026-07-05' }), at)).toBe(false) + expect(isBeforeStart(spec('daily'), at)).toBe(false) + }) + it('startBucketOf renders the anchor in the cadence, undefined when unset', () => { + expect(startBucketOf(spec('daily', { start: '2026-08-01' }))).toBe('2026-08-01') + expect(startBucketOf(spec('monthly', { start: '2026-08-15' }))).toBe('2026-08') + expect(startBucketOf(spec('hourly', { start: '2026-08-01' }))).toBe('2026-08-01T00') + expect(startBucketOf(spec('daily'))).toBeUndefined() + }) +}) + +describe('malformed metadata fails safe (parity with backend validation)', () => { + it('isValidTimeZone rejects garbage, accepts real zones and absence', () => { + expect(isValidTimeZone(undefined)).toBe(true) + expect(isValidTimeZone('UTC')).toBe(true) + expect(isValidTimeZone('America/New_York')).toBe(true) + expect(isValidTimeZone('Not/AZone')).toBe(false) + expect(isValidTimeZone('garbage')).toBe(false) + }) + it('isValidStart rejects malformed and JS-normalized dates', () => { + expect(isValidStart(undefined)).toBe(true) + expect(isValidStart('2026-08-01')).toBe(true) + expect(isValidStart('2026-02-31')).toBe(false) // JS would roll to Mar 3 + expect(isValidStart('2026-13-01')).toBe(false) + expect(isValidStart('08/01/2026')).toBe(false) + }) + it('partitionMetadataError reports the first problem, else undefined', () => { + expect(partitionMetadataError(spec('daily'))).toBeUndefined() + expect( + partitionMetadataError(spec('daily', { tz: 'America/New_York', start: '2026-08-01' })) + ).toBeUndefined() + expect(partitionMetadataError(spec('daily', { tz: 'Not/AZone' }))).toContain('timezone') + expect(partitionMetadataError(spec('daily', { start: '2026-02-31' }))).toContain('start date') + }) + it('bucketFor never throws on an invalid tz (falls back to UTC)', () => { + expect(bucketFor(spec('daily', { tz: 'Not/AZone' }), at)).toBe('2026-07-05') + }) + it('an invalid start is treated as no anchor (isBeforeStart/startBucketOf)', () => { + expect(isBeforeStart(spec('daily', { start: '2026-02-31' }), at)).toBe(false) + expect(startBucketOf(spec('daily', { start: '2026-02-31' }))).toBeUndefined() + }) +}) + +describe('partitionInputType', () => { + it('maps each calendar kind to its native input', () => { + expect(partitionInputType(spec('daily'))).toBe('date') + expect(partitionInputType(spec('hourly'))).toBe('datetime-local') + expect(partitionInputType(spec('weekly'))).toBe('week') + expect(partitionInputType(spec('monthly'))).toBe('month') + }) + it('falls back to text for dynamic and custom-format specs', () => { + expect(partitionInputType(spec('dynamic', { key: '$.tenant' }))).toBe('text') + expect(partitionInputType(spec('daily', { format: '%Y/%m/%d' }))).toBe('text') + expect(usesCalendarPicker(spec('dynamic', { key: '$.tenant' }))).toBe(false) + expect(usesCalendarPicker(spec('daily', { format: '%Y/%m/%d' }))).toBe(false) + }) +}) + +describe('native input <-> bucket round-trip', () => { + it('hourly truncates the datetime-local minutes and restores :00', () => { + expect(bucketFromInputValue(spec('hourly'), '2026-07-05T14:37')).toBe('2026-07-05T14') + expect(inputValueFromBucket(spec('hourly'), '2026-07-05T14')).toBe('2026-07-05T14:00') + }) + it('non-hourly kinds pass through unchanged', () => { + expect(bucketFromInputValue(spec('daily'), '2026-07-05')).toBe('2026-07-05') + expect(inputValueFromBucket(spec('weekly'), '2026-W27')).toBe('2026-W27') + }) + it('empty stays empty', () => { + expect(bucketFromInputValue(spec('hourly'), '')).toBe('') + expect(inputValueFromBucket(spec('daily'), '')).toBe('') + }) +}) + +describe('recentBuckets (UTC)', () => { + it('walks back day by day, most recent first', () => { + expect(recentBuckets(spec('daily'), at, 3)).toEqual(['2026-07-05', '2026-07-04', '2026-07-03']) + }) + it('walks back hour by hour', () => { + expect(recentBuckets(spec('hourly'), at, 3)).toEqual([ + '2026-07-05T14', + '2026-07-05T13', + '2026-07-05T12' + ]) + }) + it('walks back month by month across a year boundary without day roll-over', () => { + const jan31 = new Date(Date.UTC(2026, 0, 31, 0, 0, 0)) + expect(recentBuckets(spec('monthly'), jan31, 3)).toEqual(['2026-01', '2025-12', '2025-11']) + }) + it('walks back week by week', () => { + expect(recentBuckets(spec('weekly'), at, 3)).toEqual(['2026-W27', '2026-W26', '2026-W25']) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts new file mode 100644 index 0000000000..401ef82cae --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts @@ -0,0 +1,261 @@ +// Client-side partition-bucket math for the run form's partition picker. +// Produces the same canonical bucket strings the backend renders in +// `windmill-common/src/partition_ee.rs` (`resolve_time_partition` / +// `default_format`): the instant is localized to the spec's timezone +// (`spec.tz`, defaulting to UTC — NOT the browser's zone) and then formatted. +// Getting the zone right matters — a browser in a non-UTC zone would otherwise +// seed a default bucket, and compare against materialized rows, off by a +// day/hour near every boundary and always for an explicit `tz=` spec. +// +// daily %Y-%m-%d -> 2026-07-05 +// hourly %Y-%m-%dT%H -> 2026-07-05T14 +// weekly %G-W%V (ISO) -> 2026-W27 +// monthly %Y-%m -> 2026-07 +// +// Pure module (no Svelte runes) so the mapping is unit-testable. + +import type { PartitionSpec } from './parsePipelineAnnotations' + +export type PartitionInputType = 'date' | 'month' | 'week' | 'datetime-local' | 'text' + +// A spec carrying a custom strftime `format` can't be reproduced by the native +// date pickers (arbitrary strftime), and `dynamic` partitions are a free-form +// key extracted from the payload — both fall back to a plain text input. +export function usesCalendarPicker(spec: PartitionSpec): boolean { + return spec.kind !== 'dynamic' && !spec.format +} + +export function partitionInputType(spec: PartitionSpec): PartitionInputType { + if (!usesCalendarPicker(spec)) return 'text' + switch (spec.kind) { + case 'monthly': + return 'month' + case 'weekly': + return 'week' + case 'hourly': + return 'datetime-local' + default: + return 'date' + } +} + +function pad(n: number): string { + return String(n).padStart(2, '0') +} + +const ZONED_PARTS_OPTS: Intl.DateTimeFormatOptions = { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hourCycle: 'h23' +} + +// Re-express `at` as a Date whose UTC fields equal the wall-clock in `tz`, so +// all downstream field reads / calendar arithmetic can use the UTC getters and +// stay in the producer's zone. `hourCycle: 'h23'` keeps hours 00–23. A +// malformed `tz` (rejected by `Intl`) falls back to UTC rather than throwing — +// the backend validates `tz=` and is the source of truth for the error; the +// picker must never crash the graph view (`partitionMetadataError` gates +// auto-seeding so a bogus bucket isn't silently sent). +function zonedAsUtc(at: Date, tz: string): Date { + let parts: Intl.DateTimeFormatPart[] + try { + parts = new Intl.DateTimeFormat('en-US', { ...ZONED_PARTS_OPTS, timeZone: tz }).formatToParts( + at + ) + } catch { + parts = new Intl.DateTimeFormat('en-US', { + ...ZONED_PARTS_OPTS, + timeZone: 'UTC' + }).formatToParts(at) + } + const g = (t: string) => Number(parts.find((p) => p.type === t)?.value) + return new Date( + Date.UTC(g('year'), g('month') - 1, g('day'), g('hour'), g('minute'), g('second')) + ) +} + +// `tz=` is valid iff `Intl` accepts it (absent === UTC === valid). +export function isValidTimeZone(tz?: string): boolean { + if (!tz) return true + try { + new Intl.DateTimeFormat('en-US', { timeZone: tz }) + return true + } catch { + return false + } +} + +// `start=` is valid iff it's a real `YYYY-MM-DD` calendar date. The round-trip +// check rejects dates JS would silently normalize (e.g. `2026-02-31` → Mar 3), +// which the backend's `NaiveDate::parse_from_str` also rejects. +export function isValidStart(start?: string): boolean { + if (!start) return true + const m = start.match(/^(\d{4})-(\d{2})-(\d{2})$/) + if (!m) return false + const y = Number(m[1]) + const mo = Number(m[2]) + const d = Number(m[3]) + const dt = new Date(Date.UTC(y, mo - 1, d)) + return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d +} + +// A human-readable reason the `// partitioned` metadata is unusable, or +// undefined when it's sound. Mirrors the backend's `tz=` / `start=` validation +// so the picker can refuse to auto-seed a bucket the backend would reject. +export function partitionMetadataError(spec: PartitionSpec): string | undefined { + if (!isValidTimeZone(spec.tz)) return `invalid timezone "${spec.tz}"` + if (!isValidStart(spec.start)) return `invalid start date "${spec.start}" (want YYYY-MM-DD)` + return undefined +} + +// ISO 8601 week-numbering year + week (chrono's %G / %V) of a UTC-substituted +// date. Standard "nearest Thursday" algorithm, all in UTC. +function isoWeekOf(d: Date): { isoYear: number; week: number } { + const date = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())) + const dayNum = (date.getUTCDay() + 6) % 7 // Mon=0 … Sun=6 + date.setUTCDate(date.getUTCDate() - dayNum + 3) // Thursday of this week + const isoYear = date.getUTCFullYear() + const firstThursday = new Date(Date.UTC(isoYear, 0, 4)) + const fdNum = (firstThursday.getUTCDay() + 6) % 7 + firstThursday.setUTCDate(firstThursday.getUTCDate() - fdNum + 3) + const week = 1 + Math.round((date.getTime() - firstThursday.getTime()) / (7 * 86400000)) + return { isoYear, week } +} + +// Render a UTC-substituted date into its canonical bucket for the cadence. +function fmtBucket(kind: PartitionSpec['kind'], d: Date): string { + const y = d.getUTCFullYear() + const m = d.getUTCMonth() + 1 + const day = d.getUTCDate() + const h = d.getUTCHours() + switch (kind) { + case 'hourly': + return `${y}-${pad(m)}-${pad(day)}T${pad(h)}` + case 'monthly': + return `${y}-${pad(m)}` + case 'weekly': { + const { isoYear, week } = isoWeekOf(d) + return `${isoYear}-W${pad(week)}` + } + default: + return `${y}-${pad(m)}-${pad(day)}` + } +} + +export function bucketFor(spec: PartitionSpec, at: Date): string { + return fmtBucket(spec.kind, zonedAsUtc(at, spec.tz ?? 'UTC')) +} + +// The zoned start date (`spec.start`, `YYYY-MM-DD`) as a UTC-substituted Date at +// 00:00, or undefined if unset/malformed. `start` is a plain date in the +// producer's tz — the backend parses it as a NaiveDate and compares by date. +function startDate(spec: PartitionSpec): Date | undefined { + if (!spec.start || !isValidStart(spec.start)) return undefined + const m = spec.start.match(/^(\d{4})-(\d{2})-(\d{2})$/)! + return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]))) +} + +// True when `at` (localized to the producer tz) falls on a date before the +// `start=` anchor — exactly the backend's `local.date_naive() < start_date` +// check (`resolve_time_partition`), which resolves such an instant to no +// partition. +export function isBeforeStart(spec: PartitionSpec, at: Date): boolean { + const start = startDate(spec) + if (!start) return false + const zoned = zonedAsUtc(at, spec.tz ?? 'UTC') + const zonedDate = Date.UTC(zoned.getUTCFullYear(), zoned.getUTCMonth(), zoned.getUTCDate()) + return zonedDate < start.getTime() +} + +// The canonical bucket of the `start=` anchor (its date at 00:00), or undefined +// if unset. Buckets sort lexicographically within a cadence, so callers can +// compare against it to drop pre-start buckets. +export function startBucketOf(spec: PartitionSpec): string | undefined { + const start = startDate(spec) + return start ? fmtBucket(spec.kind, start) : undefined +} + +// The bucket to pre-fill the picker with. Normally the current bucket (matching +// the backend's "absent partition arg -> current bucket" resolution), but when +// the current instant is before the `start=` anchor the backend would resolve +// to NO partition — so default to the first valid bucket (the start) rather +// than a pre-start one the worker would take verbatim and materialize early. +export function defaultBucket(spec: PartitionSpec, at: Date): string { + if (isBeforeStart(spec, at)) { + const start = startDate(spec) + if (start) return fmtBucket(spec.kind, start) + } + return bucketFor(spec, at) +} + +// Native input value -> canonical bucket. Only hourly differs: datetime-local +// carries a minute component the hourly bucket drops. The picked wall-clock is +// taken verbatim as the bucket (the user picks in the producer's frame), so no +// timezone conversion happens here. +export function bucketFromInputValue(spec: PartitionSpec, inputValue: string): string { + if (!inputValue) return '' + if (spec.kind === 'hourly') { + const m = inputValue.match(/^(\d{4}-\d{2}-\d{2}T\d{2})/) + return m ? m[1] : inputValue + } + return inputValue +} + +// Canonical bucket -> native input value. Only hourly differs: datetime-local +// needs a minute component the bucket omits. +export function inputValueFromBucket(spec: PartitionSpec, bucket: string): string { + if (!bucket) return '' + if (spec.kind === 'hourly') { + return /T\d{2}$/.test(bucket) ? `${bucket}:00` : bucket + } + return bucket +} + +// The last `count` buckets ending at (and including) `now`, most-recent first, +// localized to `spec.tz`. Arithmetic walks calendar units in the zoned frame, +// so it's exact across DST (no ±1 drift). Undefined for non-calendar specs +// (the caller guards on `usesCalendarPicker`). +export function recentBuckets(spec: PartitionSpec, now: Date, count: number): string[] { + const base = zonedAsUtc(now, spec.tz ?? 'UTC') + const out: string[] = [] + for (let i = 0; i < count; i++) { + const d = new Date(base) + switch (spec.kind) { + case 'hourly': + d.setUTCHours(d.getUTCHours() - i) + break + case 'weekly': + d.setUTCDate(d.getUTCDate() - 7 * i) + break + case 'monthly': + // Normalize to the 1st first so subtracting months can't roll over + // a short target month (e.g. Mar 31 − 1mo → Mar 3). + d.setUTCDate(1) + d.setUTCMonth(d.getUTCMonth() - i) + break + default: + d.setUTCDate(d.getUTCDate() - i) + } + out.push(fmtBucket(spec.kind, d)) + } + return out +} + +// How many recent buckets the "missing partitions" hint scans, per kind — a +// window that reads as "recent" for each cadence without flooding the hint. +export function recentWindow(kind: PartitionSpec['kind']): number { + switch (kind) { + case 'hourly': + return 24 + case 'weekly': + return 8 + case 'monthly': + return 6 + default: + return 14 + } +} diff --git a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte index ab36b4c1f7..e15c3abdf7 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte @@ -1,6 +1,7 @@ +{#snippet circleStack()} + + + {#each visibleUsers as u, i (i)} + + {initials(u)} + + {/each} + {#if overflowCount > 0} + + +{overflowCount} + + {/if} + +{/snippet} + {#if showBadge} @@ -204,32 +233,23 @@ bind:isOpen={popoverOpen} > {#snippet trigger()} - - {#if orderedUsers.length > 0} - - - {#each visibleUsers as u, i (i)} - - {initials(u)} - - {/each} - {#if overflowCount > 0} - - +{overflowCount} - - {/if} - - {/if} - {draft_only ? 'Draft only' : 'Draft'} - + {#if iconOnly} + + + + {#if orderedUsers.length > 0} + {@render circleStack()} + {/if} + + {:else} + + {#if orderedUsers.length > 0} + {@render circleStack()} + {/if} + {draft_only ? 'Draft only' : 'Draft'} + + {/if} {/snippet} {#snippet content()}
diff --git a/frontend/src/lib/components/common/badge/Badge.svelte b/frontend/src/lib/components/common/badge/Badge.svelte index cf7a7cad66..c57d41029c 100644 --- a/frontend/src/lib/components/common/badge/Badge.svelte +++ b/frontend/src/lib/components/common/badge/Badge.svelte @@ -90,7 +90,7 @@ const hovers: Partial> = { gray: 'hover:bg-surface-hover', - blue: 'hover:bg-blue-200 dark:hover:bg-blue-700/40', + blue: 'hover:bg-blue-100 dark:hover:bg-blue-700/60', red: 'hover:bg-red-200 dark:hover:bg-red-500/25', green: 'hover:bg-green-200 dark:hover:bg-green-500/25', yellow: 'hover:bg-yellow-200 dark:hover:bg-yellow-500/25', diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 5f5835bd13..c67bcd17b5 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -44,6 +44,9 @@ import { buildSummaryMessageContent } from './compactionPrompt' import { dfs } from '$lib/components/flows/previousResults' +import { SvelteSet } from 'svelte/reactivity' +import type { UserDraftItemKind } from '$lib/gen' +import { maskKey } from '$lib/components/sessions/modifiedItemsMask' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' import { chatDraft, expanded } from './chatDraft' @@ -343,6 +346,75 @@ export class AIChatManager { // session rather than the UI-active one — keeps backgrounded sessions isolated. sessionId: string | undefined = undefined + // Fired whenever the active chat id changes away from the one the consumer + // knows (a "/clear" rotation or a history switch). Session runtimes wire this + // to keep the session record's chatId aligned — the compare-page handoff + // (`from_session`) reads it, and a stale id would preselect the previous + // chat's items. Set here (not imported) to avoid a copilot→sessions cycle. + onChatRotated: ((chatId: string) => void) | undefined = undefined + + // Workspace items the CURRENT chat modified via AI tool calls, as + // `${UserDraftItemKind}:${storagePath}` keys (see modifiedItemsMask.ts). + // undefined = untracked: the global side-panel chat (never initialised) and + // loaded legacy chats with no stored mask, both of which fall back to the + // show-all bar. A SvelteSet (even empty) = tracked. Reactive so the session + // bar updates as tools record mid-turn. + modifiedItems = $state | undefined>(undefined) + + // Start tracking for a brand-new session chat (empty = "tracked, nothing yet"). + initModifiedItemsTracking() { + this.modifiedItems = new SvelteSet() + } + + // Record an item an AI tool call created/edited/deleted. No-op when untracked + // (the global singleton never initialises the set), so it stays unaffected. + recordModifiedItem(itemKind: UserDraftItemKind, storagePath: string) { + this.modifiedItems?.add(maskKey(itemKind, storagePath)) + } + + // Un-record an item whose chat-made change was discarded — without this the + // still-existing deployed item would keep reading as this chat's "Deployed" + // edit. Persisted immediately: unlike recordModifiedItem (whose persistence + // rides on the turn's saveChat), a discard can fire from the review dock + // outside any turn, and waiting would resurrect the entry on reload. + async removeModifiedItem(itemKind: UserDraftItemKind, storagePath: string) { + if (!this.modifiedItems?.delete(maskKey(itemKind, storagePath))) return + await this.#persistModifiedItems() + } + + // Move a mask entry to the path a draft actually deployed to. A draft-only + // flow/app parks at a synthetic `draft_{uuid}` storage path and deploys to + // its chosen path — without the move, the existence check at the synthetic + // path fails after reload and the deployed row vanishes from the dock. + async renameModifiedItem(itemKind: UserDraftItemKind, fromPath: string, toPath: string) { + if (fromPath === toPath) return + if (!this.modifiedItems?.delete(maskKey(itemKind, fromPath))) return + this.modifiedItems.add(maskKey(itemKind, toPath)) + await this.#persistModifiedItems() + } + + // Serialized, snapshot-at-write-time persistence: two rapid dock actions + // would otherwise race their saveChat writes, and the earlier (staler) + // snapshot could land last — dropping the later mutation until the next + // turn-end save. + #maskPersistQueue: Promise = Promise.resolve() + #persistModifiedItems(): Promise { + this.#maskPersistQueue = this.#maskPersistQueue.then(() => + this.historyManager + .saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) + // Swallow (and log) a failed write so it can't wedge the queue as a + // rejected link — the next persist snapshots the full current set, so + // a lost write self-heals on the next mutation or turn-end save. + .catch((e) => console.error('Failed to persist modified-items mask', e)) + ) + return this.#maskPersistQueue + } + // Workspace AI skills (name + description) advertised in the GLOBAL system // prompt and surfaced as slash commands in session chat. Loaded // asynchronously when entering GLOBAL mode; the system message is rebuilt @@ -659,7 +731,12 @@ export class AIChatManager { ) switch (result) { case 'ok': - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) sendUserToast('Conversation compacted.') break case 'empty': @@ -1636,7 +1713,12 @@ export class AIChatManager { const projectedContextTokens = this.contextTokens + this.estimateMessagesTokens([userMessage]) this.messages.push(userMessage) - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) this.currentReply = '' this.currentReasoning = '' @@ -1672,7 +1754,12 @@ export class AIChatManager { this.contextUsage = Math.max(0, this.contextUsage - freed) } } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } } // Rollback anchors for restoreUnsentTurn: captured after compaction so @@ -1758,7 +1845,10 @@ export class AIChatManager { }, requestConfirmation: this.requestConfirmation, shouldAutoAcceptToolConfirmations: () => this.autoAcceptToolConfirmationsActive, - requestUserQuestion: this.requestUserQuestion + requestUserQuestion: this.requestUserQuestion, + onItemModified: (kind, path) => this.recordModifiedItem(kind, path), + onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), + onItemDiscarded: (kind, path) => void this.removeModifiedItem(kind, path) } } @@ -1794,7 +1884,12 @@ export class AIChatManager { this.contextUsage = result?.lastIterationUsage ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion : undefined - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) // Still counts as the saved first turn — skipping the hook here would // permanently miss it (the next turn isn't "first" anymore). if (isFirstUserTurn && this.afterFirstTurnSaved) { @@ -1824,7 +1919,12 @@ export class AIChatManager { // user message on reload. Remove it instead. this.historyManager.deletePastChat(this.historyManager.getCurrentChatId()) } else { - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } if (!wasAborted) { sendUserToast('The model returned no response — your message was restored to the input.') @@ -1843,7 +1943,12 @@ export class AIChatManager { if (this.autoAcceptEditsActive) { this.acceptPendingFlowEdits() } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) // Only this branch is a clean send: the queued-message flush below // auto-sends the next message after it (set after saveChat so a // persistence failure falls through to the restore path instead). @@ -1867,7 +1972,12 @@ export class AIChatManager { // compaction on the next send instead of failing the same way again. this.contextUsage = undefined try { - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } catch (saveErr) { console.error('Failed to persist partial chat after error', saveErr) } @@ -1996,15 +2106,25 @@ export class AIChatManager { // Drop any message queued in this conversation so it can't auto-send into // the fresh chat or linger as a card across the switch. this.queuedMessage = '' - await this.historyManager.save(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.save( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) this.displayMessages = [] this.messages = [] this.contextUsage = undefined + // The mask belongs to the conversation just saved — the fresh chat starts + // its own (empty) tracking; carrying entries over would claim the previous + // conversation's edits for the new one. Untracked chats stay untracked. + if (this.modifiedItems) this.modifiedItems = new SvelteSet() // In an AI session, linked files are session-scoped: they persist across conversations // (cleared only when the session is deleted). The ephemeral global side-panel chat has no // session, so "New chat" must clear them — otherwise the next, unrelated conversation // would still get the previous file roster and could read/search it. if (!this.isSessionChat) this.attachedFiles.clear() + this.onChatRotated?.(this.historyManager.getCurrentChatId()) } loadPastChat = async (id: string) => { @@ -2019,7 +2139,16 @@ export class AIChatManager { this.displayMessages = chat.displayMessages this.messages = chat.actualMessages this.contextUsage = normalizeContextUsage(chat.contextUsage) + // Seed the modified-items mask from the stored chat. A stored array + // (even empty) → tracked; a legacy chat with no field stays untracked + // (undefined) so the session bar falls back to showing all drafts. The + // global side-panel chat never tracks, so leave it untouched there. + if (this.isSessionChat) { + const stored = this.historyManager.getModifiedItems(id) + this.modifiedItems = stored !== undefined ? new SvelteSet(stored) : undefined + } this.#automaticScroll = true + this.onChatRotated?.(id) } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index d23cbf96a1..a415377a7b 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -693,7 +693,9 @@ describe('AIChatManager context compaction', () => { expect(manager.messages[0]).toMatchObject({ role: 'user', content: 'c'.repeat(400) }) // Mid-turn, the report is debited by the freed estimate (visible in the // compaction-time save) so a rolled-back turn keeps a consistent value - expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000) + // 4th arg: the modified-items mask rides on every save (undefined here — + // this bare manager never initialised tracking). + expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000, undefined) // At commit, the no-report turn clears the stored value; the readable // number falls back to estimating the now-tiny compacted history expect(manager.contextUsage).toBeUndefined() diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 9173e4c441..0db0751293 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -25,6 +25,12 @@ interface ChatSchema extends IDBSchema { // New writes store the plain reported token count; chats persisted by // earlier versions may still hold the legacy anchor object. contextUsage?: PersistedContextUsage + // Workspace items this chat modified via AI tool calls, as + // `${UserDraftItemKind}:${storagePath}` keys. Persisted out-of-band from + // the message arrays so it survives compaction. Absent (undefined) on + // chats predating this feature → consumers fall back to showing all + // workspace drafts; a defined array (even empty) means "tracked". + modifiedItems?: string[] } } } @@ -80,6 +86,29 @@ export function __resetLegacyChatClaimForTesting(): void { legacyChatClaim = undefined } +// Read a chat's modified-items mask by chatId WITHOUT mounting an AIChatManager, +// for the standalone /forks/compare route. Returns undefined for a legacy chat +// (no field) so the page falls back to selecting all items; a defined array +// (even empty) narrows the preselection. Opens a throwaway user-scoped handle; +// the `get` is O(1) on the `id` keyPath. +export async function readChatModifiedItems(chatId: string): Promise { + const dbh = userScopedDb(DB_NAME, { + version: 1, + upgrade: createChatStore, + migrate: migrateLegacyChatDb + }) + try { + const db = await dbh.whenReady() + const chat = await db?.get('chats', chatId) + return chat?.modifiedItems + } catch (err) { + console.error('Could not read chat modified items', err) + return undefined + } finally { + dbh.close() + } +} + export default class HistoryManager { // Per-instance handle to the shared per-user DB lifecycle. There is one // HistoryManager per AIChatManager (the singleton + one per session runtime), @@ -100,6 +129,7 @@ export default class HistoryManager { lastModified: number sessionId?: string contextUsage?: PersistedContextUsage + modifiedItems?: string[] } > = $state({}) @@ -173,10 +203,15 @@ export default class HistoryManager { return Object.values(this.savedChats) } + getModifiedItems(id: string): string[] | undefined { + return this.savedChats[id]?.modifiedItems + } + async saveChat( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], - contextUsage?: number + contextUsage?: number, + modifiedItems?: string[] ) { if (displayMessages.length > 0) { // Compaction replaces the original first message with a summary boundary. @@ -203,7 +238,18 @@ export default class HistoryManager { id: this.currentChatId, lastModified: Date.now(), ...(this.sessionId ? { sessionId: this.sessionId } : {}), - ...(contextUsage !== undefined ? { contextUsage } : {}) + ...(contextUsage !== undefined ? { contextUsage } : {}), + // Only persist when the caller passes a defined array. Loaded legacy + // chats keep their accumulator undefined, so we never retroactively + // stamp them with [] (which would flip them to the filtered view). + // But since `put` replaces the whole record, a caller that omits the + // argument must not ERASE a tracked chat's stored mask — fall back to + // the previously saved field. + ...(modifiedItems !== undefined + ? { modifiedItems } + : this.savedChats[this.currentChatId]?.modifiedItems !== undefined + ? { modifiedItems: this.savedChats[this.currentChatId].modifiedItems } + : {}) } this.savedChats = { ...this.savedChats, @@ -218,9 +264,10 @@ export default class HistoryManager { async save( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], - contextUsage?: number + contextUsage?: number, + modifiedItems?: string[] ) { - await this.saveChat(displayMessages, messages, contextUsage) + await this.saveChat(displayMessages, messages, contextUsage, modifiedItems) this.currentChatId = createLongHash() } diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index 38fb7ecf0d..540dfaf72e 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -141,3 +141,30 @@ describe('HistoryManager title across compaction', () => { expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('original first question') }) }) + +describe('HistoryManager modified-items mask persistence', () => { + const msgs = [{ role: 'user', content: 'hello', index: 0 }] as DisplayMessage[] + + it('a save without the argument preserves a previously stored mask', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[], undefined, ['script:u/a/x']) + expect(hm.getModifiedItems(id)).toEqual(['script:u/a/x']) + + // e.g. manual compaction re-saving the transcript: the whole record is + // rewritten, but the tracked mask must survive. + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[]) + expect(hm.getModifiedItems(id)).toEqual(['script:u/a/x']) + }) + + it('never retroactively stamps an untracked chat', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[]) + expect(hm.getModifiedItems(id)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 90dcb655f4..24045280d0 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -114,6 +114,7 @@ import { getEphemeralSecretVariableDraftValue, getGlobalDraft, getGlobalDraftStoragePath, + itemKindFor, listGlobalDrafts, persistGlobalDraft, readGlobalDraftValue, @@ -2779,6 +2780,7 @@ function finishAppDraftWrite( ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const { content, message } = onSaved() ctx.toolCallbacks.setToolStatus(ctx.toolId, { content, result: 'Saved as draft' }) return JSON.stringify({ success: true, message }, null, 2) @@ -2791,6 +2793,7 @@ function finishDraftWrite( ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const stored = result.item const verb = existed ? 'Updated' : 'Created' // Don't echo the flow value back: the model just sent it in the write call, @@ -3898,6 +3901,16 @@ async function discardLocalDraft( await deleteGlobalDraft(workspace, type, path, triggerKind) + // The chat's touch on the item is undone — drop it from the mask so a + // pre-existing deployed item doesn't keep reading as this chat's edit. + const discardedKind = itemKindFor(type, triggerKind) + if (discardedKind) { + toolCallbacks.onItemDiscarded?.( + discardedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind) + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Discarded ${type} "${path}" draft`, result: 'Draft discarded' @@ -4237,6 +4250,9 @@ async function deployDraft( }) let actions: ToolDisplayAction[] | undefined + // Where the deploy actually lands — the app branch can resolve a different + // target from the draft's own path fields; the mask rename below must track it. + let deployedPath = path if (type === 'script' || type === 'flow') { // Promote the full persisted draft via the shared deploy module — the same @@ -4429,6 +4445,7 @@ async function deployDraft( throw e } } + deployedPath = targetPath if (await AppService.existsApp({ workspace, path: targetPath })) { // Omit custom_path on update for now. The backend preserves it when absent, while // sending it requires admin privileges; this chat deploy path does not yet mirror @@ -4478,6 +4495,18 @@ async function deployDraft( await deleteGlobalDraft(workspace, type, path, triggerKind, { preserveLiveDraft: true }) + // Move the chat's mask entry to the deployed path: a draft-only item's + // synthetic storage key never exists deployed, so the entry would otherwise + // stop matching anything after the draft is gone. + const deployedKind = itemKindFor(type, triggerKind) + if (deployedKind) { + toolCallbacks.onItemDeployed?.( + deployedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind), + deployedPath + ) + } + // Reload the session preview if it's open on the deployed item. Map the // deploy type to the preview kind — a raw app deploys under 'app' but the // preview addresses it as 'raw_app'; non-previewable types map to undefined. @@ -4548,6 +4577,17 @@ async function deleteWorkspaceItem( await deleteGlobalDraft(workspace, type, path, triggerKind) + // Record the deletion in the chat's modified-items mask. In a fork this leaves a + // reviewable "removed" diff vs the parent that stays scoped to this chat. Keyed + // by the same (itemKind, storagePath) as writes so it joins the draft/fork lists. + const deletedKind = itemKindFor(type, triggerKind) + if (deletedKind) { + toolCallbacks.onItemModified?.( + deletedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind) + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Deleted ${type} "${path}"`, result: 'Deleted' diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index 87ca767904..2717d41608 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -108,7 +108,7 @@ function clearEphemeralSecretVariableDraftValues(workspace: string): void { secretVariableDraftValues.delete(workspace) } -function itemKindFor( +export function itemKindFor( type: WorkspaceItemType, triggerKind?: TriggerKind ): UserDraftItemKind | undefined { @@ -377,10 +377,25 @@ export async function readGlobalDraftValue( return (await fetchBackendDraftValue(workspace, itemKind, storagePath)) as V | undefined } +// `itemKind` + `storagePath` are the canonical identity of the persisted draft +// (NOT item.path, which is the friendly display path). Callers use them to record +// the chat's modified-items mask. export type DraftPersistResult = - | { status: 'saved'; item: WorkspaceItem } - | { status: 'conflict'; item: WorkspaceItem; serverTimestamp?: string } - | { status: 'error'; item: WorkspaceItem; message: string } + | { status: 'saved'; item: WorkspaceItem; itemKind: UserDraftItemKind; storagePath: string } + | { + status: 'conflict' + item: WorkspaceItem + itemKind: UserDraftItemKind + storagePath: string + serverTimestamp?: string + } + | { + status: 'error' + item: WorkspaceItem + itemKind: UserDraftItemKind + storagePath: string + message: string + } // Persist a built draft value. `UserDraft.seed` reflects it into an open editor's // cell WITHOUT a double-POST (no-ops if no cell; its seedNextWrite suppresses the @@ -417,14 +432,20 @@ export async function persistGlobalDraft( // the chat "saved" while the DB-backed source of truth was never updated. const saveState = UserDraftDbSyncer.getState({ workspace, itemKind, path: storagePath }) if (saveState.state === 'failed') { - return { status: 'error', item, message: saveState.failureMessage ?? 'Draft save failed' } + return { + status: 'error', + item, + itemKind, + storagePath, + message: saveState.failureMessage ?? 'Draft save failed' + } } const conflict = opts.force ? undefined : UserDraftDbSyncer.getConflict({ workspace, itemKind, path: storagePath }).conflict return conflict - ? { status: 'conflict', item, serverTimestamp: conflict.serverTimestamp } - : { status: 'saved', item } + ? { status: 'conflict', item, itemKind, storagePath, serverTimestamp: conflict.serverTimestamp } + : { status: 'saved', item, itemKind, storagePath } } export async function getGlobalDraft( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 7abdc75168..c454f15fe6 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -3,6 +3,7 @@ import type { ChatCompletionMessageFunctionToolCall, ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import type { UserDraftItemKind } from '$lib/gen' /** * Special module IDs used throughout the flow system @@ -759,6 +760,15 @@ export interface ToolCallbacks { toolId: string, question: UserQuestionDisplay ) => Promise + /** Records a workspace item the tool call created/edited/deleted, by its + * canonical (itemKind, storagePath). Session chats wire this to accumulate the + * chat's modified-items mask; the global side-panel chat omits it (no-op). */ + onItemModified?: (itemKind: UserDraftItemKind, storagePath: string) => void + /** A tool deployed a draft: the mask entry moves from the draft's storage path + * to the deployed path (they differ for synthetic draft-only storage keys). */ + onItemDeployed?: (itemKind: UserDraftItemKind, storagePath: string, deployedPath: string) => void + /** A tool discarded a draft: the chat's touch on the item is undone. */ + onItemDiscarded?: (itemKind: UserDraftItemKind, storagePath: string) => void } export function createToolDef( diff --git a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte deleted file mode 100644 index 0b529b2056..0000000000 --- a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte +++ /dev/null @@ -1,204 +0,0 @@ - - - buildEditUrl(d as unknown as WorkspaceItemDiff, workspaceId)} -> - {#snippet titleExtra()} -
- - {ws?.name ?? workspaceId} -
- {/snippet} - diff --git a/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte b/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte deleted file mode 100644 index e03fc5a49c..0000000000 --- a/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte +++ /dev/null @@ -1,125 +0,0 @@ - - - buildEditUrl(d as unknown as WorkspaceItemDiff, forkWorkspaceId)} -> - {#snippet titleExtra()} -
- - {forkWs?.name ?? forkWorkspaceId} - - {parentWs?.name ?? parentWorkspaceId} - {#if comparison} - - {comparison.summary.total_diffs} item{comparison.summary.total_diffs !== 1 ? 's' : ''} - - {#if comparison.summary.conflicts > 0} - - - {comparison.summary.conflicts} conflict{comparison.summary.conflicts !== 1 ? 's' : ''} - - {/if} - {/if} -
- {/snippet} - diff --git a/frontend/src/lib/components/sessions/SessionChangesBar.svelte b/frontend/src/lib/components/sessions/SessionChangesBar.svelte new file mode 100644 index 0000000000..9f59d1f320 --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionChangesBar.svelte @@ -0,0 +1,264 @@ + + +{#snippet dock()} + +
+ {#if dockCounts.draft > 0} + diffDrawer?.open()}> + {dockCounts.draft} draft{dockCounts.draft === 1 ? '' : 's'} + + {/if} + {#if dockCounts.deployed > 0} + diffDrawer?.open()}> + {dockCounts.deployed} deployed + + {/if} + {#if deletionOnly && compareHref} + + + Review deletions on compare page + + + {/if} +
+{/snippet} + +{#if committedId && isUnavailable} + +
+
+ +
+ The {committedIsFork ? 'fork' : 'workspace'} has been archived or deleted + + Move this session to another workspace, or discard it. + {committedId} + +
+
+
+ onMove?.(workspaceId)} + onCreateFork={async (fork) => { + await onCreateForkAndMove?.(fork) + }} + createForkCaption="Created immediately and the session moved into it." + > + {#snippet trigger()} + + {/snippet} + + +
+
+{:else if showBar && committedId} + +
+
+ + Edits +
+ {@render dock()} +
+{/if} + + +{#if committedId && !isUnavailable} + + void runtime?.manager.renameModifiedItem(item.draftKind, item.path, item.displayPath)} + onItemDiscarded={(item) => void runtime?.manager.removeModifiedItem(item.draftKind, item.path)} + /> +{/if} diff --git a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte new file mode 100644 index 0000000000..1a6692048c --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte @@ -0,0 +1,107 @@ + + + + {#snippet titleExtra()} +
+ {#if isFork} + + + {ws?.name ?? workspaceId} + + + + {parentWs?.name ?? parentWorkspaceId} + + {:else} + + + {ws?.name ?? workspaceId} + + {/if} +
+ {/snippet} + diff --git a/frontend/src/lib/components/sessions/SessionDraftBar.svelte b/frontend/src/lib/components/sessions/SessionDraftBar.svelte deleted file mode 100644 index d23070151a..0000000000 --- a/frontend/src/lib/components/sessions/SessionDraftBar.svelte +++ /dev/null @@ -1,77 +0,0 @@ - - -{#if committedId && count > 0} -
-
- - - {count} draft{count === 1 ? '' : 's'} - {#snippet text()} - Tracks all unsaved draft changes in this workspace — including edits made outside this - chat (e.g. in the editor), not only changes made by the assistant. - {/snippet} - -
-
- drawer?.open()} /> - -
-
- - -{/if} diff --git a/frontend/src/lib/components/sessions/SessionForkBar.svelte b/frontend/src/lib/components/sessions/SessionForkBar.svelte deleted file mode 100644 index a0aafee456..0000000000 --- a/frontend/src/lib/components/sessions/SessionForkBar.svelte +++ /dev/null @@ -1,212 +0,0 @@ - - -{#if committedId && isUnavailable} - -
-
- -
- The {committedIsFork ? 'fork' : 'workspace'} has been archived or deleted - - Move this session to another workspace, or discard it. - {committedId} - -
-
-
- onMove?.(workspaceId)} - onCreateFork={async (fork) => { - await onCreateForkAndMove?.(fork) - }} - createForkCaption="Created immediately and the session moved into it." - > - {#snippet trigger()} - - {/snippet} - - -
-
-{:else if forksAllowed && isFork && sessionWorkspace && parentWorkspace && parentWorkspaceId && committedId} - {@const StatusIcon = - forkStatus === 'ahead' - ? GitPullRequestArrow - : forkStatus === 'diverged' - ? GitCompareArrows - : GitFork} - {@const statusColor = - forkStatus === 'ahead' - ? 'text-blue-500' - : forkStatus === 'diverged' - ? 'text-amber-500' - : 'text-secondary'} - {@const statusTitle = - forkStatus === 'ahead' - ? 'Ahead of parent' - : forkStatus === 'diverged' - ? 'Diverged from parent' - : forkStatus === 'in_sync' - ? 'In sync with parent' - : 'Fork'} -
-
- - - - - {sessionWorkspace.name} - - - - {parentWorkspace.name} - -
-
- diffDrawer?.open()} - /> - -
-
- - -{/if} diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index aad3de26c3..0342fd53e1 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -21,7 +21,6 @@ import { slide } from 'svelte/transition' import { createSession, - deriveForkStatus, deleteSessionsForWorkspace, isForkSession, reconcileAfterWorkspaceChange, @@ -53,15 +52,6 @@ import { sendUserToast } from '$lib/toast' import { currentWorkspaceRootId, workspaceRootId } from './sessionScope.svelte' - // Look up the cached fork comparison for a session through its runtime - // (if any). The deriveForkStatus helper handles the "no runtime yet" - // and "comparison not loaded" cases by returning undefined; we render - // a neutral fork icon in that interim, then upgrade to the proper - // status icon once the comparison lands. - function forkStatusFor(session: Session) { - return deriveForkStatus(session, $userWorkspaces, getRuntime(session.id)?.forkComparison.val) - } - function isForkFor(session: Session): boolean { return isForkSession(session, $userWorkspaces) } @@ -124,8 +114,8 @@ } // Flat list passing the archive + scope filters. Grouping for display happens - // in `sessionGroups`; this flat view drives the runtime / fork-comparison - // effects, the unread total, and keyboard navigation. + // in `sessionGroups`; this flat view drives the runtime effect, the unread + // total, and keyboard navigation. const visibleSessions = $derived( sessionState.sessions.filter((s) => { if (s.transient) return false @@ -211,23 +201,6 @@ } }) - // Pre-fetch the fork comparison for every visible fork session so the - // sidebar icons reflect the right ahead/diverged state without - // requiring the user to click into each session. Cheap enough at - // typical session counts; falls back to a plain dot until the - // fetch lands. - $effect(() => { - if (sectionCollapsed.val) return - for (const session of visibleSessions) { - if (!session.workspace_id) continue - const ws = $userWorkspaces.find((w) => w.id === session.workspace_id) - if (!ws?.parent_workspace_id) continue - const rt = getRuntime(session.id) - if (!rt) continue - void rt.ensureForkComparison(ws.parent_workspace_id, session.workspace_id) - } - }) - function isUnavailableFork(session: Session): boolean { return !!session.workspace_id && !$userWorkspaces.find((w) => w.id === session.workspace_id) } @@ -241,10 +214,6 @@ if (!isUnavailableFork(session)) { syncWorkspaceTo(session.workspace_id) } - // Refresh the fork diff count — users typically click back into a - // session after editing items elsewhere in the SPA, where neither - // the visibility-change nor the AI-loading signal would fire. - void getRuntime(session.id)?.refreshForkComparison() await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) if (restoreFocus) { // goto() resets focus to — put it back on the active session button @@ -436,7 +405,7 @@ {session.summary ?? 'Untitled session'} {#if draft || unread > 0} diff --git a/frontend/src/lib/components/sessions/SessionStatusDot.svelte b/frontend/src/lib/components/sessions/SessionStatusDot.svelte index b47816ab19..35028c80cc 100644 --- a/frontend/src/lib/components/sessions/SessionStatusDot.svelte +++ b/frontend/src/lib/components/sessions/SessionStatusDot.svelte @@ -3,19 +3,16 @@ AlertCircle, AlertTriangle, Building, - GitCompareArrows, GitFork, - GitPullRequestArrow, GitPullRequestClosed } from 'lucide-svelte' import type { SessionChatStatus } from './sessionRuntime.svelte' - import type { ForkStatus } from './sessionState.svelte' let { status, isFork, - forkStatus - }: { status: SessionChatStatus; isFork: boolean; forkStatus?: ForkStatus } = $props() + unavailable = false + }: { status: SessionChatStatus; isFork: boolean; unavailable?: boolean } = $props() const statusTooltip: Record = { idle: 'No chat activity', @@ -26,13 +23,6 @@ error: 'Last message had an error' } - const forkTooltip: Record = { - in_sync: 'Fork — in sync with parent', - ahead: 'Fork — ahead of parent', - diverged: 'Fork — diverged from parent', - unavailable: 'Fork — no longer available' - } - // Live chat signals take precedence over the persistent kind/fork // indicator: streaming, needs-confirmation, and error are time-critical // and warrant briefly hijacking the icon slot. @@ -41,7 +31,11 @@ ) const persistentTitle = $derived( - isFork ? (forkStatus ? forkTooltip[forkStatus] : 'Fork session') : 'Root workspace session' + isFork + ? unavailable + ? 'Fork — no longer available' + : 'Fork session' + : 'Root workspace session' ) const title = $derived(liveOverride ? statusTooltip[status] : persistentTitle) @@ -59,11 +53,7 @@ {:else if status === 'error'} {:else if isFork} - {#if forkStatus === 'ahead'} - - {:else if forkStatus === 'diverged'} - - {:else if forkStatus === 'unavailable'} + {#if unavailable} {:else} diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index c9fdb6454f..ee8cbed6e0 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -30,8 +30,7 @@ import RawAppEditorView from './RawAppEditorView.svelte' import PipelineEditorView from './PipelineEditorView.svelte' import SessionWorkspaceBar from './SessionWorkspaceBar.svelte' - import SessionForkBar from './SessionForkBar.svelte' - import SessionDraftBar from './SessionDraftBar.svelte' + import SessionChangesBar from './SessionChangesBar.svelte' import { createSession, deleteSessionsForWorkspace, @@ -235,7 +234,7 @@ // True when the session committed to a workspace that's no longer in // the user's list (deleted / archived / access revoked). The chat is - // disabled and SessionForkBar shows a move/discard banner. + // disabled and SessionChangesBar shows a move/discard banner. const isUnavailable = $derived( !!session?.workspace_id && !$userWorkspaces.find((w) => w.id === session!.workspace_id) ) @@ -274,16 +273,16 @@ {#if !hasFirstUserMessage} {/if} - +
{#if session.archived && !isUnavailable}
{/if} - moveAndActivate(workspaceId)} onCreateForkAndMove={(fork) => createForkAndMove(fork)} onArchive={() => archiveAndReset()} onDelete={() => (deleteConfirmOpen = true)} /> -
{/snippet} diff --git a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte index e6b4e5fd2f..50c6653b0a 100644 --- a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte @@ -1,45 +1,19 @@ - - searchableText(d)} + items={displayEntries} + bind:filteredItems={searchedEntries} + f={(e: DisplayEntry) => searchableText(e)} /> -{#snippet renderTreeNode(node: TreeNode, depth: number)} + +{#snippet rowBadge(item: DeployItem)} + {#if badgeOf(item) === 'draft'} + {#if model.staleOf(item.key)} + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+ Started from an older deployed version. A newer version was deployed after this draft + began. Review the latest deploy before deploying. +
+ {/snippet} + + {/if} + + {item.draftOnly ? 'Draft only' : 'Draft'} + + {:else} + + + + + + + {/if} +{/snippet} + + +{#snippet deployFailed(item: DeployItem)} + {@const s = model.statusOf(item.key)} + {#if s?.status === 'failed'} + + Failed + + + {/if} +{/snippet} + +{#snippet renderTreeNode(node: TreeNode, depth: number)} {#if node.type === 'folder'} {@const isUserScope = node.isScope && node.name.startsWith('u/')} {@const fkey = node.key} @@ -417,36 +613,62 @@ {@const isHl = fkey === highlightedKey}
(folderOpen[fkey] = (e.currentTarget as HTMLDetailsElement).open)} + ontoggle={(e) => { + // Record real user toggles only; skip the echo fired when the `open` + // attribute is driven by state (search force-open, expandApp). + const domOpen = (e.currentTarget as HTMLDetailsElement).open + if (domOpen !== isFolderOpen(fkey)) folderOpen[fkey] = domOpen + }} class="select-none" > {#if node.app} - {@const appSummary = summaries[node.app.summaryKey] ?? node.app.summary} - + {@const appItem = segmentItems.find((it) => it.key === node.app?.summaryKey)} + setHoverHighlight(fkey)} + onclick={(e) => { + e.preventDefault() + if (appItem) revealDiff(appItem, fkey) + }} title={node.fullPath} - class="flex items-center gap-1.5 px-3 py-1.5 cursor-pointer hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl + class="flex items-center gap-2 pl-3 pr-1 py-2 rounded-md cursor-pointer hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl ? 'bg-surface-hover' : ''}" style="padding-left: {depth * 12 + 8}px" > - {appSummary ?? node.name} + {node.app.summary ?? node.name} - - + {#if appItem} + {@render rowBadge(staged[appItem.key] ?? appItem)} + {/if} + + {:else} setHoverHighlight(fkey)} - class="flex items-center gap-1.5 px-3 py-1.5 cursor-pointer text-xs font-normal font-mono text-secondary hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl + class="flex items-center gap-2 pl-3 pr-1 py-2 rounded-md cursor-pointer text-xs font-normal font-mono text-secondary hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl ? 'bg-surface-hover' : ''}" style="padding-left: {depth * 12 + 8}px" @@ -465,14 +687,13 @@ {/if} {node.name} - - + + + + {/if}
-
- { - highlightedKey = key - scrollToDiff(d) - }} - onmouseenter={() => setHoverHighlight(key)} + +
+ {#if isSynthetic(d)} + void revealSynthetic(d, key)} + onmouseenter={() => setHoverHighlight(key)} + > + {#snippet extras()} + + {/snippet} + + {:else} + revealDiff(d, key)} + onmouseenter={() => setHoverHighlight(key)} + > + {#snippet extras()} + {@render rowBadge(staged[d.key] ?? d)} + {#if d.deployKind === 'raw_app'} + + { + e.stopPropagation() + expandApp(d) + }} + > + {#if loadedDiffs[d.key]?.state === 'loading'} + + {:else} + + {/if} + + {:else} + + {/if} + {/snippet} + + {/if} +
+ {/if} +{/snippet} + + +{#snippet diffBlock(item: DeployItem)} + {@const loaded = loadedDiffs[item.key]} + {#if !mountedRows[item.key]} +
+ + Diff loads on scroll… +
+ {:else if !loaded || loaded.state === 'loading'} +
+ + Loading diff… +
+ {:else if loaded.state === 'error'} +
{loaded.error}
+ {:else if item.deployKind === 'raw_app'} +
+ {#each rawAppItems(item, loaded) as sub (displayKey(sub))} + +
+
+ {sub.path} +
+ {#if sub.kind === 'raw_app_file'} + + {:else} + {@const runnable = sub as RawAppRunnableItem} + + {/if} +
+ {/each} +
+ {:else} + {/if} {/snippet} @@ -540,178 +869,215 @@ /> {/snippet} - {/snippet} -
- {#if diffs.length > 0} - - {/if} -
-
- {#if loading && diffs.length === 0} -
- - Loading comparison... +
+
+ {#if model.items.length > 0} + + {/if} +
+
+ {#if model.loading && model.items.length === 0} +
+ + Loading changes... +
+ {:else if model.error} +
{model.error}
+ {:else if model.items.length === 0} +
No changes.
+ {:else if orderedItems.length === 0} +
No files match.
+ {:else} +
+ {#each orderedItems as d (d.key)} + + {@const view = staged[d.key] ?? d} + {@const action = actionFor(view)} + {@const editUrl = editUrlFor?.(d)} + {@const status = model.statusOf(d.key)} +
- - -
- {#if editUrl} - + +
+ {#if editUrl} + + {d.displayPath} + + {:else} +
+ {d.displayPath} +
+ {/if} +
+
+ {@render rowBadge(view)} + {#if status?.status === 'failed'} + {@render deployFailed(d)} + {:else if action.op !== 'none'} + {#if action.secondary?.length} + + {/if} +
+ + {#if staged[d.key]} + +
+ +
+ {/if} +
+ {/if} +
+
+
+ + {#if view.done} +
- {dpath} - + Deployed — no pending changes. +
{:else} -
- {dpath} +
+ {@render diffBlock(view)}
{/if}
-
- {#if d.ahead && d.ahead > 0} - {d.ahead} ahead - {/if} - {#if d.behind && d.behind > 0} - {d.behind} behind - {/if} - - - {status} - -
- -
- {#if !mountedRows[key]} - -
- - Diff loads on scroll… -
- {:else if d.kind === 'raw_app_file'} - - {@const rawFile = d as RawAppFileItem} - - {:else if 'appPath' in d} - - {@const runnable = d as RawAppRunnableItem} - - {:else if !loaded || loaded.state === 'loading'} -
- - Loading diff… -
- {:else if loaded.state === 'error'} -
{loaded.error}
- {:else if loaded.state === 'ready'} - - {/if}
-
- {/each} -
- {/if} - + {/each} + + + + {/if} + + + + + {#if model.items.length > 0 && compareSessionHref} +
+ + Deploy all / request review on compare page + + +
+ {/if} + + + +
+ + +
diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 7d639b2c75..5b86f7dbad 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -173,15 +173,24 @@ let topbarWidth = $state(0) const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 900) - // AutosaveIndicator watch key. Falls back to the full-page editor's - // global store + URL draft path; the sessions preview overrides both so the - // icon tracks the session's (forked) workspace + target path where autosave - // actually happens. - const indicatorWorkspace = $derived(autosaveWorkspace ?? $workspaceStore) + // The workspace this editor operates on: deploy, save-draft, trigger loading + // and the AutosaveIndicator all target it. Falls back to the full-page + // editor's global store; the sessions preview overrides it to the session's + // (forked) workspace, so an embedded editor acts on the session's fork rather + // than the navigation workspace ($workspaceStore, which stays put). indicatorPath + // is the matching draft path (URL path full-page, session target in preview). + const opWorkspace = $derived(autosaveWorkspace ?? $workspaceStore) const indicatorPath = $derived(autosavePath ?? userDraftPath) + // The shared `workerTags` store caches tags for the navigation workspace. A + // session editor deploys to `opWorkspace` (a fork), so it keeps a local list to + // gate/populate the tag picker without reading or clobbering the shared cache. + const usesLocalTags = $derived(opWorkspace != undefined && opWorkspace !== $workspaceStore) + let localWorkerTags = $state(undefined) + const scriptWorkerTags = $derived(usesLocalTags ? localWorkerTags : $workerTags) + function getCompactMenuItems(): Item[] { - const hasTags = ($workerTags?.length ?? 0) > 0 + const hasTags = (scriptWorkerTags?.length ?? 0) > 0 return [ ...(customUi?.topBar?.tagEdit != false && hasTags ? [ @@ -285,13 +294,13 @@ return } $triggersCount = await ScriptService.getTriggersCountOfScript({ - workspace: $workspaceStore!, + workspace: opWorkspace!, path: initialPath }) await triggersState.fetchTriggers( triggersCount, - $workspaceStore, + opWorkspace, initialPath, false, $primaryScheduleStore, @@ -443,7 +452,7 @@ } try { const templateScript = await PostgresTriggerService.getTemplateScript({ - workspace: $workspaceStore!, + workspace: opWorkspace!, id: templateId }) return templateScript @@ -497,7 +506,7 @@ if (initialPath && initialPath != '') { actual_parent_hash = ( await ScriptService.getScriptLatestVersion({ - workspace: $workspaceStore!, + workspace: opWorkspace!, path: initialPath }) )?.script_hash @@ -544,7 +553,7 @@ async function syncWithDeployed() { const latestScript = await ScriptService.getScriptByPath({ - workspace: $workspaceStore!, + workspace: opWorkspace!, path: initialPath, withStarredInfo: true }) @@ -608,7 +617,7 @@ } const newHash = await ScriptService.createScript({ - workspace: $workspaceStore!, + workspace: opWorkspace!, requestBody: { path: script.path, summary: script.summary, @@ -653,16 +662,16 @@ // New/updated path now exists server-side — drop the autocomplete // cache so it shows up immediately instead of after the 60s TTL. - invalidateWorkspacePaths($workspaceStore!) + invalidateWorkspacePaths(opWorkspace!) // Authoritative save-time schema-contract check (pipelines gap #2b): // warn-only, post-commit so a self-produced target resolves to the // content just deployed. Fire-and-forget — must never gate the deploy. - notifyContractWarnings($workspaceStore!, script.language, script.content) + notifyContractWarnings(opWorkspace!, script.language, script.content) if (!initialPath) { await CaptureService.moveCapturesAndConfigs({ - workspace: $workspaceStore!, + workspace: opWorkspace!, path: fakeInitialPath, requestBody: { new_path: script.path @@ -674,7 +683,7 @@ if (triggersToDeploy) { await deployTriggers( triggersToDeploy, - $workspaceStore, + opWorkspace, !!$userStore?.is_admin || !!$userStore?.is_super_admin, usedTriggerKinds, script.path, @@ -718,11 +727,11 @@ // syncer flushes. No toast — the AutosaveIndicator narrates the result, and // `flush` never rejects (postSave routes errors to the failures map). async function saveDraft(): Promise { - if (!$workspaceStore || !userDraftPath) return + if (!opWorkspace || !userDraftPath) return editor?.flushPendingChanges() await tick() await UserDraftDbSyncer.flush({ - workspace: $workspaceStore, + workspace: opWorkspace, itemKind: 'script', path: userDraftPath }) @@ -786,10 +795,10 @@ window.open(`/scripts/add?template=${initialPath}`) } }, - ...(!isCloudHosted() && editInForkAllowed($workspaceStore, $userWorkspaces) + ...(!isCloudHosted() && editInForkAllowed(opWorkspace, $userWorkspaces) ? [ { - label: editInForkLabel($workspaceStore, $userWorkspaces), + label: editInForkLabel(opWorkspace, $userWorkspaces), onClick: () => { window.open(buildForkEditUrl('script', initialPath)) } @@ -990,8 +999,12 @@ loadWorkerTags() async function loadWorkerTags() { - if (!$workerTags) { - $workerTags = await WorkerService.getCustomTagsForWorkspace({ workspace: $workspaceStore! }) + if (usesLocalTags) { + if (!localWorkerTags) { + localWorkerTags = await WorkerService.getCustomTagsForWorkspace({ workspace: opWorkspace! }) + } + } else if (!$workerTags) { + $workerTags = await WorkerService.getCustomTagsForWorkspace({ workspace: opWorkspace! }) } } @@ -1748,7 +1761,7 @@ /> {#if script.on_behalf_of_email && canPreserve} → { @@ -1906,13 +1919,14 @@ kind="script" summaryEditable={customUi?.topBar?.editableSummary != false} pathEditable={customUi?.topBar?.editablePath != false} + workspaceId={autosaveWorkspace} onNavigate={(item) => onNavigate?.(item)} /> {/if} - {#if indicatorWorkspace} + {#if opWorkspace} 0} + {#if scriptWorkerTags} + {#if scriptWorkerTags?.length ?? 0 > 0}
{/if} @@ -2056,6 +2071,15 @@
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` \u2014 write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, \u2026) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table.\n\n`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace \u2014 do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy:\n\n- no option \u2192 **replace** the whole table each run (full refresh; the only mode whose output columns may change);\n- `// materialize append` \u2192 INSERT-append rows (incremental);\n- `// materialize key=` \u2192 merge/upsert on ``.\n\n`// materialize manual ` opts **out** of managed writes \u2014 the script writes its own DDL and the annotation only records the output asset for lineage.\n\n`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time.\n\n`materialize` is an output **declaration** on a node \u2014 not a command. There is no \"materialize run\".\n\n## How to build one in chat\n\n1. Put every node in the **same folder**: `f//`. The folder is the pipeline.\n2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`.\n3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output.\n4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones.\n5. Leave nodes as drafts unless the user asks to deploy. A pipeline only \"runs\" once its scripts are deployed and their triggers exist.\n\nWhen the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above.\n\n## Example (DuckDB \u2192 DuckLake, scheduled ingest + downstream transform)\n\nNode `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table):\n\n```sql\n-- pipeline\n-- on schedule\n-- materialize ducklake://main/orders\nSELECT * FROM read_csv('s3://raw/orders/*.csv')\n```\n\nNode `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup):\n\n```sql\n-- pipeline\n-- on ducklake://main/orders\n-- materialize ducklake://main/orders_daily\nSELECT date_trunc('day', ts) AS day, count(*) AS n\nFROM ducklake.main.orders GROUP BY 1\n```\n"; +export declare const WORKFLOW_AS_CODE_BASE = "# Windmill Workflow-as-Code Writing Guide\n\n## Scope\n\nUse this guide when writing or modifying Windmill Workflow-as-Code (WAC) scripts.\nWAC is authored as a Windmill script and deployed with the normal script workflow. It is not an OpenFlow YAML flow.\n\nSupported WAC authoring targets:\n- Bun TypeScript scripts that import from `windmill-client`\n- Python 3 scripts that import from `wmill`\n\n## File Shape\n\nBun TypeScript:\n\n```typescript\nimport {\n task,\n taskScript,\n taskFlow,\n step,\n sleep,\n waitForApproval,\n getResumeUrls,\n parallel,\n workflow,\n} from \"windmill-client\";\n\nconst process = task(async (x: string): Promise => {\n return `processed: ${x}`;\n});\n\nexport const main = workflow(async (x: string) => {\n const result = await process(x);\n return { result };\n});\n```\n\nPython:\n\n```python\nfrom wmill import task, task_script, task_flow, step, sleep, wait_for_approval, get_resume_urls, parallel, workflow\n\n@task()\nasync def process(x: str) -> str:\n return f\"processed: {x}\"\n\n@workflow\nasync def main(x: str):\n result = await process(x)\n return {\"result\": result}\n```\n\nRules:\n- Do not call `main`.\n- Bun TypeScript should export the workflow entrypoint, preferably `export const main = workflow(async (...) => { ... })`.\n- Python must use `@workflow` on an async top-level function, usually `main`.\n- Define task functions and `taskScript`/`task_script` or `taskFlow`/`task_flow` assignments at module top level with stable names.\n- Use the exact SDK names. Do not alias `workflow`, `task`, `taskScript`, `taskFlow`, `step`, `sleep`, `waitForApproval`, `task_script`, `task_flow`, or `wait_for_approval`; the WAC parser recognizes these names directly.\n\n## Checkpoint And Replay Model\n\nThe parent workflow may rerun from the top after any suspension, retry, approval, or child task completion. Completed durable steps are replayed from the checkpoint.\n\nPut every side effect or non-deterministic value behind a durable WAC boundary:\n- Use `task()` / `@task()` for substantial work that should run as its own child job.\n- Use `taskScript()` / `task_script()` for an existing script or a relative module file.\n- Use `taskFlow()` / `task_flow()` for an existing Windmill flow.\n- Use `step(name, fn)` for lightweight inline work whose result must be checkpointed.\n- Use `sleep(seconds)` for server-side sleeps that do not hold a worker.\n- Use `waitForApproval()` / `wait_for_approval()` for external approval suspension.\n\nNever put API calls, database writes, notifications, random values, timestamps, or irreversible changes directly in the top-level workflow body. The workflow body can be rerun. Put those operations in a task or in `step()`.\n\nBranching on task or step results is safe because those results are checkpointed. Branching on current time, random data, environment reads, or external state is unsafe unless the value is first captured with `step()`.\n\n## Tasks\n\nUse `task()` / `@task()` for inline functions that become workflow steps:\n\n```typescript\nconst enrich = task(async (customerId: string) => {\n return await fetchCustomer(customerId);\n});\n```\n\n```python\n@task(timeout=600, tag=\"etl\")\nasync def enrich(customer_id: str):\n return await fetch_customer(customer_id)\n```\n\nIn TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with `@task()` or `@task`.\n\nFor existing scripts:\n\n```typescript\nconst helper = taskScript(\"./helper.ts\");\nconst existing = taskScript(\"f/data/extract\", { timeout: 600 });\nconst value = await helper({ input: x });\n```\n\n```python\nhelper = task_script(\"./helper.py\")\nexisting = task_script(\"f/data/extract\", timeout=600)\nvalue = await helper(input=x)\n```\n\nFor existing flows:\n\n```typescript\nconst pipeline = taskFlow(\"f/etl/pipeline\");\nconst output = await pipeline({ input: data });\n```\n\n```python\npipeline = task_flow(\"f/etl/pipeline\")\noutput = await pipeline(input=data)\n```\n\n## Inline Steps\n\nUse `step()` for lightweight inline values that must not change during replay:\n\n```typescript\nconst urls = await step(\"get_urls\", () => getResumeUrls());\nconst startedAt = await step(\"started_at\", () => new Date().toISOString());\n```\n\n```python\nurls = await step(\"get_urls\", lambda: get_resume_urls())\n```\n\nUse stable, descriptive step names. Do not generate step names dynamically.\n\n## Parallelism\n\nTo run independent work in parallel, start task promises/coroutines before awaiting them together:\n\n```typescript\nconst [a, b] = await Promise.all([process(\"a\"), process(\"b\")]);\nconst many = await parallel(items, process, { concurrency: 5 });\n```\n\n```python\nimport asyncio\n\na, b = await asyncio.gather(process(\"a\"), process(\"b\"))\nmany = await parallel(items, process, concurrency=5)\n```\n\nOnly parallelize independent steps. Do not read the result of a task before it is awaited.\n\n## Approvals\n\nGenerate resume URLs inside `step()` before sending them:\n\n```typescript\nconst urls = await step(\"get_urls\", () => getResumeUrls());\nawait step(\"notify\", () => sendApprovalEmail(urls.approvalPage));\nconst approval = await waitForApproval({ timeout: 3600 });\n```\n\n```python\nurls = await step(\"get_urls\", lambda: get_resume_urls())\nawait step(\"notify\", lambda: send_approval_email(urls[\"approvalPage\"]))\napproval = await wait_for_approval(timeout=3600)\n```\n\n`selfApproval: false` and `self_approval=False` are Enterprise-only approval behavior. Do not use them unless the user asks for that behavior.\n\n## Error Handling\n\nLet task errors fail the workflow unless the user asks for recovery logic.\n\nPython: `except Exception` is safe around WAC calls because internal suspension inherits from `BaseException`. Avoid bare `except:` in workflow code. If the user asks for recovery logic around failed child work, catch `TaskError` from `wmill` for task failures.\n\nTypeScript: avoid broad `try/catch` around WAC SDK calls. The SDK uses an internal suspension error during initial dispatch; catching it can break workflow suspension. If a broad catch is unavoidable, rethrow internal suspension errors before handling business errors.\n"; +export declare const FLOW_CHAT_SPECIAL_MODULES = "## Special Modules\n\n- Use `set_preprocessor_module` to add, replace, or remove the top-level `value.preprocessor_module`\n- Use `set_failure_module` to add, replace, or remove the top-level `value.failure_module`\n- Use `set_flow_json` only when you are replacing the whole flow, including normal modules and optional special modules\n\n**Example - Update only the special modules:**\n```javascript\nset_preprocessor_module({\n module: JSON.stringify({\n id: \"preprocessor\",\n value: {\n type: \"rawscript\",\n language: \"bun\",\n content: \"export async function preprocessor(payload: string) { const trimmed = payload.trim(); if (!trimmed) { throw new Error('payload must not be empty'); } return { payload: trimmed }; }\",\n input_transforms: {\n payload: { type: \"javascript\", expr: \"flow_input.payload\" }\n }\n }\n })\n})\n\nset_failure_module({\n module: JSON.stringify({\n id: \"failure\",\n value: {\n type: \"rawscript\",\n language: \"bun\",\n content: \"export async function main(message: string, name: string, step_id: string) { return { message, name, step_id }; }\",\n input_transforms: {\n message: { type: \"javascript\", expr: \"error.message\" },\n name: { type: \"javascript\", expr: \"error.name\" },\n step_id: { type: \"javascript\", expr: \"error.step_id\" }\n }\n }\n })\n})\n```\n"; +export declare const SDK_TYPESCRIPT = "# TypeScript SDK (windmill-client)\n\nImport: import * as wmill from 'windmill-client'\n\nworkerHasInternalServer(): boolean\n\n/**\n * Initialize the Windmill client with authentication token and base URL\n * @param token - Authentication token (defaults to WM_TOKEN env variable)\n * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable)\n */\nsetClient(token?: string, baseUrl?: string): void\n\n/**\n * Create a client configuration from env variables\n * @returns client configuration\n */\ngetWorkspace(): string\n\n/**\n * Get a resource value by path\n * @param path path of the resource, default to internal state path\n * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error\n * @returns resource value\n */\nasync getResource(path?: string, undefinedIfEmpty?: boolean): Promise\n\n/**\n * Get the true root job id\n * @param jobId job id to get the root job id from (default to current job)\n * @returns root job id\n */\nasync getRootJobId(jobId?: string): Promise\n\n/**\n * @deprecated Use runScriptByPath or runScriptByHash instead\n */\nasync runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Run a script synchronously by its path and wait for the result\n * @param path - Script path in Windmill\n * @param args - Arguments to pass to the script\n * @param verbose - Enable verbose logging\n * @param tag - Override the worker tag the job runs on\n * @returns Script execution result\n */\nasync runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Run a script synchronously by its hash and wait for the result\n * @param hash_ - Script hash in Windmill\n * @param args - Arguments to pass to the script\n * @param verbose - Enable verbose logging\n * @param tag - Override the worker tag the job runs on\n * @returns Script execution result\n */\nasync runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Append a text to the result stream\n * @param text text to append to the result stream\n */\nappendToResultStream(text: string): void\n\n/**\n * Stream to the result stream\n * @param stream stream to stream to the result stream\n */\nasync streamResult(stream: AsyncIterable): Promise\n\n/**\n * Run a flow synchronously by its path and wait for the result\n * @param path - Flow path in Windmill\n * @param args - Arguments to pass to the flow\n * @param verbose - Enable verbose logging\n * @param tag - Override the worker tag the job runs on\n * @returns Flow execution result\n */\nasync runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Wait for a job to complete and return its result\n * @param jobId - ID of the job to wait for\n * @param verbose - Enable verbose logging\n * @returns Job result when completed\n */\nasync waitJob(jobId: string, verbose: boolean = false): Promise\n\n/**\n * Get the result of a completed job\n * @param jobId - ID of the completed job\n * @returns Job result\n */\nasync getResult(jobId: string): Promise\n\n/**\n * Get the result of a job if completed, or its current status\n * @param jobId - ID of the job\n * @returns Object with started, completed, success, and result properties\n */\nasync getResultMaybe(jobId: string): Promise\n\n/**\n * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead\n */\nasync runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise\n\n/**\n * Run a script asynchronously by its path\n * @param path - Script path in Windmill\n * @param args - Arguments to pass to the script\n * @param scheduledInSeconds - Schedule execution for a future time (in seconds)\n * @param tag - Override the worker tag the job runs on\n * @returns Job ID of the created job\n */\nasync runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise\n\n/**\n * Run a script asynchronously by its hash\n * @param hash_ - Script hash in Windmill\n * @param args - Arguments to pass to the script\n * @param scheduledInSeconds - Schedule execution for a future time (in seconds)\n * @param tag - Override the worker tag the job runs on\n * @returns Job ID of the created job\n */\nasync runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise\n\n/**\n * Run a flow asynchronously by its path\n * @param path - Flow path in Windmill\n * @param args - Arguments to pass to the flow\n * @param scheduledInSeconds - Schedule execution for a future time (in seconds)\n * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job)\n * @param tag - Override the worker tag the job runs on\n * @returns Job ID of the created job\n */\nasync runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true, tag: string | null = null): Promise\n\n/**\n * Resolve a resource value in case the default value was picked because the input payload was undefined\n * @param obj resource value or path of the resource under the format `$res:path`\n * @returns resource value\n */\nasync resolveDefaultResource(obj: any): Promise\n\n/**\n * Get the state file path from environment variables\n * @returns State path string\n */\ngetStatePath(): string\n\n/**\n * Set a resource value by path\n * @param path path of the resource to set, default to state path\n * @param value new value of the resource to set\n * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type\n */\nasync setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise\n\n/**\n * Set the state\n * @param state state to set\n * @deprecated use setState instead\n */\nasync setInternalState(state: any): Promise\n\n/**\n * Set the state\n * @param state state to set\n * @param path Optional state resource path override. Defaults to `getStatePath()`.\n */\nasync setState(state: any, path?: string): Promise\n\n/**\n * Set the progress\n * Progress cannot go back and limited to 0% to 99% range\n * @param percent Progress to set in %\n * @param jobId? Job to set progress for\n */\nasync setProgress(percent: number, jobId?: any): Promise\n\n/**\n * Get the progress\n * @param jobId? Job to get progress from\n * @returns Optional clamped between 0 and 100 progress value\n */\nasync getProgress(jobId?: any): Promise\n\n/**\n * Set a flow user state\n * @param key key of the state\n * @param value value of the state\n */\nasync setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise\n\n/**\n * Get a flow user state\n * @param path path of the variable\n */\nasync getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise\n\n/**\n * Get the internal state\n * @deprecated use getState instead\n */\nasync getInternalState(): Promise\n\n/**\n * Get the state shared across executions\n * @param path Optional state resource path override. Defaults to `getStatePath()`.\n */\nasync getState(path?: string): Promise\n\n/**\n * Get a variable by path\n * @param path path of the variable\n * @returns variable value\n */\nasync getVariable(path: string): Promise\n\n/**\n * Set a variable by path, create if not exist\n * @param path path of the variable\n * @param value value of the variable\n * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false)\n * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: \"\")\n */\nasync setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise\n\n/**\n * Build a PostgreSQL connection URL from a database resource\n * @param path - Path to the database resource\n * @returns PostgreSQL connection URL string\n */\nasync databaseUrlFromResource(path: string): Promise\n\nasync polarsConnectionSettings(s3_resource_path: string | undefined): Promise\n\nasync duckdbConnectionSettings(s3_resource_path: string | undefined): Promise\n\n/**\n * Get S3 client settings from a resource or workspace default\n * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined)\n * @param workspace - Workspace to read from (defaults to the `WM_WORKSPACE` env var)\n * @returns S3 client configuration settings\n */\nasync denoS3LightClientSettings(s3_resource_path: string | undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.\n * \n * ```typescript\n * let fileContent = await wmill.loadS3FileContent(inputFile)\n * // if the file is a raw text file, it can be decoded and printed directly:\n * const text = new TextDecoder().decode(fileContentStream)\n * console.log(text);\n * ```\n * \n * @param workspace - Workspace to read from (defaults to the `WM_WORKSPACE` env var)\n */\nasync loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.\n * \n * ```typescript\n * let fileContentBlob = await wmill.loadS3FileStream(inputFile)\n * // if the content is plain text, the blob can be read directly:\n * console.log(await fileContentBlob.text());\n * ```\n * \n * @param workspace - Workspace to read from (defaults to the `WM_WORKSPACE` env var)\n */\nasync loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.\n * \n * ```typescript\n * const s3object = await writeS3File(s3Object, \"Hello Windmill!\")\n * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8')\n * console.log(fileContentAsUtf8Str)\n * ```\n * \n * @param workspace - Workspace to write to (defaults to the `WM_WORKSPACE` env var)\n */\nasync writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Permanently delete a file from S3 by key.\n * \n * ```typescript\n * await wmill.deleteS3File({ s3: \"path/to/file.txt\" })\n * ```\n * \n * @param s3object - S3 object identifying the file to delete (must have `s3` set)\n * @param workspace - Workspace to delete from (defaults to the `WM_WORKSPACE` env var)\n */\nasync deleteS3File(s3object: S3Object, workspace: string | undefined = undefined): Promise\n\n/**\n * Sign S3 objects to be used by anonymous users in public apps\n * @param s3objects s3 objects to sign\n * @returns signed s3 objects\n */\nasync signS3Objects(s3objects: S3Object[]): Promise\n\n/**\n * Sign S3 object to be used by anonymous users in public apps\n * @param s3object s3 object to sign\n * @returns signed s3 object\n */\nasync signS3Object(s3object: S3Object): Promise\n\n/**\n * Generate a presigned public URL for an array of S3 objects.\n * If an S3 object is not signed yet, it will be signed first.\n * @param s3Objects s3 objects to sign\n * @returns list of signed public URLs\n */\nasync getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise\n\n/**\n * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first.\n * @param s3Object s3 object to sign\n * @returns signed public URL\n */\nasync getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise\n\n/**\n * Get URLs needed for resuming a flow after this step\n * @param approver approver name\n * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step.\n * This allows pre-approvals that can be consumed by any later suspend step in the same flow.\n * @returns approval page UI URL, resume and cancel API URLs for resuming the flow\n */\nasync getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{\n approvalPage: string;\n resume: string;\n cancel: string;\n}>\n\n/**\n * @deprecated use getResumeUrls instead\n */\ngetResumeEndpoints(approver?: string): Promise<{\n approvalPage: string;\n resume: string;\n cancel: string;\n}>\n\n/**\n * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only)\n * @param audience audience of the token\n * @param expiresIn Optional number of seconds until the token expires\n * @returns jwt token\n */\nasync getIdToken(audience: string, expiresIn?: number): Promise\n\n/**\n * Convert a base64-encoded string to Uint8Array\n * @param data - Base64-encoded string\n * @returns Decoded Uint8Array\n */\nbase64ToUint8Array(data: string): Uint8Array\n\n/**\n * Convert a Uint8Array to base64-encoded string\n * @param arrayBuffer - Uint8Array to encode\n * @returns Base64-encoded string\n */\nuint8ArrayToBase64(arrayBuffer: Uint8Array): string\n\n/**\n * Get email from workspace username\n * This method is particularly useful for apps that require the email address of the viewer.\n * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app.\n * @param username\n * @returns email address\n */\nasync usernameToEmail(username: string): Promise\n\n/**\n * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields.\n * \n * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form**\n * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form).\n * \n * @param {Object} options - The configuration options for the Slack approval request.\n * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill.\n * @param {string} options.channelId - The Slack channel ID where the approval request will be sent.\n * @param {string} [options.message] - Optional custom message to include in the Slack approval request.\n * @param {string} [options.approver] - Optional user ID or name of the approver for the request.\n * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field.\n * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field.\n * @param {string} [options.resumeButtonText] - Optional text for the resume button.\n * @param {string} [options.cancelButtonText] - Optional text for the cancel button.\n * \n * @returns {Promise} Resolves when the Slack approval request is successfully sent.\n * \n * @throws {Error} If the function is not called within a flow or flow preview.\n * @throws {Error} If the `JobService.getSlackApprovalPayload` call fails.\n * \n * **Usage Example:**\n * ```typescript\n * await requestInteractiveSlackApproval({\n * slackResourcePath: \"/u/alex/my_slack_resource\",\n * channelId: \"admins-slack-channel\",\n * message: \"Please approve this request\",\n * approver: \"approver123\",\n * defaultArgsJson: { key1: \"value1\", key2: 42 },\n * dynamicEnumsJson: { foo: [\"choice1\", \"choice2\"], bar: [\"optionA\", \"optionB\"] },\n * resumeButtonText: \"Resume\",\n * cancelButtonText: \"Cancel\",\n * });\n * ```\n * \n * **Note:** This function requires execution within a Windmill flow or flow preview.\n */\nasync requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, resumeButtonText, cancelButtonText, }: SlackApprovalOptions): Promise\n\n/**\n * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields.\n * \n * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form**\n * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form).\n * \n * @param {Object} options - The configuration options for the Teams approval request.\n * @param {string} options.teamName - The Teams team name where the approval request will be sent.\n * @param {string} options.channelName - The Teams channel name where the approval request will be sent.\n * @param {string} [options.message] - Optional custom message to include in the Teams approval request.\n * @param {string} [options.approver] - Optional user ID or name of the approver for the request.\n * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field.\n * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field.\n * \n * @returns {Promise} Resolves when the Teams approval request is successfully sent.\n * \n * @throws {Error} If the function is not called within a flow or flow preview.\n * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails.\n * \n * **Usage Example:**\n * ```typescript\n * await requestInteractiveTeamsApproval({\n * teamName: \"admins-teams\",\n * channelName: \"admins-teams-channel\",\n * message: \"Please approve this request\",\n * approver: \"approver123\",\n * defaultArgsJson: { key1: \"value1\", key2: 42 },\n * dynamicEnumsJson: { foo: [\"choice1\", \"choice2\"], bar: [\"optionA\", \"optionB\"] },\n * });\n * ```\n * \n * **Note:** This function requires execution within a Windmill flow or flow preview.\n */\nasync requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise\n\nsetWorkflowCtx(ctx: WorkflowCtx | null): void\n\nasync sleep(seconds: number): Promise\n\nasync step(name: string, fn: () => T | Promise): Promise\n\n/**\n * Create a task that dispatches to a separate Windmill script.\n * \n * @example\n * const extract = taskScript(\"f/data/extract\");\n * // inside workflow: await extract({ url: \"https://...\" })\n */\ntaskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Create a task that dispatches to a separate Windmill flow.\n * \n * @example\n * const pipeline = taskFlow(\"f/etl/pipeline\");\n * // inside workflow: await pipeline({ input: data })\n */\ntaskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Mark an async function as a workflow-as-code entry point.\n * \n * The function must be **deterministic**: given the same inputs it must call\n * tasks in the same order on every replay. Branching on task results is fine\n * (results are replayed from checkpoint), but branching on external state\n * (current time, random values, external API calls) must use `step()` to\n * checkpoint the value so replays see the same result.\n */\nworkflow(fn: (...args: any[]) => Promise): void\n\n/**\n * Suspend the workflow and wait for an external approval.\n * \n * Use `getResumeUrls()` (wrapped in `step()`) to obtain resume/cancel/approvalPage\n * URLs before calling this function.\n * \n * @example\n * const urls = await step(\"urls\", () => getResumeUrls());\n * await step(\"notify\", () => sendEmail(urls.approvalPage));\n * const { value, approver } = await waitForApproval({ timeout: 3600 });\n */\nwaitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }>\n\n/**\n * Process items in parallel with optional concurrency control.\n * \n * Each item is processed by calling `fn(item)`, which should be a task().\n * Items are dispatched in batches of `concurrency` (default: all at once).\n * \n * @example\n * const process = task(async (item: string) => { ... });\n * const results = await parallel(items, process, { concurrency: 5 });\n */\nasync parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise\n\n/**\n * Commit Kafka offsets for a trigger with auto_commit disabled.\n * @param triggerPath - Path to the Kafka trigger (from event.wm_trigger.trigger_path)\n * @param topic - Kafka topic name (from event.topic)\n * @param partition - Partition number (from event.partition)\n * @param offset - Message offset to commit (from event.offset)\n */\nasync commitKafkaOffsets(triggerPath: string, topic: string, partition: number, offset: number,): Promise\n\n/**\n * Parse an S3 object from URI string or record format\n * @param s3Object - S3 object as URI string (`s3://storage/key`, `s3:///key`\n * for the default storage) or record. Any other string throws rather than\n * falling back to an auto-generated key: an auto key is requested by\n * omitting the object, and a fallback would silently misplace the upload\n * on any typo.\n * @returns S3 object record with storage and s3 key\n */\nparseS3Object(s3Object: S3Object): S3ObjectRecord\n\n/**\n * Create a SQL template function for PostgreSQL/datatable queries\n * @param name - Database/datatable name (default: \"main\")\n * @returns SQL template function for building parameterized queries\n * @example\n * let sql = wmill.datatable()\n * let name = 'Robin'\n * let age = 21\n * await sql`\n * SELECT * FROM friends\n * WHERE name = ${name} AND age = ${age}::int\n * `.fetch()\n */\ndatatable(name: string = \"main\"): DatatableSqlTemplateFunction\n\n/**\n * Create a SQL template function for DuckDB/ducklake queries\n * @param name - DuckDB database name, optionally with a schema as `name:schema` (default: \"main\")\n * @returns SQL template function for building parameterized queries\n * @example\n * let sql = wmill.ducklake()\n * let name = 'Robin'\n * let age = 21\n * await sql`\n * SELECT * FROM friends\n * WHERE name = ${name} AND age = ${age}\n * `.fetch()\n * @example\n * // Target a specific schema within the ducklake\n * let sql = wmill.ducklake(\"my_lake:analytics\")\n */\nducklake(name: string = \"main\"): SqlTemplateFunction\n\n/**\n * Idempotently materialize `selectSql` into a ducklake table for one\n * partition (or the whole table when `partition` is omitted) \u2014 the client-side\n * equivalent of the `// materialize` engine.\n * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it\n * replaces it (whole table \u2192 `CREATE OR REPLACE`; partition \u2192 delete + insert).\n * Safe to re-run for the same partition (backfill / failure-recovery).\n * \n * Returns a lazy statement \u2014 call `.execute()` to run it:\n * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.\n */\nupsertPartition(opts: DucklakeMaterializeOptions): SqlStatement\n\n/**\n * INSERT-only materialization (no dedup/replace) for append-only tables.\n * Re-running the same partition duplicates rows \u2014 use only for immutable\n * event-log sources.\n * \n * Returns a lazy statement \u2014 call `.execute()` to run it:\n * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.\n */\nappendPartition(opts: Omit,): SqlStatement\n"; +export declare const SDK_PYTHON = "# Python SDK (wmill)\n\nImport: import wmill\n\ndef worker_has_internal_server() -> bool\n\ndef get_mocked_api() -> Optional[dict]\n\n# Get the HTTP client instance.\n# \n# Returns:\n# Configured httpx.Client for API requests\ndef get_client() -> httpx.Client\n\n# Make an HTTP GET request to the Windmill API.\n# \n# Args:\n# endpoint: API endpoint path\n# raise_for_status: Whether to raise an exception on HTTP errors\n# **kwargs: Additional arguments passed to httpx.get\n# \n# Returns:\n# HTTP response object\ndef get(endpoint, raise_for_status = True, **kwargs) -> httpx.Response\n\n# Make an HTTP POST request to the Windmill API.\n# \n# Args:\n# endpoint: API endpoint path\n# raise_for_status: Whether to raise an exception on HTTP errors\n# **kwargs: Additional arguments passed to httpx.post\n# \n# Returns:\n# HTTP response object\ndef post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response\n\n# Create a new authentication token.\n# \n# Args:\n# duration: Token validity duration (default: 1 day)\n# \n# Returns:\n# New authentication token string\ndef create_token(duration = dt.timedelta(days=1)) -> str\n\n# Create a script job and return its job id.\n# \n# .. deprecated:: Use run_script_by_path_async or run_script_by_hash_async instead.\ndef run_script_async(path: str = None, hash_: str = None, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str\n\n# Create a script job by path and return its job id.\ndef run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str\n\n# Create a script job by hash and return its job id.\ndef run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str\n\n# Create a flow job and return its job id.\ndef run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str\n\n# Run script synchronously and return its result.\n# \n# .. deprecated:: Use run_script_by_path or run_script_by_hash instead.\ndef run_script(path: str = None, hash_: str = None, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any\n\n# Run script by path synchronously and return its result.\ndef run_script_by_path(path: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any\n\n# Run script by hash synchronously and return its result.\ndef run_script_by_hash(hash_: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any\n\n# Run a script on the current worker without creating a job.\n# \n# On agent workers (no internal server), falls back to running a normal\n# preview job and waiting for the result.\ndef run_inline_script_preview(content: str, language: str, args: dict = None) -> Any\n\n# Wait for a job to complete and return its result.\n# \n# Args:\n# job_id: ID of the job to wait for\n# timeout: Maximum time to wait (seconds or timedelta)\n# verbose: Enable verbose logging\n# cleanup: Register cleanup handler to cancel job on exit\n# assert_result_is_not_none: Raise exception if result is None\n# \n# Returns:\n# Job result when completed\n# \n# Raises:\n# TimeoutError: If timeout is reached\n# Exception: If job fails\ndef wait_job(job_id, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False)\n\n# Cancel a specific job by ID.\n# \n# Args:\n# job_id: UUID of the job to cancel\n# reason: Optional reason for cancellation\n# \n# Returns:\n# Response message from the cancel endpoint\ndef cancel_job(job_id: str, reason: str = None) -> str\n\n# Cancel currently running executions of the same script.\ndef cancel_running() -> dict\n\n# Get job details by ID.\n# \n# Args:\n# job_id: UUID of the job\n# \n# Returns:\n# Job details dictionary\ndef get_job(job_id: str) -> dict\n\n# Get the root job ID for a flow hierarchy.\n# \n# Args:\n# job_id: Job ID (defaults to current WM_JOB_ID)\n# \n# Returns:\n# Root job ID\ndef get_root_job_id(job_id: str | None = None) -> dict\n\n# Get an OIDC JWT token for authentication to external services.\n# \n# Args:\n# audience: Token audience (e.g., \"vault\", \"aws\")\n# expires_in: Optional expiration time in seconds\n# \n# Returns:\n# JWT token string\ndef get_id_token(audience: str, expires_in: int | None = None) -> str\n\n# Get the status of a job.\n# \n# Args:\n# job_id: UUID of the job\n# \n# Returns:\n# Job status: \"RUNNING\", \"WAITING\", or \"COMPLETED\"\ndef get_job_status(job_id: str) -> JobStatus\n\n# Get the result of a completed job.\n# \n# Args:\n# job_id: UUID of the completed job\n# assert_result_is_not_none: Raise exception if result is None\n# \n# Returns:\n# Job result\ndef get_result(job_id: str, assert_result_is_not_none: bool = True) -> Any\n\n# Get a variable value by path.\n# \n# Args:\n# path: Variable path in Windmill\n# \n# Returns:\n# Variable value as string\ndef get_variable(path: str) -> str\n\n# Set a variable value by path, creating it if it doesn't exist.\n# \n# Args:\n# path: Variable path in Windmill\n# value: Variable value to set\n# is_secret: Whether the variable should be secret (default: False)\ndef set_variable(path: str, value: str, is_secret: bool = False) -> None\n\n# Get a resource value by path.\n# \n# Args:\n# path: Resource path in Windmill\n# none_if_undefined: Return None instead of raising if not found\n# interpolated: if variables and resources are fully unrolled\n# \n# Returns:\n# Resource value dictionary or None\ndef get_resource(path: str, none_if_undefined: bool = False, interpolated: bool = True) -> dict | None\n\n# Set a resource value by path, creating it if it doesn't exist.\n# \n# Args:\n# value: Resource value to set\n# path: Resource path in Windmill\n# resource_type: Resource type for creation\ndef set_resource(value: Any, path: str, resource_type: str)\n\n# List resources from Windmill workspace.\n# \n# Args:\n# resource_type: Optional resource type to filter by (e.g., \"postgresql\", \"mysql\", \"s3\")\n# page: Optional page number for pagination\n# per_page: Optional number of results per page\n# \n# Returns:\n# List of resource dictionaries\ndef list_resources(resource_type: str = None, page: int = None, per_page: int = None) -> list[dict]\n\n# Set the workflow state.\n# \n# Args:\n# value: State value to set\n# path: Optional state resource path override.\ndef set_state(value: Any, path: str | None = None) -> None\n\n# Get the workflow state.\n# \n# Args:\n# path: Optional state resource path override.\n# \n# Returns:\n# State value or None if not set\ndef get_state(path: str | None = None) -> Any\n\n# Set job progress percentage (0-99).\n# \n# Args:\n# value: Progress percentage\n# job_id: Job ID (defaults to current WM_JOB_ID)\ndef set_progress(value: int, job_id: Optional[str] = None)\n\n# Get job progress percentage.\n# \n# Args:\n# job_id: Job ID (defaults to current WM_JOB_ID)\n# \n# Returns:\n# Progress value (0-100) or None if not set\ndef get_progress(job_id: Optional[str] = None) -> Any\n\n# Set the user state of a flow at a given key\ndef set_flow_user_state(key: str, value: Any) -> None\n\n# Get the user state of a flow at a given key\ndef get_flow_user_state(key: str) -> Any\n\n# Get the Windmill server version.\n# \n# Returns:\n# Version string\ndef version()\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from DuckDB\ndef get_duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings | None\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from Polars\ndef get_polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection using boto3\ndef get_boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings\n\n# Load a file from the workspace s3 bucket and returns its content as bytes.\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# my_obj_content = client.load_s3_file(s3_obj)\n# file_content = my_obj_content.decode(\"utf-8\")\n# '''\ndef load_s3_file(s3object: S3Object | str, s3_resource_path: str | None) -> bytes\n\n# Load a file from the workspace s3 bucket and returns the bytes stream.\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# with wmill.load_s3_file_reader(s3object, s3_resource_path) as file_reader:\n# print(file_reader.read())\n# '''\ndef load_s3_file_reader(s3object: S3Object | str, s3_resource_path: str | None) -> BufferedReader\n\n# Write a file to the workspace S3 bucket\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# \n# # for an in memory bytes array:\n# file_content = b'Hello Windmill!'\n# client.write_s3_file(s3_obj, file_content)\n# \n# # for a file:\n# with open(\"my_file.txt\", \"rb\") as my_file:\n# client.write_s3_file(s3_obj, my_file)\n# '''\ndef write_s3_file(s3object: S3Object | str | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, content_type: str | None = None, content_disposition: str | None = None) -> S3Object\n\n# Permanently delete a file from the workspace S3 bucket.\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# client.delete_s3_object(s3_obj)\n# '''\ndef delete_s3_object(s3object: S3Object | str, s3_resource_path: str | None = None) -> None\n\n# Sign S3 objects for use by anonymous users in public apps.\n# \n# Args:\n# s3_objects: List of S3 objects to sign\n# \n# Returns:\n# List of signed S3 objects\ndef sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object]\n\n# Sign a single S3 object for use by anonymous users in public apps.\n# \n# Args:\n# s3_object: S3 object to sign\n# \n# Returns:\n# Signed S3 object\ndef sign_s3_object(s3_object: S3Object | str) -> S3Object\n\n# Generate presigned public URLs for an array of S3 objects.\n# If an S3 object is not signed yet, it will be signed first.\n# \n# Args:\n# s3_objects: List of S3 objects to sign\n# base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL)\n# \n# Returns:\n# List of signed public URLs\n# \n# Example:\n# >>> s3_objs = [S3Object(s3=\"/path/to/file1.txt\"), S3Object(s3=\"/path/to/file2.txt\")]\n# >>> urls = client.get_presigned_s3_public_urls(s3_objs)\ndef get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None) -> list[str]\n\n# Generate a presigned public URL for an S3 object.\n# If the S3 object is not signed yet, it will be signed first.\n# \n# Args:\n# s3_object: S3 object to sign\n# base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL)\n# \n# Returns:\n# Signed public URL\n# \n# Example:\n# >>> s3_obj = S3Object(s3=\"/path/to/file.txt\")\n# >>> url = client.get_presigned_s3_public_url(s3_obj)\ndef get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None) -> str\n\n# Get the current user information.\n# \n# Returns:\n# User details dictionary\ndef whoami() -> dict\n\n# Get the current user information (alias for whoami).\n# \n# Returns:\n# User details dictionary\ndef user() -> dict\n\n# Get the state resource path from environment.\n# \n# Returns:\n# State path string\ndef state_path() -> str\n\n# Get the workflow state.\n# \n# Returns:\n# State value or None if not set\ndef state() -> Any\n\n# Set the state in the shared folder using pickle\ndef set_shared_state_pickle(value: Any, path: str = 'state.pickle') -> None\n\n# Get the state in the shared folder using pickle\ndef get_shared_state_pickle(path: str = 'state.pickle') -> Any\n\n# Set the state in the shared folder using pickle\ndef set_shared_state(value: Any, path: str = 'state.json') -> None\n\n# Get the state in the shared folder using pickle\ndef get_shared_state(path: str = 'state.json') -> None\n\n# Get URLs needed for resuming a flow after suspension.\n# \n# Args:\n# approver: Optional approver name\n# flow_level: If True, generate resume URLs for the parent flow instead of the\n# specific step. This allows pre-approvals that can be consumed by any later\n# suspend step in the same flow.\n# \n# Returns:\n# Dictionary with approvalPage, resume, and cancel URLs\ndef get_resume_urls(approver: str = None, flow_level: bool = None) -> dict\n\n# Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields.\n# \n# **[Enterprise Edition Only]** To include form fields in the Slack approval request, use the \"Advanced -> Suspend -> Form\" functionality.\n# Learn more at: https://www.windmill.dev/docs/flows/flow_approval#form\n# \n# :param slack_resource_path: The path to the Slack resource in Windmill.\n# :type slack_resource_path: str\n# :param channel_id: The Slack channel ID where the approval request will be sent.\n# :type channel_id: str\n# :param message: Optional custom message to include in the Slack approval request.\n# :type message: str, optional\n# :param approver: Optional user ID or name of the approver for the request.\n# :type approver: str, optional\n# :param default_args_json: Optional dictionary defining or overriding the default arguments for form fields.\n# :type default_args_json: dict, optional\n# :param dynamic_enums_json: Optional dictionary overriding the enum default values of enum form fields.\n# :type dynamic_enums_json: dict, optional\n# \n# :raises Exception: If the function is not called within a flow or flow preview.\n# :raises Exception: If the required flow job or flow step environment variables are not set.\n# \n# :return: None\n# \n# **Usage Example:**\n# >>> client.request_interactive_slack_approval(\n# ... slack_resource_path=\"/u/alex/my_slack_resource\",\n# ... channel_id=\"admins-slack-channel\",\n# ... message=\"Please approve this request\",\n# ... approver=\"approver123\",\n# ... default_args_json={\"key1\": \"value1\", \"key2\": 42},\n# ... dynamic_enums_json={\"foo\": [\"choice1\", \"choice2\"], \"bar\": [\"optionA\", \"optionB\"]},\n# ... )\n# \n# **Notes:**\n# - This function must be executed within a Windmill flow or flow preview.\n# - The function checks for required environment variables (`WM_FLOW_JOB_ID`, `WM_FLOW_STEP_ID`) to ensure it is run in the appropriate context.\ndef request_interactive_slack_approval(slack_resource_path: str, channel_id: str, message: str = None, approver: str = None, default_args_json: dict = None, dynamic_enums_json: dict = None) -> None\n\n# Get email from workspace username\n# This method is particularly useful for apps that require the email address of the viewer.\n# Indeed, in the viewer context WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app.\ndef username_to_email(username: str) -> str\n\n# Send a message to a Microsoft Teams conversation with conversation_id, where success is used to style the message\ndef send_teams_message(conversation_id: str, text: str, success: bool = True, card_block: dict = None)\n\n# Get a DataTable client for SQL queries.\n# \n# Args:\n# name: Database name (default: \"main\")\n# \n# Returns:\n# DataTableClient instance\ndef datatable(name: str = 'main')\n\n# Get a DuckLake client for DuckDB queries.\n# \n# Args:\n# name: Database name (default: \"main\")\n# \n# Returns:\n# DucklakeClient instance\ndef ducklake(name: str = 'main')\n\ndef init_global_client(f)\n\ndef deprecate(in_favor_of: str)\n\n# Get the current workspace ID.\n# \n# Returns:\n# Workspace ID string\ndef get_workspace() -> str\n\ndef get_version() -> str\n\n# Run a script synchronously by hash and return its result.\n# \n# Args:\n# hash: Script hash\n# args: Script arguments\n# verbose: Enable verbose logging\n# assert_result_is_not_none: Raise exception if result is None\n# cleanup: Register cleanup handler to cancel job on exit\n# timeout: Maximum time to wait\n# tag: Override the worker tag the job runs on\n# \n# Returns:\n# Script result\ndef run_script_sync(hash: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None, tag: str = None) -> Any\n\n# Run a script synchronously by path and return its result.\n# \n# Args:\n# path: Script path\n# args: Script arguments\n# verbose: Enable verbose logging\n# assert_result_is_not_none: Raise exception if result is None\n# cleanup: Register cleanup handler to cancel job on exit\n# timeout: Maximum time to wait\n# tag: Override the worker tag the job runs on\n# \n# Returns:\n# Script result\ndef run_script_by_path_sync(path: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None, tag: str = None) -> Any\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from DuckDB\ndef duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from Polars\ndef polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection using boto3\ndef boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings\n\n# Get the state resource path from environment.\n# \n# Returns:\n# State path string\ndef get_state_path() -> str\n\n# Parse resource syntax from string.\ndef parse_resource_syntax(s: str) -> Optional[str]\n\n# Parse S3 object from a `s3:///` URI string (`s3:///`\n# for the default storage) or S3Object format. Any other string raises\n# rather than falling back to an auto-generated key: an auto key is\n# requested by omitting the object, and a fallback would silently misplace\n# the upload on any typo.\ndef parse_s3_object(s3_object: S3Object | str) -> S3Object\n\n# Parse variable syntax from string.\ndef parse_variable_syntax(s: str) -> Optional[str]\n\n# Append a text to the result stream.\n# \n# Args:\n# text: text to append to the result stream\ndef append_to_result_stream(text: str) -> None\n\n# Stream to the result stream.\n# \n# Args:\n# stream: stream to stream to the result stream\ndef stream_result(stream) -> None\n\n# Execute a SQL query against the DataTable.\n# \n# Args:\n# sql: SQL query string with $1, $2, etc. placeholders\n# *args: Positional arguments to bind to query placeholders\n# \n# Returns:\n# SqlQuery instance for fetching results\ndef query(sql: str, *args) -> SqlQuery\n\n# Idempotently materialize the rows of `select_sql` into ducklake\n# `table` for one `partition` (or the whole table when `partition` is\n# None). Client-side equivalent of the `// materialize` engine: with\n# `unique_key` it upserts within the slice (delete-by-key + insert);\n# without it, it replaces (whole table \u2192 CREATE OR REPLACE; partition \u2192\n# delete the partition + insert). Re-running the same slice is safe \u2014 the\n# backfill / failure-recovery contract.\n# \n# The partition value is bound as a DuckDB arg (never string-interpolated)\n# so it cannot inject SQL. `select_sql` is trusted (your own query).\ndef upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)\n\n# INSERT-only materialization (no dedup / no replace) for an immutable\n# event-log table \u2014 for one `partition`, or the whole table when\n# `partition` is None. NOTE: unlike `upsert_partition`, re-running the same\n# slice duplicates rows \u2014 use only for append-only sources.\ndef append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)\n\n# Read a materialized ducklake table, optionally a single partition.\ndef read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)\n\n# Execute query and fetch results.\n# \n# Args:\n# result_collection: Optional result collection mode\n# \n# Returns:\n# Query results\ndef fetch(result_collection: str | None = None)\n\n# Execute query and fetch first row of results.\n# \n# Returns:\n# First row of query results\ndef fetch_one()\n\n# Execute query and fetch first row of results. Return result as a scalar value.\n# \n# Returns:\n# First row of query result as a scalar value\ndef fetch_one_scalar()\n\n# Execute query and don't return any results.\n# \ndef execute()\n\n# DuckDB executor requires explicit argument types at declaration\n# These types exist in both DuckDB and Postgres\n# Check that the types exist if you plan to extend this function for other SQL engines.\ndef infer_sql_type(value) -> str\n\ndef parse_sql_client_name(name: str) -> tuple[str, Optional[str]]\n\n# Decorator that marks a function as a workflow task.\n# \n# Works in both WAC v1 (sync, HTTP-based dispatch) and WAC v2\n# (async, checkpoint/replay) modes:\n# \n# - **v2 (inside @workflow)**: dispatches as a checkpoint step.\n# - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API.\n# - **Standalone**: executes the function body directly.\n# \n# Usage::\n# \n# @task\n# async def extract_data(url: str): ...\n# \n# @task(path=\"f/external_script\", timeout=600, tag=\"gpu\")\n# async def run_external(x: int): ...\ndef task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill script.\n# \n# Usage::\n# \n# extract = task_script(\"f/data/extract\", timeout=600)\n# \n# @workflow\n# async def main():\n# data = await extract(url=\"https://...\")\ndef task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill flow.\n# \n# Usage::\n# \n# pipeline = task_flow(\"f/etl/pipeline\", priority=10)\n# \n# @workflow\n# async def main():\n# result = await pipeline(input=data)\ndef task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Decorator marking an async function as a workflow-as-code entry point.\n# \n# The function must be **deterministic**: given the same inputs it must call\n# tasks in the same order on every replay. Branching on task results is fine\n# (results are replayed from checkpoint), but branching on external state\n# (current time, random values, external API calls) must use ``step()`` to\n# checkpoint the value so replays see the same result.\ndef workflow(func)\n\n# Execute ``fn`` inline and checkpoint the result.\n# \n# On replay the cached value is returned without re-executing ``fn``.\n# Use for lightweight deterministic operations (timestamps, random IDs,\n# config reads) that should not incur the overhead of a child job.\nasync def step(name: str, fn)\n\n# Server-side sleep \u2014 suspend the workflow for the given duration without holding a worker.\n# \n# Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``.\n# Outside a workflow, falls back to ``asyncio.sleep``.\nasync def sleep(seconds: int)\n\n# Suspend the workflow and wait for an external approval.\n# \n# Use ``get_resume_urls()`` (wrapped in ``step()``) to obtain\n# resume/cancel/approval URLs before calling this function.\n# \n# Returns a dict with ``value`` (form data), ``approver``, and ``approved``.\n# \n# Args:\n# timeout: Approval timeout in seconds (default 1800).\n# form: Optional form schema for the approval page.\n# self_approval: Whether the user who triggered the flow can approve it (default True).\n# \n# Example::\n# \n# urls = await step(\"urls\", lambda: get_resume_urls())\n# await step(\"notify\", lambda: send_email(urls[\"approvalPage\"]))\n# result = await wait_for_approval(timeout=3600)\nasync def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict\n\n# Process items in parallel with optional concurrency control.\n# \n# Each item is processed by calling ``fn(item)``, which should be a @task.\n# Items are dispatched in batches of ``concurrency`` (default: all at once).\n# \n# Example::\n# \n# @task\n# async def process(item: str):\n# ...\n# \n# results = await parallel(items, process, concurrency=5)\nasync def parallel(items, fn, concurrency: Optional[int] = None)\n\n# Commit Kafka offsets for a trigger with auto_commit disabled.\n# \n# Args:\n# trigger_path: Path to the Kafka trigger (from event['wm_trigger']['trigger_path'])\n# topic: Kafka topic name (from event['topic'])\n# partition: Partition number (from event['partition'])\n# offset: Message offset to commit (from event['offset'])\ndef commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None\n\n"; +export declare const WAC_SDK_TYPESCRIPT = "## TypeScript Workflow-as-Code API (windmill-client)\n\nImport: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getResumeUrls, parallel } from \"windmill-client\"`\n\n```typescript\nexport interface TaskOptions {\n timeout?: number;\n tag?: string;\n cache_ttl?: number;\n priority?: number;\n concurrency_limit?: number;\n concurrency_key?: string;\n concurrency_time_window_s?: number;\n}\n\n/**\n * Get URLs needed for resuming a flow after this step\n * @param approver approver name\n * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step.\n * This allows pre-approvals that can be consumed by any later suspend step in the same flow.\n * @returns approval page UI URL, resume and cancel API URLs for resuming the flow\n */\nexport async function getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ approvalPage: string; resume: string; cancel: string; }>\n\n/**\n * Wrap an async function as a workflow task.\n *\n * @example\n * const extract_data = task(async (url: string) => { ... });\n * const run_external = task(\"f/external_script\", async (x: number) => { ... });\n *\n * Inside a `workflow()`, calling a task dispatches it as a step.\n * Outside a workflow, the function body executes directly.\n */\nexport function task Promise>(fnOrPath: T | string, maybeFnOrOptions?: T | TaskOptions, maybeOptions?: TaskOptions,): T\n\n/**\n * Create a task that dispatches to a separate Windmill script.\n *\n * @example\n * const extract = taskScript(\"f/data/extract\");\n * // inside workflow: await extract({ url: \"https://...\" })\n */\nexport function taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Create a task that dispatches to a separate Windmill flow.\n *\n * @example\n * const pipeline = taskFlow(\"f/etl/pipeline\");\n * // inside workflow: await pipeline({ input: data })\n */\nexport function taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Mark an async function as a workflow-as-code entry point.\n *\n * The function must be **deterministic**: given the same inputs it must call\n * tasks in the same order on every replay. Branching on task results is fine\n * (results are replayed from checkpoint), but branching on external state\n * (current time, random values, external API calls) must use `step()` to\n * checkpoint the value so replays see the same result.\n */\nexport function workflow(fn: (...args: any[]) => Promise)\n\nexport async function step(name: string, fn: () => T | Promise): Promise\n\nexport async function sleep(seconds: number): Promise\n\n/**\n * Suspend the workflow and wait for an external approval.\n *\n * Use `getResumeUrls()` (wrapped in `step()`) to obtain resume/cancel/approvalPage\n * URLs before calling this function.\n *\n * @example\n * const urls = await step(\"urls\", () => getResumeUrls());\n * await step(\"notify\", () => sendEmail(urls.approvalPage));\n * const { value, approver } = await waitForApproval({ timeout: 3600 });\n */\nexport function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }>\n\n/**\n * Process items in parallel with optional concurrency control.\n *\n * Each item is processed by calling `fn(item)`, which should be a task().\n * Items are dispatched in batches of `concurrency` (default: all at once).\n *\n * @example\n * const process = task(async (item: string) => { ... });\n * const results = await parallel(items, process, { concurrency: 5 });\n */\nexport async function parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise\n```\n"; +export declare const WAC_SDK_PYTHON = "## Python Workflow-as-Code API (wmill)\n\nImport: `from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_resume_urls, parallel, TaskError`\n\n```python\n# Raised when a WAC task step failed.\n#\n# Attributes:\n# step_key: The checkpoint key of the failed step.\n# child_job_id: The UUID of the failed child job.\n# result: The error result from the child job.\nclass TaskError(Exception):\n def __init__(self, message: str, *, step_key: str = '', child_job_id: str = '', result = None)\n\n# Get URLs needed for resuming a flow after suspension.\n#\n# Args:\n# approver: Optional approver name\n# flow_level: If True, generate resume URLs for the parent flow instead of the\n# specific step. This allows pre-approvals that can be consumed by any later\n# suspend step in the same flow.\n#\n# Returns:\n# Dictionary with approvalPage, resume, and cancel URLs\ndef get_resume_urls(approver: str = None, flow_level: bool = None) -> dict\n\n# Decorator that marks a function as a workflow task.\n#\n# Works in both WAC v1 (sync, HTTP-based dispatch) and WAC v2\n# (async, checkpoint/replay) modes:\n#\n# - **v2 (inside @workflow)**: dispatches as a checkpoint step.\n# - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API.\n# - **Standalone**: executes the function body directly.\n#\n# Usage::\n#\n# @task\n# async def extract_data(url: str): ...\n#\n# @task(path=\"f/external_script\", timeout=600, tag=\"gpu\")\n# async def run_external(x: int): ...\ndef task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill script.\n#\n# Usage::\n#\n# extract = task_script(\"f/data/extract\", timeout=600)\n#\n# @workflow\n# async def main():\n# data = await extract(url=\"https://...\")\ndef task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill flow.\n#\n# Usage::\n#\n# pipeline = task_flow(\"f/etl/pipeline\", priority=10)\n#\n# @workflow\n# async def main():\n# result = await pipeline(input=data)\ndef task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Decorator marking an async function as a workflow-as-code entry point.\n#\n# The function must be **deterministic**: given the same inputs it must call\n# tasks in the same order on every replay. Branching on task results is fine\n# (results are replayed from checkpoint), but branching on external state\n# (current time, random values, external API calls) must use ``step()`` to\n# checkpoint the value so replays see the same result.\ndef workflow(func)\n\n# Execute ``fn`` inline and checkpoint the result.\n#\n# On replay the cached value is returned without re-executing ``fn``.\n# Use for lightweight deterministic operations (timestamps, random IDs,\n# config reads) that should not incur the overhead of a child job.\nasync def step(name: str, fn)\n\n# Server-side sleep \u2014 suspend the workflow for the given duration without holding a worker.\n#\n# Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``.\n# Outside a workflow, falls back to ``asyncio.sleep``.\nasync def sleep(seconds: int)\n\n# Suspend the workflow and wait for an external approval.\n#\n# Use ``get_resume_urls()`` (wrapped in ``step()``) to obtain\n# resume/cancel/approval URLs before calling this function.\n#\n# Returns a dict with ``value`` (form data), ``approver``, and ``approved``.\n#\n# Args:\n# timeout: Approval timeout in seconds (default 1800).\n# form: Optional form schema for the approval page.\n# self_approval: Whether the user who triggered the flow can approve it (default True).\n#\n# Example::\n#\n# urls = await step(\"urls\", lambda: get_resume_urls())\n# await step(\"notify\", lambda: send_email(urls[\"approvalPage\"]))\n# result = await wait_for_approval(timeout=3600)\nasync def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict\n\n# Process items in parallel with optional concurrency control.\n#\n# Each item is processed by calling ``fn(item)``, which should be a @task.\n# Items are dispatched in batches of ``concurrency`` (default: all at once).\n#\n# Example::\n#\n# @task\n# async def process(item: str):\n# ...\n#\n# results = await parallel(items, process, concurrency=5)\nasync def parallel(items, fn, *, concurrency: Optional[int] = None)\n```\n"; +export declare const DATATABLE_SDK_TYPESCRIPT = "## TypeScript Datatable API (windmill-client)\n\nImport: `import * as wmill from 'windmill-client'`\n\nSQL statement object with query content, arguments, and execution methods\n```typescript\ntype SqlStatement = {\n /** Raw SQL content with formatted arguments */\n content: string;\n\n /** Argument values keyed by parameter name */\n args: Record;\n\n /**\n * Execute the SQL query and return results\n * @param params - Optional parameters including result collection mode\n * @returns Query results based on the result collection mode\n */\n fetch(\n params?: FetchParams // The union is for auto-completion\n ): Promise>;\n\n /**\n * Execute the SQL query and return only the first row\n * @param params - Optional parameters\n * @returns First row of the query result\n */\n fetchOne(\n params?: Omit, \"resultCollection\">\n ): Promise>;\n\n /**\n * Execute the SQL query and return only the first row as a scalar value\n * @param params - Optional parameters\n * @returns First row of the query result\n */\n fetchOneScalar(\n params?: Omit<\n FetchParams<\"last_statement_first_row_scalar\">,\n \"resultCollection\"\n >\n ): Promise>;\n\n /**\n * Execute the SQL query without fetching rows\n * @param params - Optional parameters\n */\n execute(\n params?: Omit, \"resultCollection\">\n ): Promise;\n};\n```\n\n```typescript\n// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch()\ninterface DatatableSqlTemplateFunction {\n // Tagged template usage:\n (strings: TemplateStringsArray, ...values: any[]): SqlStatement;\n query(sql: string, ...params: any[]): SqlStatement;\n};\n```\n\nCreate a SQL template function for PostgreSQL/datatable queries\n@param name - Database/datatable name (default: \"main\")\n@returns SQL template function for building parameterized queries\n@example\nlet sql = wmill.datatable()\nlet name = 'Robin'\nlet age = 21\nawait sql`\n SELECT * FROM friends\n WHERE name = ${name} AND age = ${age}::int\n`.fetch()\n```typescript\nfunction datatable(name: string = \"main\"): DatatableSqlTemplateFunction\n```\n"; +export declare const DATATABLE_SDK_PYTHON = "## Python Datatable API (wmill)\n\nImport: `import wmill`\n\n# Get a DataTable client for SQL queries.\n# \n# Args:\n# name: Database name (default: \"main\")\n# \n# Returns:\n# DataTableClient instance\ndef datatable(name: str = 'main') -> DataTableClient\n\n# Client for executing SQL queries against Windmill DataTables.\nclass DataTableClient:\n # Initialize DataTableClient.\n # \n # Args:\n # client: Windmill client instance\n # name: DataTable name\n def __init__(client: Windmill, name: str)\n\n # Execute a SQL query against the DataTable.\n # \n # Args:\n # sql: SQL query string with $1, $2, etc. placeholders\n # *args: Positional arguments to bind to query placeholders\n # \n # Returns:\n # SqlQuery instance for fetching results\n def query(sql: str, *args) -> SqlQuery\n\n\n# Query result handler for DataTable and DuckLake queries.\nclass SqlQuery:\n # Initialize SqlQuery.\n # \n # Args:\n # sql: SQL query string\n # fetch_fn: Function to execute the query\n def __init__(sql: str, fetch_fn)\n\n # Execute query and fetch results.\n # \n # Args:\n # result_collection: Optional result collection mode\n # \n # Returns:\n # Query results\n def fetch(result_collection: str | None = None)\n\n # Execute query and fetch first row of results.\n # \n # Returns:\n # First row of query results\n def fetch_one()\n\n # Execute query and fetch first row of results. Return result as a scalar value.\n # \n # Returns:\n # First row of query result as a scalar value\n def fetch_one_scalar()\n\n # Execute query and don't return any results.\n # \n def execute()\n\n\n"; +export declare const OPENFLOW_SCHEMA = "## OpenFlow Schema\n\n{\"OpenFlow\":{\"type\":\"object\",\"description\":\"Top-level flow definition containing metadata, configuration, and the flow structure\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Short description of what this flow does\"},\"description\":{\"type\":\"string\",\"description\":\"Detailed documentation for this flow\"},\"value\":{\"$ref\":\"#/components/schemas/FlowValue\"},\"schema\":{\"type\":\"object\",\"description\":\"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')\"},\"on_behalf_of_email\":{\"type\":\"string\",\"description\":\"The flow will be run with the permissions of the user with this email.\"}},\"required\":[\"summary\",\"value\"]},\"FlowValue\":{\"type\":\"object\",\"description\":\"The flow structure containing modules and optional preprocessor/failure handlers\",\"properties\":{\"modules\":{\"type\":\"array\",\"description\":\"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"failure_module\":{\"description\":\"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types\",\"$ref\":\"#/components/schemas/FlowModule\"},\"preprocessor_module\":{\"description\":\"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results\",\"$ref\":\"#/components/schemas/FlowModule\"},\"same_worker\":{\"type\":\"boolean\",\"description\":\"If true, all steps run on the same worker for better performance\"},\"preserve_step_tags\":{\"type\":\"boolean\",\"description\":\"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag.\"},\"concurrent_limit\":{\"type\":\"number\",\"description\":\"Maximum number of concurrent executions of this flow\"},\"concurrency_key\":{\"type\":\"string\",\"description\":\"Expression to group concurrent executions (e.g., by user ID)\"},\"concurrency_time_window_s\":{\"type\":\"number\",\"description\":\"Time window in seconds for concurrent_limit\"},\"debounce_delay_s\":{\"type\":\"integer\",\"description\":\"Delay in seconds to debounce flow executions\"},\"debounce_key\":{\"type\":\"string\",\"description\":\"Expression to group debounced executions\"},\"debounce_args_to_accumulate\":{\"type\":\"array\",\"description\":\"Arguments to accumulate across debounced executions\",\"items\":{\"type\":\"string\"}},\"max_total_debouncing_time\":{\"type\":\"integer\",\"description\":\"Maximum total time in seconds that a job can be debounced\"},\"max_total_debounces_amount\":{\"type\":\"integer\",\"description\":\"Maximum number of times a job can be debounced\"},\"skip_expr\":{\"type\":\"string\",\"description\":\"JavaScript expression to conditionally skip the entire flow\"},\"cache_ttl\":{\"type\":\"number\",\"description\":\"Cache duration in seconds for flow results\"},\"cache_ignore_s3_path\":{\"type\":\"boolean\"},\"delete_after_secs\":{\"type\":\"integer\",\"description\":\"If set, delete the flow job's args, result and logs after this many seconds following job completion\"},\"flow_env\":{\"type\":\"object\",\"description\":\"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).\",\"additionalProperties\":{}},\"priority\":{\"type\":\"number\",\"description\":\"Execution priority (higher numbers run first)\"},\"early_return\":{\"type\":\"string\",\"description\":\"JavaScript expression to return early from the flow\"},\"chat_input_enabled\":{\"type\":\"boolean\",\"description\":\"Whether this flow accepts chat-style input\"},\"notes\":{\"type\":\"array\",\"description\":\"Sticky notes attached to the flow\",\"items\":{\"$ref\":\"#/components/schemas/FlowNote\"}},\"groups\":{\"type\":\"array\",\"description\":\"Semantic groups of modules for organizational purposes\",\"items\":{\"$ref\":\"#/components/schemas/FlowGroup\"}}},\"required\":[\"modules\"]},\"Retry\":{\"type\":\"object\",\"description\":\"Retry configuration for failed module executions\",\"properties\":{\"constant\":{\"type\":\"object\",\"description\":\"Retry with constant delay between attempts\",\"properties\":{\"attempts\":{\"type\":\"integer\",\"description\":\"Number of retry attempts\"},\"seconds\":{\"type\":\"integer\",\"description\":\"Seconds to wait between retries\"}}},\"exponential\":{\"type\":\"object\",\"description\":\"Retry with exponential backoff (delay doubles each time)\",\"properties\":{\"attempts\":{\"type\":\"integer\",\"description\":\"Number of retry attempts\"},\"multiplier\":{\"type\":\"integer\",\"description\":\"Multiplier for exponential backoff\"},\"seconds\":{\"type\":\"integer\",\"minimum\":1,\"description\":\"Initial delay in seconds\"},\"random_factor\":{\"type\":\"integer\",\"minimum\":0,\"maximum\":100,\"description\":\"Random jitter percentage (0-100) to avoid thundering herd\"}}},\"retry_if\":{\"$ref\":\"#/components/schemas/RetryIf\"}}},\"FlowNote\":{\"type\":\"object\",\"description\":\"A sticky note attached to a flow for documentation and annotation\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"Unique identifier for the note\"},\"text\":{\"type\":\"string\",\"description\":\"Content of the note\"},\"position\":{\"type\":\"object\",\"description\":\"Position of the note in the flow editor\",\"properties\":{\"x\":{\"type\":\"number\",\"description\":\"X coordinate\"},\"y\":{\"type\":\"number\",\"description\":\"Y coordinate\"}},\"required\":[\"x\",\"y\"]},\"size\":{\"type\":\"object\",\"description\":\"Size of the note in the flow editor\",\"properties\":{\"width\":{\"type\":\"number\",\"description\":\"Width in pixels\"},\"height\":{\"type\":\"number\",\"description\":\"Height in pixels\"}},\"required\":[\"width\",\"height\"]},\"color\":{\"type\":\"string\",\"description\":\"Color of the note (e.g., \\\"yellow\\\", \\\"#ffff00\\\")\"},\"type\":{\"type\":\"string\",\"enum\":[\"free\",\"group\"],\"description\":\"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes\"},\"locked\":{\"type\":\"boolean\",\"default\":false,\"description\":\"Whether the note is locked and cannot be edited or moved\"},\"contained_node_ids\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"For group notes, the IDs of nodes contained within this group\"}},\"required\":[\"id\",\"text\",\"color\",\"type\"]},\"FlowGroup\":{\"type\":\"object\",\"description\":\"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Display name for this group\"},\"note\":{\"type\":\"string\",\"description\":\"Markdown note shown below the group header\"},\"autocollapse\":{\"type\":\"boolean\",\"default\":false,\"description\":\"If true, this group is collapsed by default in the flow editor. UI hint only.\"},\"start_id\":{\"type\":\"string\",\"description\":\"ID of the first flow module in this group (topological entry point)\"},\"end_id\":{\"type\":\"string\",\"description\":\"ID of the last flow module in this group (topological exit point)\"},\"color\":{\"type\":\"string\",\"description\":\"Color for the group in the flow editor\"}},\"required\":[\"start_id\",\"end_id\"]},\"RetryIf\":{\"type\":\"object\",\"description\":\"Conditional retry based on error or result\",\"properties\":{\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables\"}},\"required\":[\"expr\"]},\"StopAfterIf\":{\"type\":\"object\",\"description\":\"Early termination condition for a module\",\"properties\":{\"skip_if_stopped\":{\"type\":\"boolean\",\"description\":\"If true, following steps are skipped when this condition triggers\"},\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop\"},\"error_message\":{\"type\":\"string\",\"nullable\":true,\"description\":\"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.\"},\"error_include_result\":{\"type\":\"boolean\",\"description\":\"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.\"}},\"required\":[\"expr\"]},\"FlowModule\":{\"type\":\"object\",\"description\":\"A single step in a flow. Can be a script, subflow, loop, or branch\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)\"},\"value\":{\"$ref\":\"#/components/schemas/FlowModuleValue\"},\"stop_after_if\":{\"description\":\"Early termination condition evaluated after this step completes\",\"$ref\":\"#/components/schemas/StopAfterIf\"},\"stop_after_all_iters_if\":{\"description\":\"For loops only - early termination condition evaluated after all iterations complete\",\"$ref\":\"#/components/schemas/StopAfterIf\"},\"skip_if\":{\"type\":\"object\",\"description\":\"Conditionally skip this step based on previous results or flow inputs\",\"properties\":{\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'\"}},\"required\":[\"expr\"]},\"sleep\":{\"description\":\"Delay before executing this step (in seconds or as expression)\",\"$ref\":\"#/components/schemas/InputTransform\"},\"cache_ttl\":{\"type\":\"number\",\"description\":\"Cache duration in seconds for this step's results\"},\"cache_ignore_s3_path\":{\"type\":\"boolean\"},\"timeout\":{\"description\":\"Maximum execution time in seconds (static value or expression)\",\"$ref\":\"#/components/schemas/InputTransform\"},\"delete_after_secs\":{\"type\":\"integer\",\"description\":\"If set, delete the step's args, result and logs after this many seconds following job completion\"},\"summary\":{\"type\":\"string\",\"description\":\"Short description of what this step does\"},\"mock\":{\"type\":\"object\",\"description\":\"Mock configuration for testing without executing the actual step\",\"properties\":{\"enabled\":{\"type\":\"boolean\",\"description\":\"If true, return mock value instead of executing\"},\"return_value\":{\"description\":\"Value to return when mocked\"}}},\"suspend\":{\"type\":\"object\",\"description\":\"Configuration for approval/resume steps that wait for user input\",\"properties\":{\"required_events\":{\"type\":\"integer\",\"description\":\"Number of approvals required before continuing\"},\"timeout\":{\"type\":\"integer\",\"description\":\"Timeout in seconds before auto-continuing or canceling\"},\"resume_form\":{\"type\":\"object\",\"description\":\"Form schema for collecting input when resuming\",\"properties\":{\"schema\":{\"type\":\"object\",\"description\":\"JSON Schema for the resume form\"}}},\"user_auth_required\":{\"type\":\"boolean\",\"description\":\"If true, only authenticated users can approve\"},\"user_groups_required\":{\"description\":\"Expression or list of groups that can approve\",\"$ref\":\"#/components/schemas/InputTransform\"},\"self_approval_disabled\":{\"type\":\"boolean\",\"description\":\"If true, the user who started the flow cannot approve\"},\"hide_cancel\":{\"type\":\"boolean\",\"description\":\"If true, hide the cancel button on the approval form\"},\"continue_on_disapprove_timeout\":{\"type\":\"boolean\",\"description\":\"If true, continue flow on timeout instead of canceling\"}}},\"priority\":{\"type\":\"number\",\"description\":\"Execution priority for this step (higher numbers run first)\"},\"continue_on_error\":{\"type\":\"boolean\",\"description\":\"If true, flow continues even if this step fails\"},\"retry\":{\"description\":\"Retry configuration if this step fails\",\"$ref\":\"#/components/schemas/Retry\"},\"debouncing\":{\"description\":\"Debounce configuration for this step (EE only)\",\"type\":\"object\",\"properties\":{\"debounce_delay_s\":{\"type\":\"integer\",\"description\":\"Delay in seconds to debounce this step's executions across flow runs\"},\"debounce_key\":{\"type\":\"string\",\"description\":\"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-\"},\"debounce_args_to_accumulate\":{\"type\":\"array\",\"description\":\"Array-type arguments to accumulate across debounced executions\",\"items\":{\"type\":\"string\"}},\"max_total_debouncing_time\":{\"type\":\"integer\",\"description\":\"Maximum total time in seconds before forced execution\"},\"max_total_debounces_amount\":{\"type\":\"integer\",\"description\":\"Maximum number of debounces before forced execution\"}}}},\"required\":[\"value\",\"id\"]},\"InputTransform\":{\"description\":\"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs\",\"oneOf\":[{\"$ref\":\"#/components/schemas/StaticTransform\"},{\"$ref\":\"#/components/schemas/JavascriptTransform\"},{\"$ref\":\"#/components/schemas/AiTransform\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"static\":\"#/components/schemas/StaticTransform\",\"javascript\":\"#/components/schemas/JavascriptTransform\",\"ai\":\"#/components/schemas/AiTransform\"}}},\"StaticTransform\":{\"type\":\"object\",\"description\":\"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'\",\"properties\":{\"value\":{\"description\":\"The static value. For resources, use format '$res:path/to/resource'\"},\"type\":{\"type\":\"string\",\"enum\":[\"static\"]}},\"required\":[\"type\"]},\"JavascriptTransform\":{\"type\":\"object\",\"description\":\"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value\",\"properties\":{\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)\"},\"type\":{\"type\":\"string\",\"enum\":[\"javascript\"]}},\"required\":[\"expr\",\"type\"]},\"AiTransform\":{\"type\":\"object\",\"description\":\"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"ai\"]}},\"required\":[\"type\"]},\"AIProviderKind\":{\"type\":\"string\",\"description\":\"Supported AI provider types\",\"enum\":[\"openai\",\"azure_openai\",\"azure_foundry\",\"anthropic\",\"mistral\",\"deepseek\",\"googleai\",\"groq\",\"openrouter\",\"togetherai\",\"customai\",\"aws_bedrock\"]},\"ProviderConfig\":{\"type\":\"object\",\"description\":\"Complete AI provider configuration with resource reference and model selection\",\"properties\":{\"kind\":{\"$ref\":\"#/components/schemas/AIProviderKind\"},\"resource\":{\"type\":\"string\",\"description\":\"Resource reference in format '$res:{resource_path}' pointing to provider credentials\"},\"model\":{\"type\":\"string\",\"description\":\"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')\"},\"reasoning_effort\":{\"type\":\"string\",\"description\":\"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.\"}},\"required\":[\"kind\",\"resource\",\"model\"]},\"StaticProviderTransform\":{\"type\":\"object\",\"description\":\"Static provider configuration passed directly to the AI agent\",\"properties\":{\"value\":{\"$ref\":\"#/components/schemas/ProviderConfig\"},\"type\":{\"type\":\"string\",\"enum\":[\"static\"]}},\"required\":[\"type\",\"value\"]},\"ProviderTransform\":{\"description\":\"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined\",\"oneOf\":[{\"$ref\":\"#/components/schemas/StaticProviderTransform\"},{\"$ref\":\"#/components/schemas/JavascriptTransform\"},{\"$ref\":\"#/components/schemas/AiTransform\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"static\":\"#/components/schemas/StaticProviderTransform\",\"javascript\":\"#/components/schemas/JavascriptTransform\",\"ai\":\"#/components/schemas/AiTransform\"}}},\"MemoryOff\":{\"type\":\"object\",\"description\":\"No conversation memory/context\",\"properties\":{\"kind\":{\"type\":\"string\",\"enum\":[\"off\"]}},\"required\":[\"kind\"]},\"MemoryAuto\":{\"type\":\"object\",\"description\":\"Automatic context management\",\"properties\":{\"kind\":{\"type\":\"string\",\"enum\":[\"auto\"]},\"context_length\":{\"type\":\"integer\",\"description\":\"Maximum number of messages to retain in context\"},\"memory_id\":{\"type\":\"string\",\"description\":\"Identifier for persistent memory across agent invocations\"}},\"required\":[\"kind\"]},\"MemoryMessage\":{\"type\":\"object\",\"description\":\"A single message in conversation history\",\"properties\":{\"role\":{\"type\":\"string\",\"enum\":[\"user\",\"assistant\",\"system\"]},\"content\":{\"type\":\"string\"}},\"required\":[\"role\",\"content\"]},\"MemoryManual\":{\"type\":\"object\",\"description\":\"Explicit message history\",\"properties\":{\"kind\":{\"type\":\"string\",\"enum\":[\"manual\"]},\"messages\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/MemoryMessage\"}}},\"required\":[\"kind\",\"messages\"]},\"MemoryConfig\":{\"description\":\"Conversation memory configuration\",\"oneOf\":[{\"$ref\":\"#/components/schemas/MemoryOff\"},{\"$ref\":\"#/components/schemas/MemoryAuto\"},{\"$ref\":\"#/components/schemas/MemoryManual\"}],\"discriminator\":{\"propertyName\":\"kind\",\"mapping\":{\"off\":\"#/components/schemas/MemoryOff\",\"auto\":\"#/components/schemas/MemoryAuto\",\"manual\":\"#/components/schemas/MemoryManual\"}}},\"StaticMemoryTransform\":{\"type\":\"object\",\"description\":\"Static memory configuration passed directly to the AI agent\",\"properties\":{\"value\":{\"$ref\":\"#/components/schemas/MemoryConfig\"},\"type\":{\"type\":\"string\",\"enum\":[\"static\"]}},\"required\":[\"type\",\"value\"]},\"MemoryTransform\":{\"description\":\"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined\",\"oneOf\":[{\"$ref\":\"#/components/schemas/StaticMemoryTransform\"},{\"$ref\":\"#/components/schemas/JavascriptTransform\"},{\"$ref\":\"#/components/schemas/AiTransform\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"static\":\"#/components/schemas/StaticMemoryTransform\",\"javascript\":\"#/components/schemas/JavascriptTransform\",\"ai\":\"#/components/schemas/AiTransform\"}}},\"FlowModuleValue\":{\"description\":\"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type\",\"oneOf\":[{\"$ref\":\"#/components/schemas/RawScript\"},{\"$ref\":\"#/components/schemas/PathScript\"},{\"$ref\":\"#/components/schemas/PathFlow\"},{\"$ref\":\"#/components/schemas/ForloopFlow\"},{\"$ref\":\"#/components/schemas/WhileloopFlow\"},{\"$ref\":\"#/components/schemas/BranchOne\"},{\"$ref\":\"#/components/schemas/BranchAll\"},{\"$ref\":\"#/components/schemas/Identity\"},{\"$ref\":\"#/components/schemas/AiAgent\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"rawscript\":\"#/components/schemas/RawScript\",\"script\":\"#/components/schemas/PathScript\",\"flow\":\"#/components/schemas/PathFlow\",\"forloopflow\":\"#/components/schemas/ForloopFlow\",\"whileloopflow\":\"#/components/schemas/WhileloopFlow\",\"branchone\":\"#/components/schemas/BranchOne\",\"branchall\":\"#/components/schemas/BranchAll\",\"identity\":\"#/components/schemas/Identity\",\"aiagent\":\"#/components/schemas/AiAgent\"}}},\"RawScript\":{\"type\":\"object\",\"description\":\"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments\",\"additionalProperties\":{\"$ref\":\"#/components/schemas/InputTransform\"}},\"content\":{\"type\":\"string\",\"description\":\"The script source code. Should export a 'main' function\"},\"language\":{\"type\":\"string\",\"description\":\"Programming language for this script\",\"enum\":[\"deno\",\"bun\",\"python3\",\"go\",\"bash\",\"powershell\",\"postgresql\",\"mysql\",\"bigquery\",\"snowflake\",\"mssql\",\"oracledb\",\"graphql\",\"nativets\",\"php\",\"rust\",\"ansible\",\"csharp\",\"nu\",\"java\",\"ruby\",\"rlang\",\"duckdb\"]},\"path\":{\"type\":\"string\",\"description\":\"Optional path for saving this script\"},\"lock\":{\"type\":\"string\",\"description\":\"Lock file content for dependencies\"},\"type\":{\"type\":\"string\",\"enum\":[\"rawscript\"]},\"tag\":{\"type\":\"string\",\"description\":\"Worker group tag for execution routing\"},\"concurrent_limit\":{\"type\":\"number\",\"description\":\"Maximum concurrent executions of this script\"},\"concurrency_time_window_s\":{\"type\":\"number\",\"description\":\"Time window for concurrent_limit\"},\"custom_concurrency_key\":{\"type\":\"string\",\"description\":\"Custom key for grouping concurrent executions\"},\"is_trigger\":{\"type\":\"boolean\",\"description\":\"If true, this script is a trigger that can start the flow\"},\"assets\":{\"type\":\"array\",\"description\":\"External resources this script accesses (S3 objects, resources, etc.)\",\"items\":{\"type\":\"object\",\"required\":[\"path\",\"kind\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the asset\"},\"kind\":{\"type\":\"string\",\"description\":\"Type of asset\",\"enum\":[\"s3object\",\"resource\",\"ducklake\",\"datatable\",\"volume\"]},\"access_type\":{\"type\":\"string\",\"nullable\":true,\"description\":\"Access level for this asset\",\"enum\":[\"r\",\"w\",\"rw\"]},\"alt_access_type\":{\"type\":\"string\",\"nullable\":true,\"description\":\"Alternative access level\",\"enum\":[\"r\",\"w\",\"rw\"]}}}}},\"required\":[\"type\",\"content\",\"language\",\"input_transforms\"]},\"PathScript\":{\"type\":\"object\",\"description\":\"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments\",\"additionalProperties\":{\"$ref\":\"#/components/schemas/InputTransform\"}},\"path\":{\"type\":\"string\",\"description\":\"Path to the script in the workspace (e.g., 'f/scripts/send_email')\"},\"hash\":{\"type\":\"string\",\"description\":\"Optional specific version hash of the script to use\"},\"type\":{\"type\":\"string\",\"enum\":[\"script\"]},\"tag_override\":{\"type\":\"string\",\"description\":\"Override the script's default worker group tag\"},\"is_trigger\":{\"type\":\"boolean\",\"description\":\"If true, this script is a trigger that can start the flow\"}},\"required\":[\"type\",\"path\",\"input_transforms\"]},\"PathFlow\":{\"type\":\"object\",\"description\":\"Reference to an existing flow by path. Use this to call another flow as a subflow\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments\",\"additionalProperties\":{\"$ref\":\"#/components/schemas/InputTransform\"}},\"path\":{\"type\":\"string\",\"description\":\"Path to the flow in the workspace (e.g., 'f/flows/process_user')\"},\"type\":{\"type\":\"string\",\"enum\":[\"flow\"]}},\"required\":[\"type\",\"path\",\"input_transforms\"]},\"ForloopFlow\":{\"type\":\"object\",\"description\":\"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations\",\"properties\":{\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"iterator\":{\"description\":\"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'\",\"$ref\":\"#/components/schemas/InputTransform\"},\"skip_failures\":{\"type\":\"boolean\",\"description\":\"If true, iteration failures don't stop the loop. Failed iterations return null\"},\"type\":{\"type\":\"string\",\"enum\":[\"forloopflow\"]},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency\"},\"parallelism\":{\"description\":\"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression\",\"$ref\":\"#/components/schemas/InputTransform\"},\"squash\":{\"type\":\"boolean\"}},\"required\":[\"modules\",\"iterator\",\"skip_failures\",\"type\"]},\"WhileloopFlow\":{\"type\":\"object\",\"description\":\"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination\",\"properties\":{\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute in each iteration. Use stop_after_if to control when the loop ends\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"skip_failures\":{\"type\":\"boolean\",\"description\":\"If true, iteration failures don't stop the loop. Failed iterations return null\"},\"type\":{\"type\":\"string\",\"enum\":[\"whileloopflow\"]},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, iterations run concurrently (use with caution in while loops)\"},\"parallelism\":{\"description\":\"Maximum number of concurrent iterations when parallel=true\",\"$ref\":\"#/components/schemas/InputTransform\"},\"squash\":{\"type\":\"boolean\"}},\"required\":[\"modules\",\"skip_failures\",\"type\"]},\"BranchOne\":{\"type\":\"object\",\"description\":\"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes\",\"properties\":{\"branches\":{\"type\":\"array\",\"description\":\"Array of branches to evaluate in order. The first branch with expr evaluating to true executes\",\"items\":{\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Short description of this branch condition\"},\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins\"},\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute if this branch's expr is true\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}}},\"required\":[\"modules\",\"expr\"]}},\"default\":{\"type\":\"array\",\"description\":\"Steps to execute if no branch expressions match\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"type\":{\"type\":\"string\",\"enum\":[\"branchone\"]}},\"required\":[\"branches\",\"default\",\"type\"]},\"BranchAll\":{\"type\":\"object\",\"description\":\"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently\",\"properties\":{\"branches\":{\"type\":\"array\",\"description\":\"Array of branches that all execute (either in parallel or sequentially)\",\"items\":{\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Short description of this branch's purpose\"},\"skip_failure\":{\"type\":\"boolean\",\"description\":\"If true, failure in this branch doesn't fail the entire flow\"},\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute in this branch\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}}},\"required\":[\"modules\"]}},\"type\":{\"type\":\"string\",\"enum\":[\"branchall\"]},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, all branches execute concurrently. If false, they execute sequentially\"}},\"required\":[\"branches\",\"type\"]},\"AgentTool\":{\"type\":\"object\",\"description\":\"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')\"},\"summary\":{\"type\":\"string\",\"description\":\"Short description of what this tool does (shown to the AI)\"},\"value\":{\"$ref\":\"#/components/schemas/ToolValue\"}},\"required\":[\"id\",\"value\"]},\"ToolValue\":{\"description\":\"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference\",\"oneOf\":[{\"$ref\":\"#/components/schemas/FlowModuleTool\"},{\"$ref\":\"#/components/schemas/McpToolValue\"},{\"$ref\":\"#/components/schemas/WebsearchToolValue\"}],\"discriminator\":{\"propertyName\":\"tool_type\",\"mapping\":{\"flowmodule\":\"#/components/schemas/FlowModuleTool\",\"mcp\":\"#/components/schemas/McpToolValue\",\"websearch\":\"#/components/schemas/WebsearchToolValue\"}}},\"FlowModuleTool\":{\"description\":\"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module\",\"allOf\":[{\"type\":\"object\",\"properties\":{\"tool_type\":{\"type\":\"string\",\"enum\":[\"flowmodule\"]}},\"required\":[\"tool_type\"]},{\"$ref\":\"#/components/schemas/FlowModuleValue\"}]},\"WebsearchToolValue\":{\"type\":\"object\",\"description\":\"A tool implemented as a websearch tool. The AI can call this like any other websearch tool\",\"properties\":{\"tool_type\":{\"type\":\"string\",\"enum\":[\"websearch\"]}},\"required\":[\"tool_type\"]},\"McpToolValue\":{\"type\":\"object\",\"description\":\"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers\",\"properties\":{\"tool_type\":{\"type\":\"string\",\"enum\":[\"mcp\"]},\"resource_path\":{\"type\":\"string\",\"description\":\"Path to the MCP resource/server configuration\"},\"include_tools\":{\"type\":\"array\",\"description\":\"Whitelist of specific tools to include from this MCP server\",\"items\":{\"type\":\"string\"}},\"exclude_tools\":{\"type\":\"array\",\"description\":\"Blacklist of tools to exclude from this MCP server\",\"items\":{\"type\":\"string\"}}},\"required\":[\"tool_type\",\"resource_path\"]},\"AiAgent\":{\"type\":\"object\",\"description\":\"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Input parameters for the AI agent mapped to their values\",\"properties\":{\"provider\":{\"$ref\":\"#/components/schemas/ProviderTransform\"},\"output_type\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n\"},\"user_message\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax.\"},\"system_prompt\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"System instructions that guide the AI's behavior, persona, and response style. Optional.\"},\"streaming\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n\"},\"memory\":{\"$ref\":\"#/components/schemas/MemoryTransform\"},\"output_schema\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n\"},\"user_attachments\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n\"},\"max_completion_tokens\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n\"},\"temperature\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n\"},\"max_iterations\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n\"}},\"required\":[\"provider\",\"user_message\",\"output_type\"]},\"tools\":{\"type\":\"array\",\"description\":\"Array of tools the agent can use. The agent decides which tools to call based on the task\",\"items\":{\"$ref\":\"#/components/schemas/AgentTool\"}},\"type\":{\"type\":\"string\",\"enum\":[\"aiagent\"]},\"tag\":{\"type\":\"string\",\"description\":\"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)\"},\"omit_output_from_conversation\":{\"type\":\"boolean\",\"default\":false,\"description\":\"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.\"},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, the agent can execute multiple tool calls in parallel\"}},\"required\":[\"tools\",\"type\",\"input_transforms\"]},\"Identity\":{\"type\":\"object\",\"description\":\"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"identity\"]},\"flow\":{\"type\":\"boolean\",\"description\":\"If true, marks this as a flow identity (special handling)\"}},\"required\":[\"type\"]},\"FlowStatus\":{\"type\":\"object\",\"properties\":{\"step\":{\"type\":\"integer\"},\"modules\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/FlowStatusModule\"}},\"user_states\":{\"additionalProperties\":true},\"preprocessor_module\":{\"allOf\":[{\"$ref\":\"#/components/schemas/FlowStatusModule\"}]},\"failure_module\":{\"allOf\":[{\"$ref\":\"#/components/schemas/FlowStatusModule\"},{\"type\":\"object\",\"properties\":{\"parent_module\":{\"type\":\"string\"}}}]},\"retry\":{\"type\":\"object\",\"properties\":{\"fail_count\":{\"type\":\"integer\"},\"failed_jobs\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"format\":\"uuid\"}}}}},\"required\":[\"step\",\"modules\",\"failure_module\"]},\"FlowStatusModule\":{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"WaitingForPriorSteps\",\"WaitingForEvents\",\"WaitingForExecutor\",\"InProgress\",\"Success\",\"Failure\"]},\"id\":{\"type\":\"string\"},\"job\":{\"type\":\"string\",\"format\":\"uuid\"},\"count\":{\"type\":\"integer\"},\"progress\":{\"type\":\"integer\"},\"iterator\":{\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"integer\"},\"itered\":{\"type\":\"array\",\"items\":{}},\"itered_len\":{\"type\":\"integer\"},\"args\":{}}},\"flow_jobs\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"flow_jobs_success\":{\"type\":\"array\",\"items\":{\"type\":\"boolean\"}},\"flow_jobs_duration\":{\"type\":\"object\",\"properties\":{\"started_at\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"duration_ms\":{\"type\":\"array\",\"items\":{\"type\":\"integer\"}}}},\"branch_chosen\":{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"branch\",\"default\"]},\"branch\":{\"type\":\"integer\"}},\"required\":[\"type\"]},\"branchall\":{\"type\":\"object\",\"properties\":{\"branch\":{\"type\":\"integer\"},\"len\":{\"type\":\"integer\"}},\"required\":[\"branch\",\"len\"]},\"approvers\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"resume_id\":{\"type\":\"integer\"},\"approver\":{\"type\":\"string\"}},\"required\":[\"resume_id\",\"approver\"]}},\"failed_retries\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"format\":\"uuid\"}},\"skipped\":{\"type\":\"boolean\"},\"agent_actions\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"oneOf\":[{\"type\":\"object\",\"properties\":{\"job_id\":{\"type\":\"string\",\"format\":\"uuid\"},\"function_name\":{\"type\":\"string\"},\"type\":{\"type\":\"string\",\"enum\":[\"tool_call\"]},\"module_id\":{\"type\":\"string\"}},\"required\":[\"job_id\",\"function_name\",\"type\",\"module_id\"]},{\"type\":\"object\",\"properties\":{\"call_id\":{\"type\":\"string\",\"format\":\"uuid\"},\"function_name\":{\"type\":\"string\"},\"resource_path\":{\"type\":\"string\"},\"type\":{\"type\":\"string\",\"enum\":[\"mcp_tool_call\"]},\"arguments\":{\"type\":\"object\"}},\"required\":[\"call_id\",\"function_name\",\"resource_path\",\"type\"]},{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"web_search\"]}},\"required\":[\"type\"]},{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"message\"]}},\"required\":[\"content\",\"type\"]}]}},\"agent_actions_success\":{\"type\":\"array\",\"items\":{\"type\":\"boolean\"}}},\"required\":[\"type\"]}}"; +export declare const CLI_COMMANDS = "# Windmill CLI Commands\n\nThe Windmill CLI (`wmill`) provides commands for managing scripts, flows, apps, and other resources.\n\n## Global Options\n\n- `--workspace ` - Specify the target workspace. This overrides the default workspace.\n- `--debug --verbose` - Show debug/verbose logs\n- `--show-diffs` - Show diff informations when syncing (may show sensitive informations)\n- `--token ` - Specify an API token. This will override any stored token.\n- `--base-url ` - Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used.\n- `--config-dir ` - Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location.\n\n## Commands\n\n### app\n\napp related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `app list` - list all apps\n - `--json` - Output as JSON (for piping to jq)\n- `app get ` - get an app's details\n - `--json` - Output as JSON (for piping to jq)\n- `app push [file_path:string] [remote_path:string]` - push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml.\n- `app dev [app_folder:string]` - Start a development server for building apps with live reload and hot module replacement\n - `--port ` - Port to run the dev server on (will find next available port if occupied)\n - `--host ` - Host to bind the dev server to\n - `--entry ` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)\n - `--no-open` - Don't automatically open the browser\n- `app lint [app_folder:string]` - Lint a raw app folder to validate structure and buildability\n - `--fix` - Attempt to fix common issues (not implemented yet)\n- `app new` - create a new raw app from a template\n - `--summary ` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode.\n - `--path ` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode.\n - `--framework ` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode.\n - `--datatable ` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured.\n - `--schema ` - Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist.\n - `--overwrite` - Overwrite the target directory if it already exists, without prompting.\n - `--no-open-in-desktop` - Do not prompt to open the new app in Claude Desktop.\n- `app generate-agents [app_folder:string]` - regenerate AGENTS.md and DATATABLES.md from remote workspace\n- `app set-permissioned-as ` - Set the on_behalf_of_email for an app (requires admin or wm_deployers group)\n\n### audit\n\nView audit logs (requires admin)\n\n**Subcommands:**\n\n- `audit list` - List audit log entries\n- `audit get ` - Get a specific audit log entry\n - `--json` - Output as JSON (for piping to jq)\n\n### config\n\nShow all available wmill.yaml configuration options\n\n**Options:**\n- `--json` - Output as JSON for programmatic consumption\n\n**Subcommands:**\n\n- `config migrate` - Migrate wmill.yaml from gitBranches/environments to workspaces format\n\n### datatable\n\ndatatable related commands\n\n**Subcommands:**\n\n- `datatable list` - list all datatables in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `datatable run ` - run a SQL query on a datatable\n - `-n --name ` - Datatable name (default: main)\n - `-s --silent` - Output only the final result as JSON. Useful for scripting.\n- `datatable migrate` - manage datatable migrations\n - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files)\n - `-d --datatable ` - Target datatable (default: main)\n - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable)\n - `-d --datatable ` - Target datatable (default: main)\n - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable)\n - `-d --datatable ` - Target datatable (default: main)\n- `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://\n - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database\n - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved)\n- `datatable serve` - Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string\n - `--port ` - Port to listen on (default: first free port in 5433-5500)\n - `--host ` - Bind address (default: 127.0.0.1)\n - `--password ` - Password for Postgres clients (default: generate a random password at startup)\n- `datatable psql` - Start a serve listener and launch psql connected to it\n - `-n --name ` - Datatable to connect psql to (default: main)\n - `--port ` - Port the proxy listens on (default: first free port in 5433-5500)\n - `--host ` - Bind address for the proxy (default: 127.0.0.1)\n - `--password ` - Password for the temporary Postgres proxy (default: generate a random password at startup)\n\n### dependencies\n\nworkspace dependencies related commands\n\n**Alias:** `deps`\n\n**Subcommands:**\n\n- `dependencies push ` - Push workspace dependencies from a local file\n\n### dev\n\nWatch local file changes and live-reload the dev page for preview. Does NOT deploy to the remote workspace \u2014 use wmill sync push for that.\n\n**Options:**\n- `--includes ` - Filter paths given a glob pattern or path\n- `--proxy-port ` - Port for a localhost reverse proxy to the remote Windmill server\n- `--path ` - Watch a specific windmill path (e.g., u/admin/my_script or f/my_flow)\n- `--no-open` - Do not open the browser automatically\n\n### docs\n\nSearch Windmill documentation.\n\n**Arguments:** ``\n\n**Options:**\n- `--json` - Output results as JSON.\n\n### ducklake\n\nducklake related commands\n\n**Subcommands:**\n\n- `ducklake list` - list all ducklakes in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `ducklake run ` - run a SQL query on a ducklake\n - `-n --name ` - Ducklake name (default: main)\n - `-s --silent` - Output only the final result as JSON. Useful for scripting.\n\n### flow\n\nflow related commands\n\n**Options:**\n- `--show-archived` - Enable archived flows in output\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `flow list` - list all flows\n - `--show-archived` - Enable archived flows in output\n - `--json` - Output as JSON (for piping to jq)\n- `flow get ` - get a flow's details\n - `--json` - Output as JSON (for piping to jq)\n- `flow push ` - push a local flow spec. This overrides any remote versions.\n - `--message ` - Deployment message\n- `flow run ` - run a flow by path.\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting.\n- `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow).\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not output anything other then the final output. Useful for scripting.\n - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files.\n - `--step ` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does.\n- `flow new ` - create a new empty flow\n - `--summary ` - flow summary\n - `--description ` - flow description\n- `flow bootstrap ` - create a new empty flow (alias for new)\n - `--summary ` - flow summary\n - `--description ` - flow description\n- `flow history ` - Show version history for a flow\n - `--json` - Output as JSON (for piping to jq)\n- `flow show-version ` - Show a specific version of a flow\n - `--json` - Output as JSON (for piping to jq)\n- `flow set-permissioned-as ` - Set the on_behalf_of_email for a flow (requires admin or wm_deployers group)\n\n### folder\n\nfolder related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `folder list` - list all folders\n - `--json` - Output as JSON (for piping to jq)\n- `folder get ` - get a folder's details\n - `--json` - Output as JSON (for piping to jq)\n- `folder new ` - create a new folder locally\n - `--summary ` - folder summary\n- `folder push ` - push a local folder to the remote by name. This overrides any remote versions.\n- `folder add-missing` - create default folder.meta.yaml for all subdirectories of f/ that are missing one\n - `-y, --yes` - skip confirmation prompt\n- `folder show-rules ` - Show default_permissioned_as rules for a folder. Use --test-path to see which rule matches a given item path.\n - `--test-path ` - Test which rule matches this item path (e.g. f/prod/jobs/my_script)\n - `--json` - Output as JSON\n\n### generate-metadata\n\nRegenerate stale local locks and script schemas and refresh wmill-lock.yaml content hashes (scripts, flows, apps). Writes local files only, not a deploy. Run it after edits that add or remove imports or change a script's arguments, so the lock, the auto-generated UI schema, and wmill-lock.yaml stay in sync.\n\n**Arguments:** `[folder:string]`\n\n**Options:**\n- `--yes` - Skip confirmation prompt\n- `--dry-run` - Show what would be updated without making changes\n- `--lock-only` - Re-generate only the lock files\n- `--schema-only` - Re-generate only script schemas (skips flows and apps)\n- `--skip-scripts` - Skip processing scripts\n- `--skip-flows` - Skip processing flows\n- `--skip-apps` - Skip processing apps\n- `--strict-folder-boundaries` - Only update items inside the specified folder (requires folder argument)\n- `--parallel ` - Number of items to process in parallel\n- `-i --includes ` - Comma separated patterns to specify which files to include\n- `-e --excludes ` - Comma separated patterns to specify which files to exclude\n\n**Subcommands:**\n\n- `generate-metadata rehash [folder:string]` - Refresh wmill-lock.yaml content hashes from the on-disk .lock and .script.yaml without re-resolving dependencies or hitting the backend. Use when those files are already correct and only the hashes need updating: bootstrapping missing entries or recovering from hash drift.\n - `--skip-scripts` - Skip processing scripts\n - `--skip-flows` - Skip processing flows\n - `--skip-apps` - Skip processing apps\n - `--parallel ` - Number of items to process in parallel\n - `-i --includes ` - Comma separated patterns to specify which files to include\n - `-e --excludes ` - Comma separated patterns to specify which files to exclude\n\n### gitsync-settings\n\nManage git-sync settings between local wmill.yaml and Windmill backend\n\n**Subcommands:**\n\n- `gitsync-settings pull` - Pull git-sync settings from Windmill backend to local wmill.yaml\n - `--repository ` - Specify repository path (e.g., u/user/repo)\n - `--default` - Write settings to top-level defaults instead of overrides\n - `--replace` - Replace existing settings (non-interactive mode)\n - `--override` - Add branch-specific override (non-interactive mode)\n - `--diff` - Show differences without applying changes\n - `--json-output` - Output in JSON format\n - `--with-backend-settings ` - Use provided JSON settings instead of querying backend (for testing)\n - `--yes` - Skip interactive prompts and use default behavior\n - `--promotion ` - Use promotionOverrides from the specified branch instead of regular overrides\n- `gitsync-settings push` - Push git-sync settings from local wmill.yaml to Windmill backend\n - `--repository ` - Specify repository path (e.g., u/user/repo)\n - `--diff` - Show what would be pushed without applying changes\n - `--json-output` - Output in JSON format\n - `--with-backend-settings ` - Use provided JSON settings instead of querying backend (for testing)\n - `--yes` - Skip interactive prompts and use default behavior\n - `--promotion ` - Use promotionOverrides from the specified branch instead of regular overrides\n\n### group\n\nManage workspace groups\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `group list` - List all groups in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `group get ` - Get group details and members\n - `--json` - Output as JSON (for piping to jq)\n- `group create ` - Create a new group\n - `--summary ` - Group summary/description\n- `group delete ` - Delete a group\n- `group add-user ` - Add a user to a group\n- `group remove-user ` - Remove a user from a group\n\n### hub\n\nHub related commands. EXPERIMENTAL. INTERNAL USE ONLY.\n\n**Subcommands:**\n\n- `hub pull` - pull any supported definitions. EXPERIMENTAL.\n\n### init\n\nBootstrap a windmill project with a wmill.yaml file\n\n**Options:**\n- `--use-default` - Use default settings without checking backend\n- `--use-backend` - Use backend git-sync settings if available\n- `--repository ` - Specify repository path (e.g., u/user/repo) when using backend settings\n- `--bind-profile` - Automatically bind active workspace profile to current Git branch\n- `--no-bind-profile` - Skip workspace profile binding prompt\n\n### instance\n\nsync local with a remote instance or the opposite (push or pull)\n\n**Subcommands:**\n\n- `instance add [instance_name:string] [remote:string] [token:string]` - Add a new instance\n- `instance remove ` - Remove an instance\n- `instance switch ` - Switch the current instance\n- `instance pull` - Pull instance settings, users, configs, instance groups and overwrite local\n - `--yes` - Pull without needing confirmation\n - `--dry-run` - Perform a dry run without making changes\n - `--skip-users` - Skip pulling users\n - `--skip-settings` - Skip pulling settings\n - `--skip-configs` - Skip pulling configs (worker groups)\n - `--skip-groups` - Skip pulling instance groups\n - `--include-workspaces` - Also pull workspaces\n - `--folder-per-instance` - Create a folder per instance\n - `--instance ` - Name of the instance to pull from, override the active instance\n - `--prefix ` - Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces\n - `--prefix-settings` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance\n- `instance push` - Push instance settings, users, configs, group and overwrite remote\n - `--yes` - Push without needing confirmation\n - `--dry-run` - Perform a dry run without making changes\n - `--skip-users` - Skip pushing users\n - `--skip-settings` - Skip pushing settings\n - `--skip-configs` - Skip pushing configs (worker groups)\n - `--skip-groups` - Skip pushing instance groups\n - `--include-workspaces` - Also push workspaces\n - `--folder-per-instance` - Create a folder per instance\n - `--instance ` - Name of the instance to push to, override the active instance\n - `--prefix ` - Prefix of the local workspaces folders to push\n - `--prefix-settings` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance\n- `instance whoami` - Display information about the currently logged-in user\n- `instance get-config` - Dump the current instance config (global settings + worker configs) as YAML\n - `-o, --output-file ` - Write YAML to a file instead of stdout\n - `--show-secrets` - Include sensitive fields (license key, JWT secret) without prompting\n - `--instance ` - Name of the instance, override the active instance\n- `instance connect-slack` - Non-interactively connect Slack at the instance level using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: global_settings 'slack' row + encrypted f/slack_bot/global_bot_token variable and resource in the admins workspace.\n - `--bot-token ` - Slack bot token (xoxb-...)\n - `--team-id ` - Slack team id\n - `--team-name ` - Slack team name\n - `--instance ` - Instance profile to connect against (defaults to the active instance)\n\n### job\n\nManage jobs (list, inspect, cancel)\n\n**Subcommands:**\n\n- `job list` - List recent jobs\n- `job get ` - Get job details. For flows: shows step tree with sub-job IDs\n - `--json` - Output as JSON (for piping to jq)\n- `job result ` - Get the result of a completed job (machine-friendly)\n- `job logs ` - Get job logs. For flows: aggregates all step logs\n- `job cancel ` - Cancel a running or queued job\n - `--reason ` - Reason for cancellation\n- `job rerun ` - Re-run a completed job with the same args. Prints the new job UUID on stdout.\n- `job restart ` - Restart a completed flow at a given top-level step. Prints the new flow job UUID on stdout.\n - `--step ` - Top-level step id to restart the flow from\n - `--iteration ` - For a top-level branchall or for-loop step, the iteration to restart at\n\n### jobs\n\nManage jobs (import/export)\n\n**Subcommands:**\n\n- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace\n - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json)\n - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json)\n - `--skip-worker-check` - Skip checking for active workers before export\n- `jobs push [workspace:string]` - Push completed and queued jobs to workspace\n - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json)\n - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json)\n - `--skip-worker-check` - Skip checking for active workers before import\n\n### lint\n\nValidate Windmill flow, schedule, and trigger YAML files in a directory\n\n**Arguments:** `[directory:string]`\n\n**Options:**\n- `--json` - Output results in JSON format\n- `--fail-on-warn` - Exit with code 1 when warnings are emitted\n- `--locks-required` - Fail if scripts or flow inline scripts that need locks have no locks\n- `-w, --watch` - Watch for file changes and re-lint automatically\n\n### object-storage\n\nObject storage (S3) related commands. Operates on the workspace's default object storage; use --storage to target a configured secondary storage.\n\n**Alias:** `s3`\n\n**Subcommands:**\n\n- `object-storage list` - List configured object storages for the workspace (default + secondary).\n - `--json` - Output as JSON (for piping to jq)\n- `object-storage files [prefix:string]` - List files in an object storage. Optionally filter by prefix.\n - `--json` - Output as JSON (for piping to jq)\n - `--max-keys ` - Page size (default 100)\n - `--marker ` - Pagination marker from a previous response\n - `--storage ` - Secondary storage name (omit for the workspace default)\n- `object-storage upload ` - Upload a local file to object storage at the given file key.\n - `--storage ` - Secondary storage name\n - `--content-type ` - Content-Type header to set on the object\n - `--content-disposition ` - Content-Disposition header to set on the object\n- `object-storage download [output_path:string]` - Download an object to a local file (or stdout). Default output path is the basename of the file key in the current directory.\n - `--storage ` - Secondary storage name\n - `--stdout` - Write file contents to stdout instead of a file\n- `object-storage delete ` - Delete an object from object storage. Prompts for confirmation unless --yes is set.\n - `--storage ` - Secondary storage name\n - `--yes` - Skip the confirmation prompt\n- `object-storage move ` - Move an object within the same storage (rename or relocate by key).\n - `--storage ` - Secondary storage name\n- `object-storage info ` - Show metadata (size, mime, last-modified) for an object.\n - `--json` - Output as JSON (for piping to jq)\n - `--storage ` - Secondary storage name\n- `object-storage preview ` - Preview the contents of an object (text/CSV). Use --bytes-from / --bytes-length to peek at a slice of binary files.\n - `--storage ` - Secondary storage name\n - `--mime ` - Override the detected mime type (e.g. text/csv)\n - `--bytes-from ` - Start offset in bytes\n - `--bytes-length ` - Number of bytes to read\n - `--csv-separator ` - CSV column separator (default ,)\n - `--csv-header` - Treat the first CSV row as a header\n\n### pipeline\n\ninspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on ` annotations)\n\n**Subcommands:**\n\n- `pipeline list` - list pipeline folders in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `pipeline show ` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal\n - `--json` - Output the raw asset graph as JSON\n - `--local` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace \u2014 no deploy needed.\n- `pipeline run ` - run a cascade: from --from (a root OR any mid-DAG model), fan downstream up to the --to end node(s)\n - `--from ` - Start script (short name or path). May be any node, including a mid-DAG model \u2014 that node plus its transitive downstream runs, upstream is NOT re-run (dbt `--select model+`). Defaults to the folder's sole schedule/manual root.\n - `--to ` - End node(s) to stop at \u2014 script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream.\n - `--dry-run` - Print the topological run plan without executing.\n - `--json` - Output the plan as JSON (for piping to jq).\n - `--local` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files.\n - `--upload ` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable.\n - `--arg ` - Pass a plain run arg to a script in the cascade, as SCRIPT:PARAM=VALUE (VALUE is parsed as JSON when possible, else taken as a string \u2014 e.g. daily_report:partition=2026-07-02). Repeatable.\n - `--partition ` - Partition value for `// partitioned` scripts in the run (e.g. 2026-06-30) \u2014 use it to backfill a past slice. With --local, time kinds (daily/hourly/weekly/monthly) default to the current UTC period when omitted; `dynamic` always needs it. Deployed runs without it defer to backend run-start resolution.\n- `pipeline docs ` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop\n - `--local` - Build the graph from local working-tree files instead of the deployed workspace.\n- `pipeline dev [folder:string]` - Live-preview a data pipeline from local files: watch an `f/` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).\n - `--port ` - Port for the dev WebSocket server.\n - `--no-open` - Do not open the browser automatically.\n - `--frontend ` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.\n\n### protection-rules\n\nSync workspace protection rules between protection-rules.yaml and Windmill. The file is keyed by workspace name; keys must match wmill.yaml 'workspaces'.\n\n**Subcommands:**\n\n- `protection-rules pull [workspace:string]` - Pull protection rules from Windmill into protection-rules.yaml for a workspace\n - `--all` - Pull every workspace defined in wmill.yaml\n - `--dry-run` - Show what would change without writing the file\n - `--json-output` - Output in JSON format\n- `protection-rules push [workspace:string]` - Push protection rules from protection-rules.yaml to Windmill for a workspace (full reconcile: creates, updates, and deletes)\n - `--all` - Push every workspace defined in protection-rules.yaml\n - `--dry-run` - Show what would change without applying\n - `--json-output` - Output in JSON format\n - `--yes` - Skip the confirmation prompt (including deletions)\n\n### queues\n\nList all queues with their metrics\n\n**Arguments:** `[workspace:string] the optional workspace to filter by (default to all workspaces)`\n\n**Options:**\n- `--instance [instance]` - Name of the instance to push to, override the active instance\n- `--base-url [baseUrl]` - If used with --token, will be used as the base url for the instance\n\n### refresh\n\nRefresh wmill-managed project files (AGENTS.wmill.md, skills, tsconfig.wmill.json)\n\n**Subcommands:**\n\n- `refresh prompts` - Refresh AGENTS.wmill.md and managed skills. User-owned AGENTS.md and CLAUDE.md are never overwritten unless you opt in.\n - `--yes` - Non-interactive: append the @AGENTS.wmill.md include to an existing AGENTS.md / CLAUDE.md without prompting. Without it, a non-interactive run leaves an unlinked file untouched.\n- `refresh tsconfig` - Refresh the wmill-managed tsconfig.wmill.json (and Deno import map for Deno projects)\n - `--yes` - Non-interactive: wire an existing custom tsconfig.json/deno.json to the managed file without prompting (a previously-generated config is always migrated automatically).\n\n### resource\n\nresource related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `resource list` - list all resources\n - `--json` - Output as JSON (for piping to jq)\n- `resource get ` - get a resource's details\n - `--json` - Output as JSON (for piping to jq)\n- `resource new ` - create a new resource locally\n- `resource push ` - push a local resource spec. This overrides any remote versions.\n\n### resource-type\n\nresource type related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `resource-type list` - list all resource types\n - `--schema` - Show schema in the output\n - `--json` - Output as JSON (for piping to jq)\n- `resource-type get ` - get a resource type's details\n - `--json` - Output as JSON (for piping to jq)\n- `resource-type new ` - create a new resource type locally\n- `resource-type push ` - push a local resource spec. This overrides any remote versions.\n- `resource-type generate-namespace` - Create a TypeScript definition file with the RT namespace generated from the resource types\n\n### schedule\n\nschedule related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `schedule list` - list all schedules\n - `--json` - Output as JSON (for piping to jq)\n- `schedule get ` - get a schedule's details\n - `--json` - Output as JSON (for piping to jq)\n- `schedule new ` - create a new schedule locally\n- `schedule push ` - push a local schedule spec. This overrides any remote versions.\n- `schedule enable ` - Enable a schedule\n - `--force` - Bypass the fork-conflict warning when the parent workspace has the same schedule (acknowledges that both crons will fire)\n- `schedule disable ` - Disable a schedule\n- `schedule set-permissioned-as ` - Set the email (run-as user) for a schedule (requires admin or wm_deployers group)\n\n### script\n\nscript related commands\n\n**Options:**\n- `--show-archived` - Show archived scripts instead of active ones\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `script list` - list all scripts\n - `--show-archived` - Show archived scripts instead of active ones\n - `--json` - Output as JSON (for piping to jq)\n- `script push ` - push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh)\n - `--message ` - Deployment message\n- `script get ` - get a script's details\n - `--json` - Output as JSON (for piping to jq)\n- `script show ` - show a script's content (alias for get)\n- `script run ` - run a script by path\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not output anything other then the final output. Useful for scripting.\n- `script preview ` - preview a local script without deploying it. Supports both regular and codebase scripts.\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not output anything other than the final output. Useful for scripting.\n- `script new ` - create a new script\n - `--summary ` - script summary\n - `--description ` - script description\n- `script bootstrap ` - create a new script (alias for new)\n - `--summary ` - script summary\n - `--description ` - script description\n- `script set-permissioned-as ` - Set the on_behalf_of_email for a script (requires admin or wm_deployers group)\n- `script history ` - show version history for a script\n - `--json` - Output as JSON (for piping to jq)\n\n### sync\n\nsync local with a remote workspaces or the opposite (push or pull)\n\n**Subcommands:**\n\n- `sync pull` - Pull any remote changes and apply them locally.\n - `--yes` - Pull without needing confirmation\n - `--dry-run` - Show changes that would be pulled without actually pushing\n - `--plain-secrets` - Pull secrets as plain text\n - `--json` - Use JSON instead of YAML\n - `--skip-variables` - Skip syncing variables (including secrets)\n - `--skip-secrets` - Skip syncing only secrets variables\n - `--include-secrets` - Include secrets in sync (overrides skipSecrets in wmill.yaml)\n - `--skip-resources` - Skip syncing resources\n - `--skip-resource-types` - Skip syncing resource types\n - `--skip-scripts` - Skip syncing scripts\n - `--skip-flows` - Skip syncing flows\n - `--skip-apps` - Skip syncing apps\n - `--skip-folders` - Skip syncing folders\n - `--skip-workspace-dependencies` - Skip syncing workspace dependencies\n - `--skip-scripts-metadata` - Skip syncing scripts metadata, focus solely on logic\n - `--include-schedules` - Include syncing schedules\n - `--include-triggers` - Include syncing triggers\n - `--include-users` - Include syncing users\n - `--include-groups` - Include syncing groups\n - `--include-settings` - Include syncing workspace settings\n - `--include-key` - Include workspace encryption key\n - `--skip-branch-validation` - Skip git branch validation and prompts\n - `--json-output` - Output results in JSON format\n - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes\n - `-e --excludes ` - Comma separated patterns to specify which file to NOT take into account. Overrides wmill.yaml excludes\n - `--extra-includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy\n - `--repository ` - Specify repository path (e.g., u/user/repo) when multiple repositories exist\n - `--promotion ` - Use promotionOverrides from the specified branch instead of regular overrides\n - `--branch, --env ` - [Deprecated: use --workspace] Override the current git branch/environment\n- `sync push` - Push any local changes and apply them remotely.\n - `--yes` - Push without needing confirmation\n - `--dry-run` - Show changes that would be pushed without actually pushing\n - `--plain-secrets` - Push secrets as plain text\n - `--json` - Use JSON instead of YAML\n - `--skip-variables` - Skip syncing variables (including secrets)\n - `--skip-secrets` - Skip syncing only secrets variables\n - `--include-secrets` - Include secrets in sync (overrides skipSecrets in wmill.yaml)\n - `--skip-resources` - Skip syncing resources\n - `--skip-resource-types` - Skip syncing resource types\n - `--skip-scripts` - Skip syncing scripts\n - `--skip-flows` - Skip syncing flows\n - `--skip-apps` - Skip syncing apps\n - `--skip-folders` - Skip syncing folders\n - `--skip-workspace-dependencies` - Skip syncing workspace dependencies\n - `--skip-scripts-metadata` - Skip syncing scripts metadata, focus solely on logic\n - `--include-schedules` - Include syncing schedules\n - `--include-triggers` - Include syncing triggers\n - `--include-users` - Include syncing users\n - `--include-groups` - Include syncing groups\n - `--include-settings` - Include syncing workspace settings\n - `--include-key` - Include workspace encryption key\n - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.\n - `--skip-branch-validation` - Skip git branch validation and prompts\n - `--json-output` - Output results in JSON format\n - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)\n - `-e --excludes ` - Comma separated patterns to specify which file to NOT take into account.\n - `--extra-includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy\n - `--message ` - Include a message that will be added to all scripts/flows/apps updated during this push\n - `--parallel ` - Number of changes to process in parallel\n - `--repository ` - Specify repository path (e.g., u/user/repo) when multiple repositories exist\n - `--branch, --env ` - [Deprecated: use --workspace] Override the current git branch/environment\n - `--lint` - Run lint validation before pushing\n - `--locks-required` - Fail if scripts or flow inline scripts that need locks have no locks\n - `--auto-metadata` - Automatically regenerate stale metadata (locks and schemas) before pushing\n - `--accept-overriding-permissioned-as-with-self` - Accept that items with a different permissioned_as will be updated with your own user\n\n### token\n\nManage API tokens\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `token list` - List API tokens\n - `--json` - Output as JSON (for piping to jq)\n- `token create` - Create a new API token\n - `--label ` - Token label\n - `--expiration ` - Token expiration (ISO 8601 timestamp)\n- `token delete ` - Delete a token by its prefix\n\n### trigger\n\ntrigger related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `trigger list` - list all triggers\n - `--json` - Output as JSON (for piping to jq)\n- `trigger get ` - get a trigger's details\n - `--json` - Output as JSON (for piping to jq)\n - `--kind ` - Trigger kind (http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email). Recommended for faster lookup\n- `trigger new ` - create a new trigger locally\n - `--kind ` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email)\n- `trigger push ` - push a local trigger spec. This overrides any remote versions.\n- `trigger set-permissioned-as ` - Set the email (run-as user) for a trigger (requires admin or wm_deployers group)\n - `--kind ` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email)\n\n### user\n\nuser related commands\n\n**Subcommands:**\n\n- `user add [password:string]` - Create a user\n - `--superadmin` - Specify to make the new user superadmin.\n - `--company ` - Specify to set the company of the new user.\n - `--name ` - Specify to set the name of the new user.\n- `user remove ` - Delete a user\n- `user create-token` - Create a new API token for the authenticated user\n - `--email ` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.\n - `--password ` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.\n\n### variable\n\nvariable related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `variable list` - list all variables\n - `--json` - Output as JSON (for piping to jq)\n- `variable get ` - get a variable's details\n - `--json` - Output as JSON (for piping to jq)\n- `variable new ` - create a new variable locally\n- `variable push ` - Push a local variable spec. This overrides any remote versions.\n - `--plain-secrets` - Push secrets as plain text\n- `variable add ` - Create a new variable on the remote. This will update the variable if it already exists.\n - `--yes` - Skip confirmation prompt when updating an existing variable\n - `--secret` - Mark the variable as secret (default when creating a new variable)\n - `--no-secret` - Mark the variable as non-secret (when updating, the existing setting is preserved if neither --secret nor --no-secret is passed)\n - `--description ` - Set the variable description (when updating, the existing description is preserved if not passed)\n - `--plain-secrets` - Push secrets as plain text\n - `--public` - Legacy option, use --no-secret instead\n\n### version\n\nShow version information\n\n### worker-groups\n\ndisplay worker groups, pull and push worker groups configs\n\n**Subcommands:**\n\n- `worker-groups pull` - Pull worker groups (similar to `wmill instance pull --skip-users --skip-settings --skip-groups`)\n - `--instance` - Name of the instance to push to, override the active instance\n - `--base-url` - Base url to be passed to the instance settings instead of the local one\n - `--yes` - Pull without needing confirmation\n- `worker-groups push` - Push worker groups (similar to `wmill instance push --skip-users --skip-settings --skip-groups`)\n - `--instance [instance]` - Name of the instance to push to, override the active instance\n - `--base-url [baseUrl]` - If used with --token, will be used as the base url for the instance\n - `--yes` - Push without needing confirmation\n\n### workers\n\nList all workers grouped by worker groups\n\n**Options:**\n- `--instance [instance]` - Name of the instance to push to, override the active instance\n- `--base-url [baseUrl]` - If used with --token, will be used as the base url for the instance\n\n### workspace\n\nworkspace related commands\n\n**Alias:** `profile`\n\n**Subcommands:**\n\n- `workspace switch ` - Switch to another workspace\n- `workspace add [workspace_name:string] [workspace_id:string] [remote:string]` - Add a workspace\n - `-c --create` - Create the workspace if it does not exist\n - `--create-workspace-name ` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.\n - `--create-username ` - Specify your own username in the newly created workspace. Ignored if --create is not specified, the workspace already exists or automatic username creation is enabled on the instance.\n- `workspace remove ` - Remove a workspace\n- `workspace whoami` - Show the currently active user\n- `workspace list` - List local workspace profiles\n- `workspace list-remote` - List workspaces on the remote server that you have access to\n - `--as-superadmin` - List ALL workspaces on the instance (requires the token to belong to a superadmin/devops user)\n- `workspace list-forks` - List forked workspaces on the remote server\n- `workspace bind` - Create or update a workspace entry in wmill.yaml from the active profile\n - `--workspace ` - Workspace name (default: current branch or workspaceId)\n - `--branch ` - Git branch to associate (default: workspace name)\n- `workspace unbind` - Remove baseUrl and workspaceId from a workspace entry\n - `--workspace ` - Workspace to unbind\n- `workspace fork [workspace_name:string] [workspace_id:string]` - Create a forked workspace\n - `--create-workspace-name ` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.\n - `--color ` - Workspace color (hex code, e.g. #ff0000)\n - `--datatable-behavior ` - How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt)\n - `--from-branch ` - Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on (its bound workspace is the parent) and rename the current branch onto wm-fork//. Usually unneeded \u2014 from a working branch `wmill workspace fork` offers this interactively; from a base branch it creates a fresh fork branch.\n - `-y --yes` - Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for.\n- `workspace delete-fork ` - Delete a forked workspace\n - `-y --yes` - Skip confirmation prompt\n- `workspace merge` - Compare and deploy changes between a fork and its parent workspace\n - `--direction ` - Deploy direction: to-parent or to-fork\n - `--all` - Deploy all changed items including conflicts\n - `--skip-conflicts` - Skip items modified in both workspaces\n - `--include ` - Comma-separated kind:path items to include (e.g. script:f/test/main,flow:f/my/flow)\n - `--exclude ` - Comma-separated kind:path items to exclude\n - `--preserve-on-behalf-of` - Preserve original on_behalf_of/permissioned_as values\n - `-y --yes` - Non-interactive mode (deploy without prompts)\n- `workspace connect-slack` - Non-interactively connect Slack to the active workspace using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: workspace_settings fields, g/slack group, f/slack_bot folder, and the encrypted bot token variable + resource at f/slack_bot/bot_token.\n - `--bot-token ` - Slack bot token (xoxb-...)\n - `--team-id ` - Slack team id\n - `--team-name ` - Slack team name\n- `workspace disconnect-slack` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group \u2014 delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override \u2014 set slack_oauth_client_id/_secret to '' in settings.yaml and push.\n\n\n\n# Object Storage CLI\n\n`wmill object-storage` (alias `wmill s3`) exposes the workspace's object storage (S3-compatible: AWS S3, MinIO, GCS, R2, Azure Blob) over the per-workspace `/job_helpers/*` endpoints.\n\n## Key concepts (not obvious from per-command --help)\n\n- **`file_key` is the path inside the bucket** (e.g. `reports/2026-05/orders.csv`), not a Windmill path. Do NOT pass `u/...` or `f/...` here \u2014 those are Windmill paths to scripts/flows/resources, unrelated to objects in the bucket.\n- **Scope is the active workspace.** Object storage is configured per-workspace (default storage + optional secondary storages). Switching workspaces switches which bucket the commands target.\n- **`--storage ` targets a secondary storage** configured on the workspace. Omit it to use the workspace's default object storage. Use `wmill object-storage list` to discover configured storages.\n- **`preview` vs `download`**: `preview` returns a peek (CSV first rows, text content, or a byte slice via `--bytes-from`/`--bytes-length`) without writing to disk. Use `download` when you want the full file on disk.\n\n## Choosing a subcommand\n\n- Look at what's there: `wmill object-storage files [prefix]` (alias `ls`) \u2014 paginated, use `--marker` to continue.\n- Inspect one file: `wmill object-storage info ` for size/mime/last-modified, `wmill object-storage preview ` for content peek.\n- Move data in: `wmill object-storage upload ` \u2014 set `--content-type` if the receiver cares (e.g. `text/csv`).\n- Move data out: `wmill object-storage download [output_path]` \u2014 `--stdout` to pipe.\n- Reorganize: `wmill object-storage move ` (same storage), `wmill object-storage delete ` (interactive confirm unless `--yes`).\n"; +export declare const LANG_ANSIBLE = "# Ansible\n\nWindmill runs Ansible playbooks with `ansible-playbook`. A script is a single YAML\ndocument made of two parts separated by a `---` line: a Windmill **header** and one or\nmore standard Ansible **plays**.\n\n## Structure\n\n```yaml\n---\n# Windmill header: configures inventories, file resources, arguments and dependencies\nextra_vars:\n world_qualifier:\n type: string\ndependencies:\n galaxy:\n collections:\n - name: community.general\n python:\n - jmespath\n---\n# Standard Ansible plays\n- name: Echo\n hosts: 127.0.0.1\n connection: local\n tasks:\n - name: Print debug message\n debug:\n msg: \"Hello, {{ world_qualifier }} world!\"\n```\n\n## Header\n\nThe header is **not** standard Ansible \u2014 it is parsed by Windmill to build the script's\ninputs and runtime environment. Supported keys:\n\n- `extra_vars`: defines the script arguments. Each entry is passed to the playbook via\n `--extra-vars` and becomes a Jinja variable usable as `{{ name }}` in the plays. Give\n each argument a `type` (`string`, `number`, `boolean`, `object`, ...) so Windmill can\n generate the input form.\n- `inventory`: lists inventories. Use `resource_type: ansible_inventory` (optionally\n pinned with `resource: u/user/your_resource`) or `resource_type: dynamic_inventory`.\n- `files`: writes Windmill resources/variables to files before the run, e.g.\n `- resource: u/user/template` with `target: ./config.j2`, or\n `- variable: u/user/ssh_key` with `target: ./ssh_key` and `mode: '0600'`.\n- `dependencies`: `galaxy` collections/roles (installed with `ansible-galaxy`) and\n `python` pip packages available to the playbook.\n- `options`: extra `ansible-playbook` flags such as `- verbosity: vvv`.\n- `vault_password`: a Windmill variable path to use as the Ansible Vault password.\n\n## Arguments\n\nReference header `extra_vars` directly as Jinja variables in the plays:\n\n```yaml\nextra_vars:\n name:\n type: string\n count:\n type: number\n---\n- hosts: localhost\n tasks:\n - debug:\n msg: \"{{ name }} x {{ count }}\"\n```\n\n## Environment variables\n\nWindmill contextual variables are available as environment variables and read with the\n`env` lookup:\n\n```yaml\n- debug:\n msg: \"Running in workspace {{ lookup('env', 'WM_WORKSPACE') }}\"\n```\n\n## Output\n\nTo return a result, write JSON to a `result.json` file in the job directory:\n\n```yaml\n- hosts: localhost\n tasks:\n - name: Write result\n copy:\n content: \"{{ { 'ok': true, 'value': 42 } | to_json }}\"\n dest: result.json\n```\n"; +export declare const LANG_BASH = "# Bash\n\n## Structure\n\nDo not include `#!/bin/bash`. Arguments are obtained as positional parameters:\n\n```bash\n# Get arguments\nvar1=\"$1\"\nvar2=\"$2\"\n\necho \"Processing $var1 and $var2\"\n\n# Return JSON by echoing to stdout\necho \"{\\\"result\\\": \\\"$var1\\\", \\\"count\\\": $var2}\"\n```\n\n**Important:**\n- Do not include shebang (`#!/bin/bash`)\n- Arguments are always strings\n- Access with `$1`, `$2`, etc.\n\n## Output\n\nThe script output is captured as the result. For structured data, output valid JSON:\n\n```bash\nname=\"$1\"\ncount=\"$2\"\n\n# Output JSON result\ncat << EOF\n{\n \"name\": \"$name\",\n \"count\": $count,\n \"timestamp\": \"$(date -Iseconds)\"\n}\nEOF\n```\n\n## Environment Variables\n\nEnvironment variables set in Windmill are available:\n\n```bash\n# Access environment variable\necho \"Workspace: $WM_WORKSPACE\"\necho \"Job ID: $WM_JOB_ID\"\n```\n"; +export declare const LANG_BIGQUERY = "# BigQuery\n\nArguments use `@name` syntax.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- @name1 (string)\n-- @name2 (int64) = 0\nSELECT * FROM users WHERE name = @name1 AND age > @name2;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as a `STRING` JSON parameter \u2014 Parquet/CSV\nfiles are decoded server-side into a JSON array of records, JSON/JSONL pass\nthrough. Consume with `JSON_EXTRACT_ARRAY` / `JSON_VALUE`:\n\n```sql\n-- @file (s3object)\nSELECT\n CAST(JSON_VALUE(row, '$.id') AS INT64) AS id,\n JSON_VALUE(row, '$.name') AS name\nFROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row;\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered, bypassing the 10000-row return cap.\n"; +export declare const LANG_BUN = "# TypeScript (Bun)\n\nBun runtime with full npm ecosystem and fastest execution.\n\n## Structure\n\nExport a single **async** function called `main`:\n\n```typescript\nexport async function main(param1: string, param2: number) {\n // Your code here\n return { result: param1, count: param2 };\n}\n```\n\nDo not call the main function. Libraries are installed automatically.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nUse the `RT` namespace for resource types:\n\n```typescript\nexport async function main(stripe: RT.Stripe) {\n // stripe contains API key and config from the resource\n}\n```\n\nOnly use resource types if you need them to satisfy the instructions. Always use the RT namespace.\n\nBefore using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`.\n\n## Imports\n\n```typescript\nimport Stripe from \"stripe\";\nimport { someFunction } from \"some-package\";\n```\n\n## Prefer `//native` when the runtime allows it\n\nIf a script only needs `fetch` and the JavaScript standard library \u2014 including when it uses `windmill-client` \u2014 prefer making it a **native** script: add `//native` as the first line and write it with the `write-script-bunnative` skill. Native scripts run on a lightweight V8 isolate, start faster, and parallelize heavily. `windmill-client` works on the native worker (its calls go over `fetch`), so needing the Windmill client is **not** a reason to avoid `//native`. Use the regular `bun` language only when the code (or a dependency) needs Node/Bun runtime APIs \u2014 `node:*` modules, the filesystem, child processes, or native addons.\n\n## Windmill Client\n\nImport the windmill client for platform interactions:\n\n```typescript\nimport * as wmill from \"windmill-client\";\n```\n\n**Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** \u2014 reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill.\n\nThe full `windmill-client` API reference (every exported function and its signature) is included in this skill below \u2014 consult it for the exact method to use instead of guessing or falling back to `fetch`.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```typescript\ntype Event = {\n kind:\n | \"webhook\"\n | \"http\"\n | \"websocket\"\n | \"kafka\"\n | \"email\"\n | \"nats\"\n | \"postgres\"\n | \"sqs\"\n | \"mqtt\"\n | \"gcp\";\n body: any;\n headers: Record;\n query: Record;\n};\n\nexport async function preprocessor(event: Event) {\n return {\n param1: event.body.field1,\n param2: event.query.id,\n };\n}\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form \u2014 always use it instead of redefining your own.\n\n### Receiving an S3Object as a script parameter\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\nexport async function main(file: wmill.S3Object) {\n const content = await wmill.loadS3File(file);\n // ...\n}\n```\n\n### S3 operations\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\n// Load file content from S3\nconst content: Uint8Array = await wmill.loadS3File(s3object);\n\n// Load file as stream\nconst blob: Blob = await wmill.loadS3FileStream(s3object);\n\n// Write file to S3\nconst result: wmill.S3Object = await wmill.writeS3File(\n s3object, // Target path (or undefined to auto-generate)\n fileContent, // string or Blob\n s3ResourcePath // Optional: specific S3 resource to use\n);\n```\n"; +export declare const LANG_BUNNATIVE = "# TypeScript (Bun Native)\n\nNative TypeScript execution. Native scripts are Bun scripts that run on the native worker \u2014 a lightweight V8 isolate that exposes `fetch` and the JavaScript standard library \u2014 and can be heavily parallelized. Every script MUST start with `//native` on its first line so Windmill routes it to the native worker; without it the exact same script runs on the regular Bun worker. You may import npm packages and other Windmill scripts (e.g. `./helper.ts`) \u2014 imports are resolved and bundled just like a regular Bun script \u2014 as long as everything (your code and its dependencies) relies only on `fetch` and the standard library. Libraries that need Node/Bun runtime APIs (filesystem, `node:*` modules, child processes, native addons) will not work on the native worker; use the regular `bun` language for those.\n\n## Structure\n\nExport a single **async** function called `main`:\n\n```typescript\n//native\nexport async function main(param1: string, param2: number) {\n // Your code here\n return { result: param1, count: param2 };\n}\n```\n\nDo not call the main function.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nUse the `RT` namespace for resource types:\n\n```typescript\n//native\nexport async function main(stripe: RT.Stripe) {\n // stripe contains API key and config from the resource\n}\n```\n\nOnly use resource types if you need them to satisfy the instructions. Always use the RT namespace.\n\nBefore using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`.\n\n## Imports\n\n**The constraint is the runtime, not the import list.** You may import npm packages and relative Windmill scripts; they are resolved and bundled exactly like a regular Bun script. But the native worker only provides `fetch` and the JavaScript standard library, so any imported code must work using only those. Anything requiring Node/Bun built-ins (`node:fs`, `child_process`, the `Bun` API, native modules) belongs in a regular `bun` script instead. Use the globally available `fetch` for HTTP:\n\n```typescript\n//native\nexport async function main(url: string) {\n const response = await fetch(url);\n return await response.json();\n}\n```\n\n## Windmill Client\n\n`windmill-client` works on the native worker (its calls go over `fetch`), so use it as the **preferred way to talk to Windmill** \u2014 reading resources/variables/states, running scripts and flows, and the S3 helpers below (`loadS3File`, `loadS3FileStream`, `writeS3File`, `S3Object`). It handles auth, the workspace, and the base URL for you. Reserve raw `fetch` for calling *external* HTTP APIs that aren't Windmill.\n\nThe full `windmill-client` API reference (every exported function and its signature) is included in this skill below \u2014 consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```typescript\n//native\ntype Event = {\n kind:\n | \"webhook\"\n | \"http\"\n | \"websocket\"\n | \"kafka\"\n | \"email\"\n | \"nats\"\n | \"postgres\"\n | \"sqs\"\n | \"mqtt\"\n | \"gcp\";\n body: any;\n headers: Record;\n query: Record;\n};\n\nexport async function preprocessor(event: Event) {\n return {\n param1: event.body.field1,\n param2: event.query.id,\n };\n}\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form \u2014 always use it instead of redefining your own.\n\n### Receiving an S3Object as a script parameter\n\n```typescript\n//native\nimport * as wmill from \"windmill-client\";\n\nexport async function main(file: wmill.S3Object) {\n const content = await wmill.loadS3File(file);\n // ...\n}\n```\n\n### S3 operations\n\n```typescript\n//native\nimport * as wmill from \"windmill-client\";\n\n// Load file content from S3\nconst content: Uint8Array = await wmill.loadS3File(s3object);\n\n// Load file as stream\nconst blob: Blob = await wmill.loadS3FileStream(s3object);\n\n// Write file to S3\nconst result: wmill.S3Object = await wmill.writeS3File(\n s3object, // Target path (or undefined to auto-generate)\n fileContent, // string or Blob\n s3ResourcePath // Optional: specific S3 resource to use\n);\n```\n"; +export declare const LANG_CSHARP = "# C#\n\nThe script must contain a public static `Main` method inside a class:\n\n```csharp\npublic class Script\n{\n public static object Main(string name, int count)\n {\n return new { Name = name, Count = count };\n }\n}\n```\n\n**Important:**\n- Class name is irrelevant\n- Method must be `public static`\n- Return type can be `object` or specific type\n\n## NuGet Packages\n\nAdd packages using the `#r` directive at the top:\n\n```csharp\n#r \"nuget: Newtonsoft.Json, 13.0.3\"\n#r \"nuget: RestSharp, 110.2.0\"\n\nusing Newtonsoft.Json;\nusing RestSharp;\n\npublic class Script\n{\n public static object Main(string url)\n {\n var client = new RestClient(url);\n var request = new RestRequest();\n var response = client.Get(request);\n return JsonConvert.DeserializeObject(response.Content);\n }\n}\n```\n"; +export declare const LANG_DENO = "# TypeScript (Deno)\n\nDeno runtime with npm support via `npm:` prefix and native Deno libraries.\n\n## Structure\n\nExport a single **async** function called `main`:\n\n```typescript\nexport async function main(param1: string, param2: number) {\n // Your code here\n return { result: param1, count: param2 };\n}\n```\n\nDo not call the main function. Libraries are installed automatically.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nUse the `RT` namespace for resource types:\n\n```typescript\nexport async function main(stripe: RT.Stripe) {\n // stripe contains API key and config from the resource\n}\n```\n\nOnly use resource types if you need them to satisfy the instructions. Always use the RT namespace.\n\nBefore using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`.\n\n## Imports\n\n```typescript\n// npm packages use npm: prefix\nimport Stripe from \"npm:stripe\";\nimport { someFunction } from \"npm:some-package\";\n\n// Deno standard library\nimport { serve } from \"https://deno.land/std/http/server.ts\";\n```\n\n## Windmill Client\n\nImport the windmill client for platform interactions:\n\n```typescript\nimport * as wmill from \"windmill-client\";\n```\n\n**Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** \u2014 reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill.\n\nThe full `windmill-client` API reference (every exported function and its signature) is included in this skill below \u2014 consult it for the exact method instead of guessing or falling back to `fetch`.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```typescript\ntype Event = {\n kind:\n | \"webhook\"\n | \"http\"\n | \"websocket\"\n | \"kafka\"\n | \"email\"\n | \"nats\"\n | \"postgres\"\n | \"sqs\"\n | \"mqtt\"\n | \"gcp\";\n body: any;\n headers: Record;\n query: Record;\n};\n\nexport async function preprocessor(event: Event) {\n return {\n param1: event.body.field1,\n param2: event.query.id,\n };\n}\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form \u2014 always use it instead of redefining your own.\n\n### Receiving an S3Object as a script parameter\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\nexport async function main(file: wmill.S3Object) {\n const content = await wmill.loadS3File(file);\n // ...\n}\n```\n\n### S3 operations\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\n// Load file content from S3\nconst content: Uint8Array = await wmill.loadS3File(s3object);\n\n// Load file as stream\nconst blob: Blob = await wmill.loadS3FileStream(s3object);\n\n// Write file to S3\nconst result: wmill.S3Object = await wmill.writeS3File(\n s3object, // Target path (or undefined to auto-generate)\n fileContent, // string or Blob\n s3ResourcePath // Optional: specific S3 resource to use\n);\n```\n"; +export declare const LANG_DUCKDB = "# DuckDB\n\nArguments are defined with comments and used with `$name` syntax:\n\n```sql\n-- $name (text) = default\n-- $age (integer)\nSELECT * FROM users WHERE name = $name AND age > $age;\n```\n\n## Ducklake Integration\n\nAttach Ducklake for data lake operations:\n\n```sql\n-- Main ducklake\nATTACH 'ducklake' AS dl;\n\n-- Named ducklake\nATTACH 'ducklake://my_lake' AS dl;\n\n-- Then query\nSELECT * FROM dl.schema.table;\n```\n\n## External Database Connections\n\nConnect to external databases using resources:\n\n```sql\nATTACH '$res:path/to/resource' AS db (TYPE postgres);\nSELECT * FROM db.schema.table;\n```\n\n## S3 File Operations\n\nRead files from S3 storage:\n\n```sql\n-- Default storage\nSELECT * FROM read_csv('s3:///path/to/file.csv');\n\n-- Named storage\nSELECT * FROM read_csv('s3://storage_name/path/to/file.csv');\n\n-- Parquet files\nSELECT * FROM read_parquet('s3:///path/to/file.parquet');\n\n-- JSON files\nSELECT * FROM read_json('s3:///path/to/file.json');\n```\n\n### Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for it\nand binds the arg as the bare `s3://storage/key` URI, which DuckDB's reader\nfunctions consume directly:\n\n```sql\n-- $file (s3object)\nSELECT * FROM read_parquet($file);\n```\n\nWorks with any DuckDB reader: `read_csv($file)`, `read_json($file)`, etc.\n\n### Writing query results to S3\n\nDuckDB writes to S3 natively via `COPY ... TO`:\n\n```sql\nCOPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET);\n```\n\nUse this instead of the `-- s3` streaming directive supported by the other SQL\ndialects \u2014 that directive is not available in DuckDB.\n"; +export declare const LANG_GO = "# Go\n\n## Structure\n\nThe file package must be `inner` and export a function called `main`:\n\n```go\npackage inner\n\nfunc main(param1 string, param2 int) (map[string]interface{}, error) {\n return map[string]interface{}{\n \"result\": param1,\n \"count\": param2,\n }, nil\n}\n```\n\n**Important:**\n- Package must be `inner`\n- Return type must be `({return_type}, error)`\n- Function name is `main` (lowercase)\n\n## Return Types\n\nThe return type can be any Go type that can be serialized to JSON:\n\n```go\npackage inner\n\ntype Result struct {\n Name string `json:\"name\"`\n Count int `json:\"count\"`\n}\n\nfunc main(name string, count int) (Result, error) {\n return Result{\n Name: name,\n Count: count,\n }, nil\n}\n```\n\n## Error Handling\n\nReturn errors as the second return value:\n\n```go\npackage inner\n\nimport \"errors\"\n\nfunc main(value int) (string, error) {\n if value < 0 {\n return \"\", errors.New(\"value must be positive\")\n }\n return \"success\", nil\n}\n```\n"; +export declare const LANG_GRAPHQL = "# GraphQL\n\n## Structure\n\nWrite GraphQL queries or mutations. Arguments can be added as query parameters:\n\n```graphql\nquery GetUser($id: ID!) {\n user(id: $id) {\n id\n name\n email\n }\n}\n```\n\n## Variables\n\nVariables are passed as script arguments and automatically bound to the query:\n\n```graphql\nquery SearchProducts($query: String!, $limit: Int = 10) {\n products(search: $query, first: $limit) {\n edges {\n node {\n id\n name\n price\n }\n }\n }\n}\n```\n\n## Mutations\n\n```graphql\nmutation CreateUser($input: CreateUserInput!) {\n createUser(input: $input) {\n id\n name\n createdAt\n }\n}\n```\n"; +export declare const LANG_JAVA = "# Java\n\nThe script must contain a Main public class with a `public static main()` method:\n\n```java\npublic class Main {\n public static Object main(String name, int count) {\n java.util.Map result = new java.util.HashMap<>();\n result.put(\"name\", name);\n result.put(\"count\", count);\n return result;\n }\n}\n```\n\n**Important:**\n- Class must be named `Main`\n- Method must be `public static Object main(...)`\n- Return type is `Object` or `void`\n\n## Maven Dependencies\n\nAdd dependencies using comments at the top:\n\n```java\n//requirements:\n//com.google.code.gson:gson:2.10.1\n//org.apache.httpcomponents:httpclient:4.5.14\n\nimport com.google.gson.Gson;\n\npublic class Main {\n public static Object main(String input) {\n Gson gson = new Gson();\n return gson.fromJson(input, Object.class);\n }\n}\n```\n"; +export declare const LANG_MSSQL = "# Microsoft SQL Server (MSSQL)\n\nArguments use `@P1`, `@P2`, etc.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- @P1 name1 (varchar)\n-- @P2 name2 (int) = 0\nSELECT * FROM users WHERE name = @P1 AND age > @P2;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as `nvarchar(max)` JSON text \u2014 Parquet/CSV\nfiles are decoded server-side into a JSON array of records, JSON/JSONL pass\nthrough. Consume with `OPENJSON`:\n\n```sql\n-- @P1 file (s3object)\nSELECT id, name\nFROM OPENJSON(@P1)\nWITH (id INT, name NVARCHAR(200));\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered as the script return value.\n"; +export declare const LANG_MYSQL = "# MySQL\n\nArguments use `?` placeholders.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- ? name1 (text)\n-- ? name2 (int) = 0\nSELECT * FROM users WHERE name = ? AND age > ?;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as JSON text \u2014 Parquet/CSV files are\ndecoded server-side into a JSON array of records, JSON/JSONL pass through.\nConsume with `JSON_TABLE`:\n\n```sql\n-- ? file (s3object)\nSELECT id, name\nFROM JSON_TABLE(?, '$[*]'\n COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name')\n) AS r;\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered as the script return value.\n"; +export declare const LANG_PHP = "# PHP\n\n## Structure\n\nThe script must start with ` $param1, \"count\" => $param2];\n}\n```\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nYou need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using `class_exists`:\n\n```php\n $2::INT;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as a `jsonb` parameter \u2014 Parquet/CSV files\nare decoded server-side into a JSON array of records, JSON/JSONL pass through.\nConsume with `jsonb_to_recordset` (or any `jsonb` API):\n\n```sql\n-- $1 file (s3object)\nSELECT *\nFROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT);\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered as the script return value.\n"; +export declare const LANG_POWERSHELL = "# PowerShell\n\n## Structure\n\nArguments are obtained by calling the `param` function on the first line:\n\n```powershell\nparam($Name, $Count = 0, [int]$Age)\n\n# Your code here\nWrite-Output \"Processing $Name, count: $Count, age: $Age\"\n\n# Return object\n@{\n name = $Name\n count = $Count\n age = $Age\n}\n```\n\n## Parameter Types\n\nYou can specify types for parameters:\n\n```powershell\nparam(\n [string]$Name,\n [int]$Count = 0,\n [bool]$Enabled = $true,\n [array]$Items\n)\n\n@{\n name = $Name\n count = $Count\n enabled = $Enabled\n items = $Items\n}\n```\n\n## Return Values\n\nReturn values by outputting them at the end of the script:\n\n```powershell\nparam($Input)\n\n$result = @{\n processed = $true\n data = $Input\n timestamp = Get-Date -Format \"o\"\n}\n\n$result\n```\n"; +export declare const LANG_PYTHON3 = "# Python\n\n## Structure\n\nThe script must contain at least one function called `main`:\n\n```python\ndef main(param1: str, param2: int):\n # Your code here\n return {\"result\": param1, \"count\": param2}\n```\n\nDo not call the main function. Libraries are installed automatically.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nYou need to **redefine** the type of the resources that are needed before the main function as TypedDict:\n\n```python\nfrom typing import TypedDict\n\nclass postgresql(TypedDict):\n host: str\n port: int\n user: str\n password: str\n dbname: str\n\ndef main(db: postgresql):\n # db contains the database connection details\n pass\n```\n\n**Important rules:**\n\n- The resource type name must be **IN LOWERCASE**\n- Only include resource types if they are actually needed\n- If an import conflicts with a resource type name, **rename the imported object, not the type name**\n- Make sure to import TypedDict from typing **if you're using it**\n\n## Imports\n\nLibraries are installed automatically. Do not show installation instructions.\n\n```python\nimport requests\nimport pandas as pd\nfrom datetime import datetime\n```\n\nIf an import name conflicts with a resource type:\n\n```python\n# Wrong - don't rename the type\nimport stripe as stripe_lib\nclass stripe_type(TypedDict): ...\n\n# Correct - rename the import\nimport stripe as stripe_sdk\nclass stripe(TypedDict):\n api_key: str\n```\n\n## Windmill Client\n\nImport the windmill client for platform interactions:\n\n```python\nimport wmill\n```\n\nSee the SDK documentation for available methods.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```python\nfrom typing import TypedDict, Literal, Any\n\nclass Event(TypedDict):\n kind: Literal[\"webhook\", \"http\", \"websocket\", \"kafka\", \"email\", \"nats\", \"postgres\", \"sqs\", \"mqtt\", \"gcp\"]\n body: Any\n headers: dict[str, str]\n query: dict[str, str]\n\ndef preprocessor(event: Event):\n # Transform the event into flow input parameters\n return {\n \"param1\": event[\"body\"][\"field1\"],\n \"param2\": event[\"query\"][\"id\"]\n }\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations.\n\n### Receiving an S3Object as a script parameter\n\nTo accept a file from S3 as input to a script, type the parameter with `S3Object` (imported from `wmill`):\n\n```python\nimport wmill\nfrom wmill import S3Object\n\ndef main(file: S3Object):\n content = wmill.load_s3_file(file)\n # ...\n```\n\n### S3 operations\n\n```python\nimport wmill\n\n# Load file content from S3\ncontent: bytes = wmill.load_s3_file(s3object)\n\n# Load file as stream reader\nreader: BufferedReader = wmill.load_s3_file_reader(s3object)\n\n# Write file to S3\nresult: S3Object = wmill.write_s3_file(\n s3object, # Target path (or None to auto-generate)\n file_content, # bytes or BufferedReader\n s3_resource_path, # Optional: specific S3 resource\n content_type, # Optional: MIME type\n content_disposition # Optional: Content-Disposition header\n)\n```\n"; +export declare const LANG_RLANG = "# R\n\n## Structure\n\nDefine a `main` function using `<-` or `=` assignment. Parameters become the script inputs:\n\n```r\nlibrary(dplyr)\nlibrary(jsonlite)\n\nmain <- function(x, name = \"default\", flag = TRUE) {\n df <- tibble(x = x, name = name)\n result <- df %>% mutate(greeting = paste(\"Hello\", name))\n return(toJSON(result, auto_unbox = TRUE))\n}\n```\n\n**Important:**\n- The `main` function is required\n- Use `library()` to load packages \u2014 they are resolved and installed automatically\n- `jsonlite` is always available (used internally for argument parsing)\n- Return values must be JSON-serializable\n\n## Parameters\n\nR types map to Windmill types:\n- `numeric` \u2192 float/int\n- `character` \u2192 string\n- `logical` \u2192 bool (use `TRUE`/`FALSE`)\n- `list` \u2192 object/dict\n- `NULL` \u2192 null\n\nDefault values are inferred from the function signature:\n\n```r\nmain <- function(\n name, # required string\n count = 10, # optional int, default 10\n verbose = FALSE # optional bool, default FALSE\n) {\n # ...\n}\n```\n\n## Resources and Variables\n\nUse the built-in Windmill helpers (no import needed):\n\n```r\nmain <- function() {\n # Get a variable\n api_key <- get_variable(\"f/my_folder/api_key\")\n\n # Get a resource (returns a list)\n db <- get_resource(\"f/my_folder/postgres_config\")\n host <- db$host\n port <- db$port\n\n return(list(host = host, port = port))\n}\n```\n\n## Output\n\nReturn any JSON-serializable value from `main`. The return value becomes the step result:\n\n```r\nmain <- function(x) {\n # Return a scalar\n return(x + 1)\n\n # Or a list (becomes JSON object)\n return(list(result = x + 1, status = \"ok\"))\n}\n```\n\n## Annotations\n\nControl execution behavior with comment annotations:\n\n```r\n#renv_verbose = true # Show verbose renv output during resolution\n#renv_install_verbose = true # Show verbose output during package installation\n#sandbox = true # Run in nsjail sandbox (requires nsjail)\n```\n"; +export declare const LANG_RUST = "# Rust\n\n## Structure\n\nThe script must contain a function called `main` with proper return type:\n\n```rust\nuse anyhow::anyhow;\nuse serde::Serialize;\n\n#[derive(Serialize, Debug)]\nstruct ReturnType {\n result: String,\n count: i32,\n}\n\nfn main(param1: String, param2: i32) -> anyhow::Result {\n Ok(ReturnType {\n result: param1,\n count: param2,\n })\n}\n```\n\n**Important:**\n- Arguments should be owned types\n- Return type must be serializable (`#[derive(Serialize)]`)\n- Return type is `anyhow::Result`\n\n## Dependencies\n\nPackages must be specified with a partial cargo.toml at the beginning of the script:\n\n```rust\n//! ```cargo\n//! [dependencies]\n//! anyhow = \"1.0.86\"\n//! reqwest = { version = \"0.11\", features = [\"json\"] }\n//! tokio = { version = \"1\", features = [\"full\"] }\n//! ```\n\nuse anyhow::anyhow;\n// ... rest of the code\n```\n\n**Note:** Serde is already included, no need to add it again.\n\n## Async Functions\n\nIf you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside:\n\n```rust\n//! ```cargo\n//! [dependencies]\n//! anyhow = \"1.0.86\"\n//! tokio = { version = \"1\", features = [\"full\"] }\n//! reqwest = { version = \"0.11\", features = [\"json\"] }\n//! ```\n\nuse anyhow::anyhow;\nuse serde::Serialize;\n\n#[derive(Serialize, Debug)]\nstruct Response {\n data: String,\n}\n\nfn main(url: String) -> anyhow::Result {\n let rt = tokio::runtime::Runtime::new()?;\n rt.block_on(async {\n let resp = reqwest::get(&url).await?.text().await?;\n Ok(Response { data: resp })\n })\n}\n```\n"; +export declare const LANG_SNOWFLAKE = "# Snowflake\n\nArguments use `?` placeholders.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- ? name1 (text)\n-- ? name2 (number) = 0\nSELECT * FROM users WHERE name = ? AND age > ?;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as JSON text \u2014 Parquet/CSV files are\ndecoded server-side into a JSON array of records, JSON/JSONL pass through.\nWrap the bind with `PARSE_JSON(?)` and walk it with `LATERAL FLATTEN`:\n\n```sql\n-- ? file (s3object)\nSELECT\n v.value:id::NUMBER AS id,\n v.value:name::STRING AS name\nFROM LATERAL FLATTEN(input => PARSE_JSON(?)) v;\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered, bypassing the 10000-row return cap.\n"; From 7efeae26d821b10667b6e3edd220468f6ae48936 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 7 Jul 2026 18:21:32 +0200 Subject: [PATCH 247/273] feat: add fork_parent_workspace claim to OIDC tokens for fork workspaces (#9987) * feat: add parent_workspace claim to OIDC job tokens for fork workspaces Co-Authored-By: Claude Fable 5 * refactor: rename claim to fork_parent_workspace for clarity Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to e2df172596e00877068d4b0a98afaef62fe429d1 This commit updates the EE repository reference after PR #651 was merged in windmill-ee-private. Previous ee-repo-ref: f73001ac6c038694cfc2604233a59be1c0daa40b New ee-repo-ref: e2df172596e00877068d4b0a98afaef62fe429d1 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-common/src/oidc_oss.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1b702a3a54..fdd2ea689e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -27672e37df5d9dfde94f19963d5ffcdf8dd5448c +e2df172596e00877068d4b0a98afaef62fe429d1 diff --git a/backend/windmill-common/src/oidc_oss.rs b/backend/windmill-common/src/oidc_oss.rs index e212558377..60a09ce9b7 100644 --- a/backend/windmill-common/src/oidc_oss.rs +++ b/backend/windmill-common/src/oidc_oss.rs @@ -61,6 +61,8 @@ pub struct JobClaim { pub email: String, pub workspace: String, #[serde(skip_serializing_if = "Option::is_none")] + pub fork_parent_workspace: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub end_user_email: Option, } From edfe7b415af6670c5855b7a0b52db4c1f7781964 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 7 Jul 2026 18:22:53 +0200 Subject: [PATCH 248/273] fix(cli): auto-derive cascade triggers in --local pipeline graph (#9978) Co-authored-by: Claude Fable 5 --- cli/src/commands/pipeline/localGraph.ts | 109 +++++++++++++++++++ cli/test/pipeline_local_graph_unit.test.ts | 119 ++++++++++++++++++++- 2 files changed, 227 insertions(+), 1 deletion(-) diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 3c531c4aa9..eb3350c818 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -341,6 +341,72 @@ function normalizeRetry(retry: ParseAssetsRaw["retry"]): ParseAssetsRaw["retry"] return { ...retry, delay: retry.delay.replace(/^delay=/, "") }; } +// Read-asset kinds whose read auto-derives a cascade trigger edge inside a +// `// pipeline`. Mirror of backend `is_auto_trigger_kind` (windmill-common +// assets.rs) / frontend `AUTO_TRIGGER_KINDS` (resolveGraph.ts) — ducklake +// tables and s3 objects only; resource/datatable/volume stay explicit-`// on`. +const AUTO_TRIGGER_KINDS = new Set(["ducklake", "s3object"]); + +// Asset-URI prefixes accepted by `// mute `, in lockstep with the +// canonical `parse_asset_syntax` / frontend `ASSET_PREFIXES`. Non-derivable +// kinds are parsed too (their muted key simply never matches a derived edge), +// so a mute line is never REinterpreted differently from the deploy path. +const MUTE_ASSET_PREFIXES: [string, string][] = [ + ["s3://", "s3object"], + ["res://", "resource"], + ["$res:", "resource"], + ["ducklake://", "ducklake"], + ["datatable://", "datatable"], + ["volume://", "volume"], +]; + +// `// mute ` / `// mute all` from the LEADING comment header, as +// `:` keys. Parsed locally because the pinned wasm asset parser +// predates the mute annotations; mirrors the canonical parsers (Rust +// `parse_pipeline_annotations`, frontend parsePipelineAnnotations.ts): any of +// the three comment prefixes is accepted regardless of language, blank lines +// are skipped, scanning stops at the first non-comment line, and `mute` must +// be a complete word (`// muted for now` never matches). +export function parseMuteAnnotations(content: string): { + muteAll: boolean; + muted: Set; +} { + const muted = new Set(); + let muteAll = false; + for (const rawLine of content.split("\n")) { + const line = rawLine.trimStart(); + if (line === "") continue; + let rest: string; + if (line.startsWith("//")) rest = line.slice(2); + else if (line.startsWith("--")) rest = line.slice(2); + else if (line.startsWith("#")) rest = line.slice(1); + else break; + rest = rest.trimStart(); + if (!rest.startsWith("mute")) continue; + const after = rest.slice("mute".length); + if (after !== "" && !/^\s/.test(after)) continue; + const arg = after.trim(); + if (arg === "all") { + muteAll = true; + continue; + } + for (const [prefix, kind] of MUTE_ASSET_PREFIXES) { + if (arg.startsWith(prefix)) { + // S3 canonicalization as in `parse_asset_syntax`: strip every leading + // slash so `s3:///key` (default storage) mutes the same node as the + // inferred bare `key`. + const p = + kind === "s3object" + ? arg.slice(prefix.length).replace(/^\/+/, "") + : arg.slice(prefix.length); + muted.add(`${kind}:${p}`); + break; + } + } + } + return { muteAll, muted }; +} + // Comment prefix for `volume:` annotations. Deliberately NOT `commentPrefix` // above (which returns `--` for SQL): volume annotations are only recognized for // the languages the backend/frontend recognize them for — mirrors @@ -685,6 +751,49 @@ export async function buildLocalPipelineGraph(args: { runnable_path: s.path, }); } + // Auto-derived cascade edges (backend `derive_pipeline_asset_trigger_refs`, + // frontend `deriveAutoAssetTriggers`): a pipeline script's read-only + // ducklake/s3 input wires its cascade trigger straight from the body read, + // so `// on ` is only needed for edges inference can't see. Skipped: + // assets this script also writes — including the `// materialize` target and + // its scd2 `_current` companion, whose writes the body SELECT doesn't + // express — plus `// mute ` opt-outs and explicit `// on` (which wins + // the dedup); `// mute all` opts the script out of derivation entirely. + // Ambiguous access (no `access_type`) fails safe and derives nothing. + const mute = parseMuteAnnotations(s.content); + if (!mute.muteAll) { + const skip = new Set(mute.muted); + for (const a of out.assets ?? []) { + if (a.access_type === "w" || a.access_type === "rw") { + skip.add(`${a.kind}:${a.path}`); + } + } + if (mat) { + skip.add(`${mat.target_kind}:${mat.target_path}`); + if (mat.scd2 && !mat.manual) { + skip.add(`${mat.target_kind}:${mat.target_path}_current`); + } + } + for (const t of out.triggers ?? []) { + if (t.kind === "asset") { + const at = t as { kind: "asset"; asset_kind: string; path: string }; + skip.add(`${at.asset_kind}:${at.path}`); + } + } + for (const a of out.assets ?? []) { + if (a.access_type !== "r" || !AUTO_TRIGGER_KINDS.has(a.kind)) continue; + const key = `${a.kind}:${a.path}`; + if (skip.has(key)) continue; + skip.add(key); + triggers.push({ + trigger_kind: "asset", + asset_kind: a.kind, + asset_path: a.path, + runnable_kind: "script", + runnable_path: s.path, + }); + } + } } const macroEdges = buildMacroEdges(all, libMacros, runnables); diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index 9b0c88ea11..59058bbcf4 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -3,7 +3,10 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { buildLocalPipelineGraph } from "../src/commands/pipeline/localGraph.ts"; +import { + buildLocalPipelineGraph, + parseMuteAnnotations, +} from "../src/commands/pipeline/localGraph.ts"; // Build a throwaway workspace tree with `f//` scripts and a // wmill.yaml at the root, then assert the graph the wasm-backed builder derives. @@ -290,6 +293,120 @@ test("HD-2: an scd2 `history` producer writes both `` and `_current` s ); }); +test("auto-derived cascade triggers: a body ducklake read wires the edge, incl. the scd2 `_current` view", async () => { + // Backend parity (#9963 `derive_pipeline_asset_trigger_refs`): inside a + // `// pipeline`, a read-only ducklake/s3 body read derives its cascade + // trigger — no `// on` needed. The scd2 `_current` companion is both written + // by the producer AND a derivable read for its consumer, so the full chain + // stg → dim → consumer must connect without a single explicit trigger. + await withFolder( + { + "stg.duckdb.sql": + `-- pipeline\n-- materialize ducklake://main/stg\nSELECT 1 AS id;\n`, + "dim.duckdb.sql": + `-- pipeline\n-- materialize ducklake://main/dim key=id history\nATTACH 'ducklake' AS dl;\nUSE dl;\nSELECT * FROM stg;\n`, + "consume.duckdb.sql": + `-- pipeline\nATTACH 'ducklake' AS dl;\nUSE dl;\nSELECT * FROM dim_current;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const ats = graph.triggers.filter((t) => t.trigger_kind === "asset") as Extract< + (typeof graph.triggers)[number], + { trigger_kind: "asset" } + >[]; + expect( + ats.map((t) => `${t.asset_path}->${t.runnable_path}`).sort(), + ).toEqual(["main/dim_current->f/mypipe/consume", "main/stg->f/mypipe/dim"]); + // the schema-level `main` read (ambiguous access) must NOT derive a trigger + expect(ats.some((t) => t.asset_path === "main")).toBe(false); + // the scd2 producer still carries the `_current` companion write the + // derived consumer edge resolves against (deployed-graph parity) + expect(graph.edges).toContainEqual({ + runnable_kind: "script", + runnable_path: "f/mypipe/dim", + asset_kind: "ducklake", + asset_path: "main/dim_current", + access_type: "w", + }); + }, + ); +}); + +test("`// mute ` suppresses one derived trigger; `// mute all` opts the script out", async () => { + await withFolder( + { + // reads two tables, mutes one → only the unmuted read derives + "partial.duckdb.sql": + `-- pipeline\n-- mute ducklake://main/lookup\nATTACH 'ducklake' AS dl;\nUSE dl;\nSELECT * FROM lookup JOIN facts USING (id);\n`, + // mute all: body read derives nothing, the explicit `// on` still stands + "optout.duckdb.sql": + `-- pipeline\n-- mute all\n-- on ducklake://main/manual\nATTACH 'ducklake' AS dl;\nUSE dl;\nSELECT * FROM facts;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const ats = graph.triggers.filter((t) => t.trigger_kind === "asset") as Extract< + (typeof graph.triggers)[number], + { trigger_kind: "asset" } + >[]; + expect( + ats.map((t) => `${t.asset_path}->${t.runnable_path}`).sort(), + ).toEqual(["main/facts->f/mypipe/partial", "main/manual->f/mypipe/optout"]); + }, + ); +}); + +test("derived triggers dedup against explicit `// on` and never self-trigger a materialize producer", async () => { + await withFolder( + { + // explicit `// on` for an asset the body also reads → exactly one trigger + "explicit.duckdb.sql": + `-- pipeline\n-- on ducklake://main/src debounce=60s\nATTACH 'ducklake' AS dl;\nUSE dl;\nSELECT * FROM src;\n`, + // an incremental model reading its own materialize target must not + // cascade on itself (the deploy path upgrades that read to rw) + "incremental.duckdb.sql": + `-- pipeline\n-- materialize ducklake://main/inc key=id\nATTACH 'ducklake' AS dl;\nUSE dl;\nSELECT * FROM inc;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + const ats = graph.triggers.filter((t) => t.trigger_kind === "asset") as Extract< + (typeof graph.triggers)[number], + { trigger_kind: "asset" } + >[]; + expect( + ats.map((t) => `${t.asset_path}->${t.runnable_path}`), + ).toEqual(["main/src->f/mypipe/explicit"]); + }, + ); +}); + +test("parseMuteAnnotations mirrors the canonical annotation grammar", () => { + // Any comment prefix regardless of language, header-only scan, complete-word + // keyword, s3 leading-slash canonicalization — in lockstep with the Rust + // `parse_pipeline_annotations` / frontend parsePipelineAnnotations.ts. + const all3 = parseMuteAnnotations( + `// mute ducklake://main/a\n-- mute datatable://main/b\n# mute s3:///lead/slash\nSELECT 1;\n// mute ducklake://main/body\n`, + ); + expect(all3.muteAll).toBe(false); + // all three prefixes accepted; s3 triple-slash canonicalizes to the bare key; + // the line PAST the first non-comment line is ignored (header-only) + expect([...all3.muted].sort()).toEqual([ + "datatable:main/b", + "ducklake:main/a", + "s3object:lead/slash", + ]); + + // `mute` must be a complete word, and prose args are not asset URIs + const prose = parseMuteAnnotations( + `// muted for now\n// mutex ducklake://main/x\n// mute for now\n`, + ); + expect(prose.muteAll).toBe(false); + expect(prose.muted.size).toBe(0); + + // `mute all` sets the opt-out; blank header lines are skipped, not a stop + const optout = parseMuteAnnotations(`-- pipeline\n\n-- mute all\nSELECT 1;\n`); + expect(optout.muteAll).toBe(true); +}); + test("a bare `.sql` (ambiguous dialect) is skipped, not a build-aborting crash", async () => { // `inferContentTypeFromFilePath` throws on a dialect-less `.sql`; one such file // must not abort the whole graph build (it also wedged `pipeline dev` at start). From 88c2d0e8e32c218787c01daed80ef41efc39dd11 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 7 Jul 2026 19:08:09 +0200 Subject: [PATCH 249/273] feat(cli): clarify fork-branch workspace auto-targeting in output (#9988) * feat(cli): clarify fork-branch workspace auto-targeting in output * fix(cli): auth comes from saved profile, not wmill.yaml, in fork notes * fix(cli): consolidate workspace resolution logs, fork-target last-used profile * chore: regenerate system prompts Co-Authored-By: Claude Fable 5 * fix(cli): fork-target interactively created profiles, dedupe workspace line --------- Co-authored-by: Claude Fable 5 --- cli/src/commands/workspace/fork.ts | 2 +- cli/src/commands/workspace/workspace.ts | 32 +++++ cli/src/core/conf.ts | 6 +- cli/src/core/context.ts | 109 +++++++++--------- cli/src/guidance/core.ts | 2 + ...ork_branch_profile_resolution_unit.test.ts | 93 +++++++++++++++ system_prompts/auto-generated/prompts.d.ts | 72 ++++++------ 7 files changed, 223 insertions(+), 93 deletions(-) create mode 100644 cli/test/fork_branch_profile_resolution_unit.test.ts diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 4248a18dc6..321154ccb7 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -390,7 +390,7 @@ async function createWorkspaceFork( log.info(`${checkoutHint} -When doing operations on the forked workspace, it will use the remote setup in the workspaces section for the branch it was forked from. +While on the branch \`${newBranchName}\`, every wmill command (sync pull/push, script run, ...) automatically targets the fork workspace \`${trueWorkspaceId}\` — no --workspace flag or profile switch needed. The remote comes from the base branch \`${clonedBranchName}\`'s entry in wmill.yaml's workspaces section, and auth is reused from that workspace's saved profile. To merge changes back to the parent workspace, you can: - Use the CLI: ` + colors.white(`git checkout ${newBranchName} && wmill workspace merge`) + ` diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index 7d6bb2c87e..bf5900b9e0 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -86,6 +86,29 @@ export async function getWorkspaceByName( return undefined; } +// When the current git branch is a fork branch (wm-fork//), commands +// resolve the fork workspace from the branch name and ignore the active +// profile — surface that wherever we display the active workspace, so users +// don't act on the wrong "Active:" line. +async function forkBranchAutoTargetNote(): Promise { + const { + getCurrentGitBranch, + getOriginalBranchForWorkspaceForks, + getWorkspaceIdForWorkspaceForkFromBranchName, + } = await import("../../utils/git.ts"); + const branch = getCurrentGitBranch(); + if (!branch || !getOriginalBranchForWorkspaceForks(branch)) { + return undefined; + } + const forkWorkspaceId = getWorkspaceIdForWorkspaceForkFromBranchName(branch); + if (!forkWorkspaceId) { + return undefined; + } + return ( + `Note: you are on fork branch \`${branch}\` — wmill commands automatically target the fork workspace \`${forkWorkspaceId}\` here, regardless of the active profile. Use --workspace to override.` + ); +} + export async function list(opts: GlobalOptions) { const workspaces = await allWorkspaces(opts.configDir); const activeName = await getActiveWorkspaceName(opts); @@ -107,6 +130,11 @@ export async function list(opts: GlobalOptions) { .render(); log.info("Active: " + colors.green.bold(activeName || "none")); + + const forkNote = await forkBranchAutoTargetNote(); + if (forkNote) { + log.info(colors.yellow(forkNote)); + } } async function switchC(opts: GlobalOptions, workspaceName: string) { @@ -140,6 +168,10 @@ async function switchC(opts: GlobalOptions, workspaceName: string) { `Switched to workspace ${workspaceName} (${workspace?.workspaceId} on ${workspace?.remote})` ) ); + const forkNote = await forkBranchAutoTargetNote(); + if (forkNote) { + log.info(colors.yellow(forkNote)); + } return; } diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index c4fda73b1b..0129a4fd18 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -405,7 +405,9 @@ export async function validateBranchConfiguration( let currentBranch: string | null; if (originalBranchIfForked) { - log.info( + // The fork targeting itself is announced by tryResolveBranchWorkspace; + // this validation detail stays at debug to avoid a near-duplicate line. + log.debug( `Workspace fork detected from branch name \`${rawBranch}\`. Validating workspace configuration using original branch \`${originalBranchIfForked}\`` ); currentBranch = originalBranchIfForked; @@ -563,7 +565,7 @@ export async function getEffectiveSettings( const branch = originalBranchIfForked ?? rawGitBranch; if (originalBranchIfForked) { - log.info( + log.debug( `Using overrides from original branch \`${originalBranchIfForked}\`` ); } diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 929a3728e1..6440213d5c 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -326,12 +326,9 @@ export async function tryResolveBranchWorkspace( workspaceIdIfForked = getWorkspaceIdForWorkspaceForkFromBranchName(rawBranch); + // The "matched via fork branch" reason logged below already explains the + // base-branch lookup, so no extra message here. const branchToLookup = originalBranchIfForked ?? rawBranch; - if (originalBranchIfForked) { - log.infoStderr( - `Using original branch \`${originalBranchIfForked}\` for finding workspace from workspaces section in wmill.yaml` - ); - } const match = findWorkspaceByGitBranch(config.workspaces, branchToLookup); if (match) { @@ -370,9 +367,9 @@ export async function tryResolveBranchWorkspace( reason = `matched current git branch '${rawBranch}'`; } - log.infoStderr( - `Using workspace '${wsName}' (${reason}) → ${workspaceId} on ${baseUrl}` - ); + // Printed as part of the single final targeting line on the happy paths; + // logged up front only when an interactive flow needs the context first. + const workspaceLine = `Using workspace '${wsName}' (${reason}) → ${workspaceId} on ${baseUrl}`; let normalizedBaseUrl: string; try { @@ -390,27 +387,34 @@ export async function tryResolveBranchWorkspace( (w) => w.remote === normalizedBaseUrl && w.workspaceId === workspaceId ); + // Every branch below must flow into the shared fork handling at the end — + // returning a profile directly would hand back the parent workspace profile + // on a fork branch. + let selectedProfile: Workspace; + let profileNote: string; + // Set when workspaceLine was already printed as context for the interactive + // profile-creation flow, so it isn't printed a second time at the end. + let workspaceLinePrinted = false; + if (matchingProfiles.length === 0) { // No matching profile exists - prompt to create one - return await createWorkspaceProfileInteractively( + log.infoStderr(workspaceLine); + workspaceLinePrinted = true; + const created = await createWorkspaceProfileInteractively( normalizedBaseUrl, workspaceId, wsName, opts, { rawBranch: rawBranch ?? wsName, isForked: !!originalBranchIfForked } ); - } - - // Handle multiple profiles - let selectedProfile: Workspace; - - if (matchingProfiles.length === 1) { + if (!created) { + return undefined; + } + selectedProfile = created; + profileNote = `profile '${selectedProfile.name}'`; + } else if (matchingProfiles.length === 1) { selectedProfile = matchingProfiles[0]; - log.infoStderr( - colors.green( - `Using workspace profile '${selectedProfile.name}' for workspace '${wsName}' with workspace id \`${workspaceId}\`` - ) - ); + profileNote = `profile '${selectedProfile.name}'`; } else { const lastUsedName = await getLastUsedProfile( wsName, @@ -418,50 +422,45 @@ export async function tryResolveBranchWorkspace( workspaceId, opts.configDir ); + const lastUsedProfile = lastUsedName + ? matchingProfiles.find((p) => p.name === lastUsedName) + : undefined; - if (lastUsedName) { - const lastUsedProfile = matchingProfiles.find( - (p) => p.name === lastUsedName + if (lastUsedProfile) { + selectedProfile = lastUsedProfile; + profileNote = `last used profile '${selectedProfile.name}'`; + } else { + // selectFromMultipleProfiles prints its own context header, and the + // final summary line below names the chosen profile. + selectedProfile = await selectFromMultipleProfiles( + matchingProfiles, + normalizedBaseUrl, + workspaceId, + `workspace '${wsName}'`, + opts.configDir ); - if (lastUsedProfile) { - log.infoStderr( - colors.green( - `Using workspace profile '${lastUsedProfile.name}' for workspace '${wsName}' (last used)` - ) - ); - return lastUsedProfile; - } + + await setLastUsedProfile( + wsName, + normalizedBaseUrl, + workspaceId, + selectedProfile.name, + opts.configDir + ); + profileNote = `profile '${selectedProfile.name}'`; } - - selectedProfile = await selectFromMultipleProfiles( - matchingProfiles, - normalizedBaseUrl, - workspaceId, - `workspace '${wsName}'`, - opts.configDir - ); - - await setLastUsedProfile( - wsName, - normalizedBaseUrl, - workspaceId, - selectedProfile.name, - opts.configDir - ); - - log.infoStderr( - colors.green( - `Using workspace profile '${selectedProfile.name}' for workspace '${wsName}'` - ) - ); } if (workspaceIdIfForked) { selectedProfile.name = `${selectedProfile.name}/${workspaceIdIfForked}`; selectedProfile.workspaceId = workspaceIdIfForked; log.infoStderr( - `Using fork workspace \`${workspaceIdIfForked}\` (parent: \`${workspaceId}\`) from branch \`${rawBranch}\`` + colors.green( + `Automatically targeting fork workspace \`${workspaceIdIfForked}\` (fork of \`${workspaceId}\` on ${baseUrl}, ${profileNote}), resolved from git branch \`${rawBranch}\`. Use --workspace to override.` + ) ); + } else if (!workspaceLinePrinted) { + log.infoStderr(`${workspaceLine} (${profileNote})`); } return selectedProfile; @@ -556,7 +555,7 @@ export async function resolveWorkspace( return workspace; } else { log.infoStderr( - `Found an active workspace \`${workspace.name}\` but the branch name indicates this is a forked workspace. Ignoring active workspace and trying to resolve the correct workspace from the branch name \`${branch}\`. Use --workspace to override.` + `You are on fork branch \`${branch}\`, which takes precedence over the active workspace profile \`${workspace.name}\`: resolving the fork workspace from the branch name instead. Use --workspace to override.` ); } } else if (opts.workspace) { diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index cd9ba686ef..1d060a58cb 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -177,6 +177,8 @@ Just run \`wmill workspace fork\` — it adapts to where you are: For non-interactive runs from a working branch, pass \`--from-branch \` to skip the prompts. The CLI refuses to rename a base branch. +While a \`wm-fork//\` branch is checked out, every wmill command automatically targets the fork workspace (resolved from the branch name), reusing the base branch workspace's remote (from wmill.yaml) and its saved profile's auth — no \`--workspace\` flag or profile switch needed. Pass \`--workspace\` to target a different workspace explicitly. + Merge a fork back into its parent with \`wmill workspace merge\` (or the Merge UI on the fork's home page). Full reference: https://www.windmill.dev/docs/advanced/workspace_forks ## Debugging Jobs diff --git a/cli/test/fork_branch_profile_resolution_unit.test.ts b/cli/test/fork_branch_profile_resolution_unit.test.ts new file mode 100644 index 0000000000..57fe2cb958 --- /dev/null +++ b/cli/test/fork_branch_profile_resolution_unit.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { execSync } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; +import { stringify as yamlStringify } from "yaml"; + +import { tryResolveBranchWorkspace } from "../src/core/context.ts"; +import { setLastUsedProfile } from "../src/core/branch-profiles.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; +import type { Workspace } from "../src/commands/workspace/workspace.ts"; + +const BASE_URL = "http://localhost:9999/"; +const PARENT_WORKSPACE_ID = "parent"; + +// Fork-branch workspace resolution: every profile-selection path in +// tryResolveBranchWorkspace must rewrite the returned profile to the fork +// workspace id derived from the wm-fork// branch — a path that +// returns the parent profile untouched silently targets the parent workspace. +async function withForkBranchSetup( + profiles: Workspace[], + fn: (opts: GlobalOptions) => Promise, +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_fork_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_fork_conf_")); + const originalCwd = process.cwd(); + try { + execSync(`git init -q -b wm-fork/main/myfork`, { cwd: repoDir }); + execSync(`git config user.email test@example.com`, { cwd: repoDir }); + execSync(`git config user.name test`, { cwd: repoDir }); + + await writeFile( + path.join(repoDir, "wmill.yaml"), + yamlStringify({ + workspaces: { + main: { baseUrl: BASE_URL, workspaceId: PARENT_WORKSPACE_ID }, + }, + }), + ); + execSync(`git add wmill.yaml && git commit -q -m init`, { cwd: repoDir }); + + const remotesPath = await getWorkspaceConfigFilePath(configDir); + await writeFile( + remotesPath, + profiles.map((p) => JSON.stringify(p)).join("\n") + "\n", + ); + + process.chdir(repoDir); + await fn({ configDir } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +function profile(name: string): Workspace { + return { + name, + remote: BASE_URL, + workspaceId: PARENT_WORKSPACE_ID, + token: `token-${name}`, + }; +} + +describe("tryResolveBranchWorkspace on a fork branch", () => { + test("single matching profile targets the fork workspace", async () => { + await withForkBranchSetup([profile("prod")], async (opts) => { + const ws = await tryResolveBranchWorkspace(opts); + expect(ws?.workspaceId).toEqual("wm-fork-myfork"); + expect(ws?.token).toEqual("token-prod"); + }); + }); + + test("last-used profile among multiple still targets the fork workspace", async () => { + await withForkBranchSetup( + [profile("prod"), profile("prod-alt")], + async (opts) => { + await setLastUsedProfile( + "main", + BASE_URL, + PARENT_WORKSPACE_ID, + "prod-alt", + opts.configDir, + ); + const ws = await tryResolveBranchWorkspace(opts); + expect(ws?.workspaceId).toEqual("wm-fork-myfork"); + expect(ws?.token).toEqual("token-prod-alt"); + }, + ); + }); +}); diff --git a/system_prompts/auto-generated/prompts.d.ts b/system_prompts/auto-generated/prompts.d.ts index 97b72d399b..e74555504e 100644 --- a/system_prompts/auto-generated/prompts.d.ts +++ b/system_prompts/auto-generated/prompts.d.ts @@ -1,35 +1,37 @@ -export declare const SCRIPT_BASE = "# Windmill Script Writing Guide\n\n## General Principles\n\n- Scripts must export a main function (do not call it)\n- Libraries are installed automatically - do not show installation instructions\n- Credentials and configuration are stored in resources and passed as parameters\n- The windmill client (`wmill`) provides APIs for interacting with the platform\n\n## Function Naming\n\n- Main function: `main` (or `preprocessor` for preprocessor scripts)\n- Must be async for TypeScript variants\n\n## Return Values\n\n- Scripts can return any JSON-serializable value\n- Return values become available to subsequent flow steps via `results.step_id`\n\n## Preprocessor Scripts\n\nPreprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean.\n\nThe returned object determines the parameter values passed to the flow.\ne.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`.\n\nThe preprocessor receives a single parameter called `event`.\n"; -export declare const FLOW_BASE = "# Windmill Flow Building Guide\n\n## OpenFlow Schema\n\nThe OpenFlow schema (openflow.openapi.yaml) is the source of truth for flow structure. Refer to OPENFLOW_SCHEMA for the complete type definitions.\n\n## Reserved Module IDs\n\n- `failure` - Reserved for failure handler module\n- `preprocessor` - Reserved for preprocessor module\n- `Input` - Reserved for flow input reference\n\n## Hard Structural Rules\n\nThese are strict Windmill schema rules. Follow them exactly.\n\n- `value.modules` is only for normal sequential steps\n- `value.preprocessor_module` and `value.failure_module` are special top-level fields inside `value`, not entries in `value.modules`\n- If a flow needs a preprocessor, create `value.preprocessor_module` with `id: preprocessor`\n- If a flow needs a failure handler, create `value.failure_module` with `id: failure`\n- Do NOT create regular modules inside `value.modules` named `preprocessor` or `failure`\n- `preprocessor_module` and `failure_module` only support `script` or `rawscript`\n- `preprocessor_module` runs before normal modules and cannot reference `results.*`\n- `failure_module` can use the `error` object with `error.message`, `error.step_id`, `error.name`, and `error.stack`\n\nCorrect shape:\n\n```yaml\nvalue:\n preprocessor_module:\n id: preprocessor\n value:\n type: rawscript\n ...\n failure_module:\n id: failure\n value:\n type: rawscript\n ...\n modules:\n - id: process_event\n value:\n type: rawscript\n ...\n```\n\nIncorrect shape:\n\n```yaml\nvalue:\n modules:\n - id: preprocessor\n ...\n - id: process_event\n ...\n - id: failure\n ...\n```\n\n## Module ID Rules\n\n- Must be unique across the entire flow\n- Use underscores, not spaces (e.g., `fetch_data` not `fetch data`)\n- Use descriptive names that reflect the step's purpose\n\n## Common Mistakes to Avoid\n\n- Missing `input_transforms` - Rawscript parameters won't receive values without them\n- Referencing future steps - `results.step_id` only works for steps that execute before the current one\n- Duplicate module IDs - Each module ID must be unique in the flow\n\n## Data Flow Between Steps\n\n- `flow_input.property` - Access flow input parameters\n- `results.step_id` - Access output from a previous step only when that step result is in scope\n- `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope\n- `flow_input.iter.value` - Current iteration value when inside a loop (`forloopflow` or `whileloopflow`)\n- `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`)\n\n## Loop Structure Rules\n\n- For `whileloopflow`, use module-level `stop_after_if` on the loop module itself when the loop should stop after an iteration result\n- Do NOT put `stop_after_if` inside `value` of a `whileloopflow`\n- `stop_after_all_iters_if` is for checks after the whole loop finishes, not the normal per-iteration break condition\n- When a `whileloopflow` carries state forward between iterations, use `flow_input.iter.value` as the current loop value and provide an explicit first-iteration fallback when needed\n- Use `flow_input.iter.index` only when the loop logic is truly based on the iteration index, not as a replacement for the current loop value\n- If the user asks for a final scalar/object after a loop, add a normal step after the loop that extracts the final value from the loop result instead of returning the whole loop result array\n\nCorrect `whileloopflow` shape:\n\n```yaml\n- id: loop_until_done\n stop_after_if:\n expr: result.done === true\n skip_if_stopped: false\n value:\n type: whileloopflow\n skip_failures: false\n modules:\n - id: advance_state\n value:\n type: rawscript\n input_transforms:\n state:\n type: javascript\n expr: flow_input.iter && flow_input.iter.value !== undefined ? flow_input.iter.value : flow_input.initial_state\n- id: return_final_state\n value:\n type: rawscript\n input_transforms:\n final_state:\n type: javascript\n expr: results.loop_until_done[results.loop_until_done.length - 1]\n```\n\nIncorrect `whileloopflow` patterns:\n\n```yaml\n- id: loop_until_done\n value:\n type: whileloopflow\n stop_after_if:\n expr: result.done === true\n```\n\n```yaml\ninput_transforms:\n state:\n type: javascript\n expr: flow_input.iter.index\n```\n\n```yaml\ninput_transforms:\n final_state:\n type: javascript\n expr: results.loop_until_done\n```\n\n## Approval / Suspend Structure\n\n- `suspend` belongs on the flow module object itself, as a sibling of `id` and `value`\n- Never put `suspend` inside `value`\n\nCorrect shape:\n\n```yaml\n- id: request_approval\n suspend:\n required_events: 1\n resume_form:\n schema:\n type: object\n properties:\n comment:\n type: string\n required: [comment]\n value:\n type: identity\n```\n\nIncorrect shape:\n\n```yaml\n- id: request_approval\n value:\n type: rawscript\n suspend:\n required_events: 1\n```\n\n## Branch Result Scope Rules\n\n- Inside a branch, you may reference earlier outer steps and earlier steps in the same branch\n- Outside a `branchone`, do NOT reference ids of steps that only exist inside its branches or default branch. Use `results.` instead\n- Outside a `branchall`, do NOT reference ids of steps inside its branches. Use `results.` instead\n- If downstream steps need a stable shape after a branch, make each branch return the same fields\n- When needed, add a normalization step immediately after the branch and consume `results.` there\n\nCorrect after `branchone`:\n\n```yaml\n- id: route_order\n value:\n type: branchone\n ...\n- id: send_confirmation\n value:\n input_transforms:\n routed:\n type: javascript\n expr: results.route_order\n```\n\nIncorrect after `branchone`:\n\n```yaml\nexpr: results.create_shipment\nexpr: results.create_backorder\n```\n\nCorrect after `branchall`:\n\n```yaml\n- id: enrich_parallel\n value:\n type: branchall\n parallel: true\n ...\n- id: combine_data\n value:\n input_transforms:\n enrichments:\n type: javascript\n expr: results.enrich_parallel\n```\n\n## Input Transforms\n\nEvery rawscript module needs `input_transforms` to map function parameters to values:\n\nStatic transform (fixed value):\n{\"param_name\": {\"type\": \"static\", \"value\": \"fixed_string\"}}\n\nJavaScript transform (dynamic expression):\n{\"param_name\": {\"type\": \"javascript\", \"expr\": \"results.previous_step.data\"}}\n\n## Resource References\n\n- For flow inputs: Use type `\"object\"` with format `\"resource-{type}\"` (e.g., `\"resource-postgresql\"`)\n- For step inputs: Use static value `\"$res:path/to/resource\"`\n\n## Final Structural Self-Check\n\nBefore finalizing a flow, verify:\n\n- any preprocessor is in `value.preprocessor_module`\n- any failure handler is in `value.failure_module`\n- any approval step has module-level `suspend`\n- no downstream step references inner branch step ids from outside the branch\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations.\n\nTo accept an S3 object as flow input:\n\n```json\n{\n \"type\": \"object\",\n \"properties\": {\n \"file\": {\n \"type\": \"object\",\n \"format\": \"resource-s3_object\",\n \"description\": \"File to process\"\n }\n }\n}\n```\n\n## Using Resources in Flows\n\nOn Windmill, credentials and configuration are stored in resources. Resource types define the format of the resource.\n\n### As Flow Input\n\nIn the flow schema, set the property type to `\"object\"` with format `\"resource-{type}\"`:\n\n```json\n{\n \"type\": \"object\",\n \"properties\": {\n \"database\": {\n \"type\": \"object\",\n \"format\": \"resource-postgresql\",\n \"description\": \"Database connection\"\n }\n }\n}\n```\n\n### As Step Input (Static Reference)\n\nReference a specific resource using `$res:` prefix:\n\n```json\n{\n \"database\": {\n \"type\": \"static\",\n \"value\": \"$res:f/folder/my_database\"\n }\n}\n```\n"; -export declare const RESOURCES_BASE = "# Windmill Resources\n\nResources store credentials and configuration for external services.\n\n## File Format\n\nResource files use the pattern: `{path}.resource.json`\n\nExample: `f/databases/postgres_prod.resource.json`\n\n## Resource Structure\n\n```json\n{\n \"value\": {\n \"host\": \"db.example.com\",\n \"port\": 5432,\n \"user\": \"admin\",\n \"password\": \"$var:g/all/db_password\",\n \"dbname\": \"production\"\n },\n \"description\": \"Production PostgreSQL database\",\n \"resource_type\": \"postgresql\"\n}\n```\n\n## Required Fields\n\n- `value` - Object containing the resource configuration\n- `resource_type` - Name of the resource type (e.g., \"postgresql\", \"slack\")\n\n## Variable References\n\nReference variables in resource values:\n\n```json\n{\n \"value\": {\n \"api_key\": \"$var:g/all/api_key\",\n \"secret\": \"$var:u/admin/secret\"\n }\n}\n```\n\n**Reference formats:**\n- `$var:g/all/name` - Global variable\n- `$var:u/username/name` - User variable\n- `$var:f/folder/name` - Folder variable\n\n## Resource References\n\nReference other resources:\n\n```json\n{\n \"value\": {\n \"database\": \"$res:f/databases/postgres\"\n }\n}\n```\n\n## Common Resource Types\n\n### PostgreSQL\n```json\n{\n \"resource_type\": \"postgresql\",\n \"value\": {\n \"host\": \"localhost\",\n \"port\": 5432,\n \"user\": \"postgres\",\n \"password\": \"$var:g/all/pg_password\",\n \"dbname\": \"windmill\",\n \"sslmode\": \"prefer\"\n }\n}\n```\n\n### MySQL\n```json\n{\n \"resource_type\": \"mysql\",\n \"value\": {\n \"host\": \"localhost\",\n \"port\": 3306,\n \"user\": \"root\",\n \"password\": \"$var:g/all/mysql_password\",\n \"database\": \"myapp\"\n }\n}\n```\n\n### Slack\n```json\n{\n \"resource_type\": \"slack\",\n \"value\": {\n \"token\": \"$var:g/all/slack_token\"\n }\n}\n```\n\n### AWS S3\n```json\n{\n \"resource_type\": \"s3\",\n \"value\": {\n \"bucket\": \"my-bucket\",\n \"region\": \"us-east-1\",\n \"accessKeyId\": \"$var:g/all/aws_access_key\",\n \"secretAccessKey\": \"$var:g/all/aws_secret_key\"\n }\n}\n```\n\n### HTTP/API\n```json\n{\n \"resource_type\": \"http\",\n \"value\": {\n \"baseUrl\": \"https://api.example.com\",\n \"headers\": {\n \"Authorization\": \"Bearer $var:g/all/api_token\"\n }\n }\n}\n```\n\n### Kafka\n```json\n{\n \"resource_type\": \"kafka\",\n \"value\": {\n \"brokers\": \"broker1:9092,broker2:9092\",\n \"sasl_mechanism\": \"PLAIN\",\n \"security_protocol\": \"SASL_SSL\",\n \"username\": \"$var:g/all/kafka_user\",\n \"password\": \"$var:g/all/kafka_password\"\n }\n}\n```\n\n### NATS\n```json\n{\n \"resource_type\": \"nats\",\n \"value\": {\n \"servers\": [\"nats://localhost:4222\"],\n \"user\": \"$var:g/all/nats_user\",\n \"password\": \"$var:g/all/nats_password\"\n }\n}\n```\n\n### MQTT\n```json\n{\n \"resource_type\": \"mqtt\",\n \"value\": {\n \"host\": \"mqtt.example.com\",\n \"port\": 8883,\n \"username\": \"$var:g/all/mqtt_user\",\n \"password\": \"$var:g/all/mqtt_password\",\n \"tls\": true\n }\n}\n```\n\n## Custom Resource Types\n\nCreate custom resource types with JSON Schema:\n\n```json\n{\n \"name\": \"custom_api\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"base_url\": {\"type\": \"string\", \"format\": \"uri\"},\n \"api_key\": {\"type\": \"string\"},\n \"timeout\": {\"type\": \"integer\", \"default\": 30}\n },\n \"required\": [\"base_url\", \"api_key\"]\n },\n \"description\": \"Custom API connection\"\n}\n```\n\nSave as: `custom_api.resource-type.json`\n\n## OAuth Resources\n\nOAuth resources are managed through the Windmill UI and marked:\n\n```json\n{\n \"is_oauth\": true,\n \"account\": 123\n}\n```\n\nOAuth tokens are automatically refreshed by Windmill.\n\n## Using Resources in Scripts\n\n### TypeScript (Bun/Deno)\n```typescript\nexport async function main(db: RT.Postgresql) {\n // db contains the resource values\n const { host, port, user, password, dbname } = db;\n}\n```\n\n### Python\n```python\nclass postgresql(TypedDict):\n host: str\n port: int\n user: str\n password: str\n dbname: str\n\ndef main(db: postgresql):\n # db contains the resource values\n pass\n```\n\n## CLI Commands\n\n```bash\n# List resources\nwmill resource list\n\n# List resource types with schemas\nwmill resource-type list --schema\n\n# Get specific resource type schema\nwmill resource-type get postgresql\n\n# Push resources to Windmill \u2014 deploys to the workspace and can be destructive to\n# remote state, so only run it when the user explicitly asks to deploy/publish/push\nwmill sync push\n```\n"; -export declare const RAW_APP_BASE = "# Windmill Raw Apps\n\nRaw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables.\n\n## App shape\n\nA raw app has three logical parts:\n\n- **Frontend** \u2014 bundled with esbuild from `index.tsx` as the entrypoint. Files include the entrypoint, components (`App.tsx`), styles, etc.\n- **Backend runnables** \u2014 server-side scripts the frontend calls, each addressed by a unique key.\n- **Data** \u2014 optional whitelisted datatables (managed PostgreSQL) that the backend runnables can query. The frontend never queries the database directly; backend runnables are the only bridge.\n\n## Frontend\n\n### Entrypoint\n\n`index.tsx` is the bundling entrypoint. It typically renders a top-level `App` component. The bundler is esbuild.\n\n**Always begin every React file (`.tsx`/`.jsx`) that uses JSX with `import React from 'react'`.** esbuild uses the classic JSX transform, so `React` must be in scope wherever JSX appears \u2014 a missing import compiles fine but throws `React is not defined` at runtime, leaving a blank screen.\n\n### Generated bindings (`wmill.d.ts` / `wmill.ts`)\n\nThe frontend imports a generated module that mirrors the backend runnables. **Never write to it directly** \u2014 it gets regenerated whenever backend runnables change. Modifying it by hand will be overwritten.\n\n### Calling backend runnables\n\nImport the generated bindings and call the runnable like a function:\n\n```typescript\nimport { backend } from './wmill';\n\n// Call a backend runnable\nconst user = await backend.get_user({ user_id: '123' });\n```\n\nThe frontend cannot reach datatables, workspace items, or external services on its own \u2014 it goes through `backend.(args)` for everything server-side.\n\n## Backend runnables\n\nEach runnable has a unique key (used to call it from the frontend) and one of four types:\n\n| Type | What it is |\n|---|---|\n| `inline` | Custom code stored on the app itself. Most common for app-specific logic. |\n| `script` | Reference to an existing workspace script by path. |\n| `flow` | Reference to an existing workspace flow by path. |\n| `hubscript` | Reference to a hub script by path. |\n\n### Inline runnables\n\nInline runnables carry their own source code. For file-based raw apps, the runnable language is determined by the backend file extension. The script must expose a `main` function as its entrypoint.\n\n**TypeScript example** (`backend/get_user.ts`):\n\n```typescript\nimport * as wmill from 'windmill-client';\n\nexport async function main(user_id: string) {\n const sql = wmill.datatable();\n const user = await sql`SELECT * FROM users WHERE id = ${user_id}`.fetchOne();\n return user;\n}\n```\n\n**Python example** (`backend/get_user.py`):\n\n```python\nimport wmill\n\ndef main(user_id: str):\n db = wmill.datatable()\n user = db.query('SELECT * FROM users WHERE id = $1', user_id).fetch_one()\n return user\n```\n\n### Path runnables (script / flow / hubscript)\n\nWhen `type` is `script`, `flow`, or `hubscript`, the runnable just stores a `path` to an existing workspace or hub item \u2014 no inline code. The referenced item's input/output schema becomes the runnable's surface.\n\n### Static inputs\n\n`staticInputs` is an optional `Record` for arguments not overridable from the frontend. Useful with path runnables to pre-fill some args while leaving the rest to the frontend caller.\n\n## Data Tables\n\nData tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly.\n\n### Critical rules\n\n1. **Whitelisted tables only**: a runnable can only query tables listed in the app's `data.tables` config. Tables not in this list are not accessible.\n2. **Add tables before using**: queries against unlisted tables fail at runtime. When you introduce a new table, register it in `data.tables` first.\n3. **Use the configured datatable/schema**: the app's `data` config sets the default datatable and schema; reference them consistently across runnables.\n\n### Querying in TypeScript (Bun/Deno)\n\n```typescript\nimport * as wmill from 'windmill-client';\n\nexport async function main(user_id: string) {\n const sql = wmill.datatable(); // Or: wmill.datatable('other_datatable')\n\n // Parameterized queries (safe from SQL injection)\n const user = await sql`SELECT * FROM users WHERE id = ${user_id}`.fetchOne();\n const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch();\n\n // Insert/Update\n await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`;\n await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`;\n\n return user;\n}\n```\n\n### Querying in Python\n\n```python\nimport wmill\n\ndef main(user_id: str):\n db = wmill.datatable() # Or: wmill.datatable('other_datatable')\n\n # Use $1, $2, etc. for parameters\n user = db.query('SELECT * FROM users WHERE id = $1', user_id).fetch_one()\n users = db.query('SELECT * FROM users WHERE active = $1', True).fetch()\n\n # Insert/Update\n db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email)\n db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id)\n\n return user\n```\n\n## Best Practices\n\n1. **Check existing tables** before creating new ones \u2014 reuse beats schema growth.\n2. **Use parameterized queries** \u2014 never concatenate user input into SQL.\n3. **Keep runnables focused** \u2014 one function per runnable; small surface area.\n4. **Use descriptive keys** \u2014 `get_user`, not `a`.\n5. **Always whitelist tables** \u2014 adding a runnable that queries a new table requires the table to be in `data.tables` first.\n"; -export declare const PIPELINE_BASE = "# Data pipeline authoring\n\nA **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a \"data pipeline\" (or to \"ingest / transform / materialize\" data across steps), build pipeline-annotated scripts \u2014 do NOT build a flow.\n\n## What makes a script a pipeline node\n\nA script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** \u2014 `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file:\n\n- `// on ` \u2014 declares an execution-DAG **input** (what triggers/feeds this node). `` is either:\n - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `s3://`, `$res:f/folder/my_resource`, `volume://name/path`.\n - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, \u2026) is created separately; the annotation only declares the binding.\n- **Outputs** are inferred from what the body writes \u2014 a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `.\n- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`.\n\n## Materialize (the managed output)\n\n> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` \u2014 write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, \u2026) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table.\n\n`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace \u2014 do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy:\n\n- no option \u2192 **replace** the whole table each run (full refresh; the only mode whose output columns may change);\n- `// materialize append` \u2192 INSERT-append rows (incremental);\n- `// materialize key=` \u2192 merge/upsert on ``.\n\n`// materialize manual ` opts **out** of managed writes \u2014 the script writes its own DDL and the annotation only records the output asset for lineage.\n\n`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time.\n\n`materialize` is an output **declaration** on a node \u2014 not a command. There is no \"materialize run\".\n\n## How to build one in chat\n\n1. Put every node in the **same folder**: `f//`. The folder is the pipeline.\n2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`.\n3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output.\n4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones.\n5. Leave nodes as drafts unless the user asks to deploy. A pipeline only \"runs\" once its scripts are deployed and their triggers exist.\n\nWhen the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above.\n\n## Example (DuckDB \u2192 DuckLake, scheduled ingest + downstream transform)\n\nNode `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table):\n\n```sql\n-- pipeline\n-- on schedule\n-- materialize ducklake://main/orders\nSELECT * FROM read_csv('s3://raw/orders/*.csv')\n```\n\nNode `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup):\n\n```sql\n-- pipeline\n-- on ducklake://main/orders\n-- materialize ducklake://main/orders_daily\nSELECT date_trunc('day', ts) AS day, count(*) AS n\nFROM ducklake.main.orders GROUP BY 1\n```\n"; -export declare const WORKFLOW_AS_CODE_BASE = "# Windmill Workflow-as-Code Writing Guide\n\n## Scope\n\nUse this guide when writing or modifying Windmill Workflow-as-Code (WAC) scripts.\nWAC is authored as a Windmill script and deployed with the normal script workflow. It is not an OpenFlow YAML flow.\n\nSupported WAC authoring targets:\n- Bun TypeScript scripts that import from `windmill-client`\n- Python 3 scripts that import from `wmill`\n\n## File Shape\n\nBun TypeScript:\n\n```typescript\nimport {\n task,\n taskScript,\n taskFlow,\n step,\n sleep,\n waitForApproval,\n getResumeUrls,\n parallel,\n workflow,\n} from \"windmill-client\";\n\nconst process = task(async (x: string): Promise => {\n return `processed: ${x}`;\n});\n\nexport const main = workflow(async (x: string) => {\n const result = await process(x);\n return { result };\n});\n```\n\nPython:\n\n```python\nfrom wmill import task, task_script, task_flow, step, sleep, wait_for_approval, get_resume_urls, parallel, workflow\n\n@task()\nasync def process(x: str) -> str:\n return f\"processed: {x}\"\n\n@workflow\nasync def main(x: str):\n result = await process(x)\n return {\"result\": result}\n```\n\nRules:\n- Do not call `main`.\n- Bun TypeScript should export the workflow entrypoint, preferably `export const main = workflow(async (...) => { ... })`.\n- Python must use `@workflow` on an async top-level function, usually `main`.\n- Define task functions and `taskScript`/`task_script` or `taskFlow`/`task_flow` assignments at module top level with stable names.\n- Use the exact SDK names. Do not alias `workflow`, `task`, `taskScript`, `taskFlow`, `step`, `sleep`, `waitForApproval`, `task_script`, `task_flow`, or `wait_for_approval`; the WAC parser recognizes these names directly.\n\n## Checkpoint And Replay Model\n\nThe parent workflow may rerun from the top after any suspension, retry, approval, or child task completion. Completed durable steps are replayed from the checkpoint.\n\nPut every side effect or non-deterministic value behind a durable WAC boundary:\n- Use `task()` / `@task()` for substantial work that should run as its own child job.\n- Use `taskScript()` / `task_script()` for an existing script or a relative module file.\n- Use `taskFlow()` / `task_flow()` for an existing Windmill flow.\n- Use `step(name, fn)` for lightweight inline work whose result must be checkpointed.\n- Use `sleep(seconds)` for server-side sleeps that do not hold a worker.\n- Use `waitForApproval()` / `wait_for_approval()` for external approval suspension.\n\nNever put API calls, database writes, notifications, random values, timestamps, or irreversible changes directly in the top-level workflow body. The workflow body can be rerun. Put those operations in a task or in `step()`.\n\nBranching on task or step results is safe because those results are checkpointed. Branching on current time, random data, environment reads, or external state is unsafe unless the value is first captured with `step()`.\n\n## Tasks\n\nUse `task()` / `@task()` for inline functions that become workflow steps:\n\n```typescript\nconst enrich = task(async (customerId: string) => {\n return await fetchCustomer(customerId);\n});\n```\n\n```python\n@task(timeout=600, tag=\"etl\")\nasync def enrich(customer_id: str):\n return await fetch_customer(customer_id)\n```\n\nIn TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with `@task()` or `@task`.\n\nFor existing scripts:\n\n```typescript\nconst helper = taskScript(\"./helper.ts\");\nconst existing = taskScript(\"f/data/extract\", { timeout: 600 });\nconst value = await helper({ input: x });\n```\n\n```python\nhelper = task_script(\"./helper.py\")\nexisting = task_script(\"f/data/extract\", timeout=600)\nvalue = await helper(input=x)\n```\n\nFor existing flows:\n\n```typescript\nconst pipeline = taskFlow(\"f/etl/pipeline\");\nconst output = await pipeline({ input: data });\n```\n\n```python\npipeline = task_flow(\"f/etl/pipeline\")\noutput = await pipeline(input=data)\n```\n\n## Inline Steps\n\nUse `step()` for lightweight inline values that must not change during replay:\n\n```typescript\nconst urls = await step(\"get_urls\", () => getResumeUrls());\nconst startedAt = await step(\"started_at\", () => new Date().toISOString());\n```\n\n```python\nurls = await step(\"get_urls\", lambda: get_resume_urls())\n```\n\nUse stable, descriptive step names. Do not generate step names dynamically.\n\n## Parallelism\n\nTo run independent work in parallel, start task promises/coroutines before awaiting them together:\n\n```typescript\nconst [a, b] = await Promise.all([process(\"a\"), process(\"b\")]);\nconst many = await parallel(items, process, { concurrency: 5 });\n```\n\n```python\nimport asyncio\n\na, b = await asyncio.gather(process(\"a\"), process(\"b\"))\nmany = await parallel(items, process, concurrency=5)\n```\n\nOnly parallelize independent steps. Do not read the result of a task before it is awaited.\n\n## Approvals\n\nGenerate resume URLs inside `step()` before sending them:\n\n```typescript\nconst urls = await step(\"get_urls\", () => getResumeUrls());\nawait step(\"notify\", () => sendApprovalEmail(urls.approvalPage));\nconst approval = await waitForApproval({ timeout: 3600 });\n```\n\n```python\nurls = await step(\"get_urls\", lambda: get_resume_urls())\nawait step(\"notify\", lambda: send_approval_email(urls[\"approvalPage\"]))\napproval = await wait_for_approval(timeout=3600)\n```\n\n`selfApproval: false` and `self_approval=False` are Enterprise-only approval behavior. Do not use them unless the user asks for that behavior.\n\n## Error Handling\n\nLet task errors fail the workflow unless the user asks for recovery logic.\n\nPython: `except Exception` is safe around WAC calls because internal suspension inherits from `BaseException`. Avoid bare `except:` in workflow code. If the user asks for recovery logic around failed child work, catch `TaskError` from `wmill` for task failures.\n\nTypeScript: avoid broad `try/catch` around WAC SDK calls. The SDK uses an internal suspension error during initial dispatch; catching it can break workflow suspension. If a broad catch is unavoidable, rethrow internal suspension errors before handling business errors.\n"; -export declare const FLOW_CHAT_SPECIAL_MODULES = "## Special Modules\n\n- Use `set_preprocessor_module` to add, replace, or remove the top-level `value.preprocessor_module`\n- Use `set_failure_module` to add, replace, or remove the top-level `value.failure_module`\n- Use `set_flow_json` only when you are replacing the whole flow, including normal modules and optional special modules\n\n**Example - Update only the special modules:**\n```javascript\nset_preprocessor_module({\n module: JSON.stringify({\n id: \"preprocessor\",\n value: {\n type: \"rawscript\",\n language: \"bun\",\n content: \"export async function preprocessor(payload: string) { const trimmed = payload.trim(); if (!trimmed) { throw new Error('payload must not be empty'); } return { payload: trimmed }; }\",\n input_transforms: {\n payload: { type: \"javascript\", expr: \"flow_input.payload\" }\n }\n }\n })\n})\n\nset_failure_module({\n module: JSON.stringify({\n id: \"failure\",\n value: {\n type: \"rawscript\",\n language: \"bun\",\n content: \"export async function main(message: string, name: string, step_id: string) { return { message, name, step_id }; }\",\n input_transforms: {\n message: { type: \"javascript\", expr: \"error.message\" },\n name: { type: \"javascript\", expr: \"error.name\" },\n step_id: { type: \"javascript\", expr: \"error.step_id\" }\n }\n }\n })\n})\n```\n"; -export declare const SDK_TYPESCRIPT = "# TypeScript SDK (windmill-client)\n\nImport: import * as wmill from 'windmill-client'\n\nworkerHasInternalServer(): boolean\n\n/**\n * Initialize the Windmill client with authentication token and base URL\n * @param token - Authentication token (defaults to WM_TOKEN env variable)\n * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable)\n */\nsetClient(token?: string, baseUrl?: string): void\n\n/**\n * Create a client configuration from env variables\n * @returns client configuration\n */\ngetWorkspace(): string\n\n/**\n * Get a resource value by path\n * @param path path of the resource, default to internal state path\n * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error\n * @returns resource value\n */\nasync getResource(path?: string, undefinedIfEmpty?: boolean): Promise\n\n/**\n * Get the true root job id\n * @param jobId job id to get the root job id from (default to current job)\n * @returns root job id\n */\nasync getRootJobId(jobId?: string): Promise\n\n/**\n * @deprecated Use runScriptByPath or runScriptByHash instead\n */\nasync runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Run a script synchronously by its path and wait for the result\n * @param path - Script path in Windmill\n * @param args - Arguments to pass to the script\n * @param verbose - Enable verbose logging\n * @param tag - Override the worker tag the job runs on\n * @returns Script execution result\n */\nasync runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Run a script synchronously by its hash and wait for the result\n * @param hash_ - Script hash in Windmill\n * @param args - Arguments to pass to the script\n * @param verbose - Enable verbose logging\n * @param tag - Override the worker tag the job runs on\n * @returns Script execution result\n */\nasync runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Append a text to the result stream\n * @param text text to append to the result stream\n */\nappendToResultStream(text: string): void\n\n/**\n * Stream to the result stream\n * @param stream stream to stream to the result stream\n */\nasync streamResult(stream: AsyncIterable): Promise\n\n/**\n * Run a flow synchronously by its path and wait for the result\n * @param path - Flow path in Windmill\n * @param args - Arguments to pass to the flow\n * @param verbose - Enable verbose logging\n * @param tag - Override the worker tag the job runs on\n * @returns Flow execution result\n */\nasync runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise\n\n/**\n * Wait for a job to complete and return its result\n * @param jobId - ID of the job to wait for\n * @param verbose - Enable verbose logging\n * @returns Job result when completed\n */\nasync waitJob(jobId: string, verbose: boolean = false): Promise\n\n/**\n * Get the result of a completed job\n * @param jobId - ID of the completed job\n * @returns Job result\n */\nasync getResult(jobId: string): Promise\n\n/**\n * Get the result of a job if completed, or its current status\n * @param jobId - ID of the job\n * @returns Object with started, completed, success, and result properties\n */\nasync getResultMaybe(jobId: string): Promise\n\n/**\n * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead\n */\nasync runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise\n\n/**\n * Run a script asynchronously by its path\n * @param path - Script path in Windmill\n * @param args - Arguments to pass to the script\n * @param scheduledInSeconds - Schedule execution for a future time (in seconds)\n * @param tag - Override the worker tag the job runs on\n * @returns Job ID of the created job\n */\nasync runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise\n\n/**\n * Run a script asynchronously by its hash\n * @param hash_ - Script hash in Windmill\n * @param args - Arguments to pass to the script\n * @param scheduledInSeconds - Schedule execution for a future time (in seconds)\n * @param tag - Override the worker tag the job runs on\n * @returns Job ID of the created job\n */\nasync runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise\n\n/**\n * Run a flow asynchronously by its path\n * @param path - Flow path in Windmill\n * @param args - Arguments to pass to the flow\n * @param scheduledInSeconds - Schedule execution for a future time (in seconds)\n * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job)\n * @param tag - Override the worker tag the job runs on\n * @returns Job ID of the created job\n */\nasync runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true, tag: string | null = null): Promise\n\n/**\n * Resolve a resource value in case the default value was picked because the input payload was undefined\n * @param obj resource value or path of the resource under the format `$res:path`\n * @returns resource value\n */\nasync resolveDefaultResource(obj: any): Promise\n\n/**\n * Get the state file path from environment variables\n * @returns State path string\n */\ngetStatePath(): string\n\n/**\n * Set a resource value by path\n * @param path path of the resource to set, default to state path\n * @param value new value of the resource to set\n * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type\n */\nasync setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise\n\n/**\n * Set the state\n * @param state state to set\n * @deprecated use setState instead\n */\nasync setInternalState(state: any): Promise\n\n/**\n * Set the state\n * @param state state to set\n * @param path Optional state resource path override. Defaults to `getStatePath()`.\n */\nasync setState(state: any, path?: string): Promise\n\n/**\n * Set the progress\n * Progress cannot go back and limited to 0% to 99% range\n * @param percent Progress to set in %\n * @param jobId? Job to set progress for\n */\nasync setProgress(percent: number, jobId?: any): Promise\n\n/**\n * Get the progress\n * @param jobId? Job to get progress from\n * @returns Optional clamped between 0 and 100 progress value\n */\nasync getProgress(jobId?: any): Promise\n\n/**\n * Set a flow user state\n * @param key key of the state\n * @param value value of the state\n */\nasync setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise\n\n/**\n * Get a flow user state\n * @param path path of the variable\n */\nasync getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise\n\n/**\n * Get the internal state\n * @deprecated use getState instead\n */\nasync getInternalState(): Promise\n\n/**\n * Get the state shared across executions\n * @param path Optional state resource path override. Defaults to `getStatePath()`.\n */\nasync getState(path?: string): Promise\n\n/**\n * Get a variable by path\n * @param path path of the variable\n * @returns variable value\n */\nasync getVariable(path: string): Promise\n\n/**\n * Set a variable by path, create if not exist\n * @param path path of the variable\n * @param value value of the variable\n * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false)\n * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: \"\")\n */\nasync setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise\n\n/**\n * Build a PostgreSQL connection URL from a database resource\n * @param path - Path to the database resource\n * @returns PostgreSQL connection URL string\n */\nasync databaseUrlFromResource(path: string): Promise\n\nasync polarsConnectionSettings(s3_resource_path: string | undefined): Promise\n\nasync duckdbConnectionSettings(s3_resource_path: string | undefined): Promise\n\n/**\n * Get S3 client settings from a resource or workspace default\n * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined)\n * @param workspace - Workspace to read from (defaults to the `WM_WORKSPACE` env var)\n * @returns S3 client configuration settings\n */\nasync denoS3LightClientSettings(s3_resource_path: string | undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.\n * \n * ```typescript\n * let fileContent = await wmill.loadS3FileContent(inputFile)\n * // if the file is a raw text file, it can be decoded and printed directly:\n * const text = new TextDecoder().decode(fileContentStream)\n * console.log(text);\n * ```\n * \n * @param workspace - Workspace to read from (defaults to the `WM_WORKSPACE` env var)\n */\nasync loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.\n * \n * ```typescript\n * let fileContentBlob = await wmill.loadS3FileStream(inputFile)\n * // if the content is plain text, the blob can be read directly:\n * console.log(await fileContentBlob.text());\n * ```\n * \n * @param workspace - Workspace to read from (defaults to the `WM_WORKSPACE` env var)\n */\nasync loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.\n * \n * ```typescript\n * const s3object = await writeS3File(s3Object, \"Hello Windmill!\")\n * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8')\n * console.log(fileContentAsUtf8Str)\n * ```\n * \n * @param workspace - Workspace to write to (defaults to the `WM_WORKSPACE` env var)\n */\nasync writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined, workspace: string | undefined = undefined): Promise\n\n/**\n * Permanently delete a file from S3 by key.\n * \n * ```typescript\n * await wmill.deleteS3File({ s3: \"path/to/file.txt\" })\n * ```\n * \n * @param s3object - S3 object identifying the file to delete (must have `s3` set)\n * @param workspace - Workspace to delete from (defaults to the `WM_WORKSPACE` env var)\n */\nasync deleteS3File(s3object: S3Object, workspace: string | undefined = undefined): Promise\n\n/**\n * Sign S3 objects to be used by anonymous users in public apps\n * @param s3objects s3 objects to sign\n * @returns signed s3 objects\n */\nasync signS3Objects(s3objects: S3Object[]): Promise\n\n/**\n * Sign S3 object to be used by anonymous users in public apps\n * @param s3object s3 object to sign\n * @returns signed s3 object\n */\nasync signS3Object(s3object: S3Object): Promise\n\n/**\n * Generate a presigned public URL for an array of S3 objects.\n * If an S3 object is not signed yet, it will be signed first.\n * @param s3Objects s3 objects to sign\n * @returns list of signed public URLs\n */\nasync getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise\n\n/**\n * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first.\n * @param s3Object s3 object to sign\n * @returns signed public URL\n */\nasync getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise\n\n/**\n * Get URLs needed for resuming a flow after this step\n * @param approver approver name\n * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step.\n * This allows pre-approvals that can be consumed by any later suspend step in the same flow.\n * @returns approval page UI URL, resume and cancel API URLs for resuming the flow\n */\nasync getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{\n approvalPage: string;\n resume: string;\n cancel: string;\n}>\n\n/**\n * @deprecated use getResumeUrls instead\n */\ngetResumeEndpoints(approver?: string): Promise<{\n approvalPage: string;\n resume: string;\n cancel: string;\n}>\n\n/**\n * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only)\n * @param audience audience of the token\n * @param expiresIn Optional number of seconds until the token expires\n * @returns jwt token\n */\nasync getIdToken(audience: string, expiresIn?: number): Promise\n\n/**\n * Convert a base64-encoded string to Uint8Array\n * @param data - Base64-encoded string\n * @returns Decoded Uint8Array\n */\nbase64ToUint8Array(data: string): Uint8Array\n\n/**\n * Convert a Uint8Array to base64-encoded string\n * @param arrayBuffer - Uint8Array to encode\n * @returns Base64-encoded string\n */\nuint8ArrayToBase64(arrayBuffer: Uint8Array): string\n\n/**\n * Get email from workspace username\n * This method is particularly useful for apps that require the email address of the viewer.\n * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app.\n * @param username\n * @returns email address\n */\nasync usernameToEmail(username: string): Promise\n\n/**\n * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields.\n * \n * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form**\n * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form).\n * \n * @param {Object} options - The configuration options for the Slack approval request.\n * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill.\n * @param {string} options.channelId - The Slack channel ID where the approval request will be sent.\n * @param {string} [options.message] - Optional custom message to include in the Slack approval request.\n * @param {string} [options.approver] - Optional user ID or name of the approver for the request.\n * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field.\n * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field.\n * @param {string} [options.resumeButtonText] - Optional text for the resume button.\n * @param {string} [options.cancelButtonText] - Optional text for the cancel button.\n * \n * @returns {Promise} Resolves when the Slack approval request is successfully sent.\n * \n * @throws {Error} If the function is not called within a flow or flow preview.\n * @throws {Error} If the `JobService.getSlackApprovalPayload` call fails.\n * \n * **Usage Example:**\n * ```typescript\n * await requestInteractiveSlackApproval({\n * slackResourcePath: \"/u/alex/my_slack_resource\",\n * channelId: \"admins-slack-channel\",\n * message: \"Please approve this request\",\n * approver: \"approver123\",\n * defaultArgsJson: { key1: \"value1\", key2: 42 },\n * dynamicEnumsJson: { foo: [\"choice1\", \"choice2\"], bar: [\"optionA\", \"optionB\"] },\n * resumeButtonText: \"Resume\",\n * cancelButtonText: \"Cancel\",\n * });\n * ```\n * \n * **Note:** This function requires execution within a Windmill flow or flow preview.\n */\nasync requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, resumeButtonText, cancelButtonText, }: SlackApprovalOptions): Promise\n\n/**\n * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields.\n * \n * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form**\n * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form).\n * \n * @param {Object} options - The configuration options for the Teams approval request.\n * @param {string} options.teamName - The Teams team name where the approval request will be sent.\n * @param {string} options.channelName - The Teams channel name where the approval request will be sent.\n * @param {string} [options.message] - Optional custom message to include in the Teams approval request.\n * @param {string} [options.approver] - Optional user ID or name of the approver for the request.\n * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field.\n * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field.\n * \n * @returns {Promise} Resolves when the Teams approval request is successfully sent.\n * \n * @throws {Error} If the function is not called within a flow or flow preview.\n * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails.\n * \n * **Usage Example:**\n * ```typescript\n * await requestInteractiveTeamsApproval({\n * teamName: \"admins-teams\",\n * channelName: \"admins-teams-channel\",\n * message: \"Please approve this request\",\n * approver: \"approver123\",\n * defaultArgsJson: { key1: \"value1\", key2: 42 },\n * dynamicEnumsJson: { foo: [\"choice1\", \"choice2\"], bar: [\"optionA\", \"optionB\"] },\n * });\n * ```\n * \n * **Note:** This function requires execution within a Windmill flow or flow preview.\n */\nasync requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise\n\nsetWorkflowCtx(ctx: WorkflowCtx | null): void\n\nasync sleep(seconds: number): Promise\n\nasync step(name: string, fn: () => T | Promise): Promise\n\n/**\n * Create a task that dispatches to a separate Windmill script.\n * \n * @example\n * const extract = taskScript(\"f/data/extract\");\n * // inside workflow: await extract({ url: \"https://...\" })\n */\ntaskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Create a task that dispatches to a separate Windmill flow.\n * \n * @example\n * const pipeline = taskFlow(\"f/etl/pipeline\");\n * // inside workflow: await pipeline({ input: data })\n */\ntaskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Mark an async function as a workflow-as-code entry point.\n * \n * The function must be **deterministic**: given the same inputs it must call\n * tasks in the same order on every replay. Branching on task results is fine\n * (results are replayed from checkpoint), but branching on external state\n * (current time, random values, external API calls) must use `step()` to\n * checkpoint the value so replays see the same result.\n */\nworkflow(fn: (...args: any[]) => Promise): void\n\n/**\n * Suspend the workflow and wait for an external approval.\n * \n * Use `getResumeUrls()` (wrapped in `step()`) to obtain resume/cancel/approvalPage\n * URLs before calling this function.\n * \n * @example\n * const urls = await step(\"urls\", () => getResumeUrls());\n * await step(\"notify\", () => sendEmail(urls.approvalPage));\n * const { value, approver } = await waitForApproval({ timeout: 3600 });\n */\nwaitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }>\n\n/**\n * Process items in parallel with optional concurrency control.\n * \n * Each item is processed by calling `fn(item)`, which should be a task().\n * Items are dispatched in batches of `concurrency` (default: all at once).\n * \n * @example\n * const process = task(async (item: string) => { ... });\n * const results = await parallel(items, process, { concurrency: 5 });\n */\nasync parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise\n\n/**\n * Commit Kafka offsets for a trigger with auto_commit disabled.\n * @param triggerPath - Path to the Kafka trigger (from event.wm_trigger.trigger_path)\n * @param topic - Kafka topic name (from event.topic)\n * @param partition - Partition number (from event.partition)\n * @param offset - Message offset to commit (from event.offset)\n */\nasync commitKafkaOffsets(triggerPath: string, topic: string, partition: number, offset: number,): Promise\n\n/**\n * Parse an S3 object from URI string or record format\n * @param s3Object - S3 object as URI string (`s3://storage/key`, `s3:///key`\n * for the default storage) or record. Any other string throws rather than\n * falling back to an auto-generated key: an auto key is requested by\n * omitting the object, and a fallback would silently misplace the upload\n * on any typo.\n * @returns S3 object record with storage and s3 key\n */\nparseS3Object(s3Object: S3Object): S3ObjectRecord\n\n/**\n * Create a SQL template function for PostgreSQL/datatable queries\n * @param name - Database/datatable name (default: \"main\")\n * @returns SQL template function for building parameterized queries\n * @example\n * let sql = wmill.datatable()\n * let name = 'Robin'\n * let age = 21\n * await sql`\n * SELECT * FROM friends\n * WHERE name = ${name} AND age = ${age}::int\n * `.fetch()\n */\ndatatable(name: string = \"main\"): DatatableSqlTemplateFunction\n\n/**\n * Create a SQL template function for DuckDB/ducklake queries\n * @param name - DuckDB database name, optionally with a schema as `name:schema` (default: \"main\")\n * @returns SQL template function for building parameterized queries\n * @example\n * let sql = wmill.ducklake()\n * let name = 'Robin'\n * let age = 21\n * await sql`\n * SELECT * FROM friends\n * WHERE name = ${name} AND age = ${age}\n * `.fetch()\n * @example\n * // Target a specific schema within the ducklake\n * let sql = wmill.ducklake(\"my_lake:analytics\")\n */\nducklake(name: string = \"main\"): SqlTemplateFunction\n\n/**\n * Idempotently materialize `selectSql` into a ducklake table for one\n * partition (or the whole table when `partition` is omitted) \u2014 the client-side\n * equivalent of the `// materialize` engine.\n * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it\n * replaces it (whole table \u2192 `CREATE OR REPLACE`; partition \u2192 delete + insert).\n * Safe to re-run for the same partition (backfill / failure-recovery).\n * \n * Returns a lazy statement \u2014 call `.execute()` to run it:\n * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.\n */\nupsertPartition(opts: DucklakeMaterializeOptions): SqlStatement\n\n/**\n * INSERT-only materialization (no dedup/replace) for append-only tables.\n * Re-running the same partition duplicates rows \u2014 use only for immutable\n * event-log sources.\n * \n * Returns a lazy statement \u2014 call `.execute()` to run it:\n * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.\n */\nappendPartition(opts: Omit,): SqlStatement\n"; -export declare const SDK_PYTHON = "# Python SDK (wmill)\n\nImport: import wmill\n\ndef worker_has_internal_server() -> bool\n\ndef get_mocked_api() -> Optional[dict]\n\n# Get the HTTP client instance.\n# \n# Returns:\n# Configured httpx.Client for API requests\ndef get_client() -> httpx.Client\n\n# Make an HTTP GET request to the Windmill API.\n# \n# Args:\n# endpoint: API endpoint path\n# raise_for_status: Whether to raise an exception on HTTP errors\n# **kwargs: Additional arguments passed to httpx.get\n# \n# Returns:\n# HTTP response object\ndef get(endpoint, raise_for_status = True, **kwargs) -> httpx.Response\n\n# Make an HTTP POST request to the Windmill API.\n# \n# Args:\n# endpoint: API endpoint path\n# raise_for_status: Whether to raise an exception on HTTP errors\n# **kwargs: Additional arguments passed to httpx.post\n# \n# Returns:\n# HTTP response object\ndef post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response\n\n# Create a new authentication token.\n# \n# Args:\n# duration: Token validity duration (default: 1 day)\n# \n# Returns:\n# New authentication token string\ndef create_token(duration = dt.timedelta(days=1)) -> str\n\n# Create a script job and return its job id.\n# \n# .. deprecated:: Use run_script_by_path_async or run_script_by_hash_async instead.\ndef run_script_async(path: str = None, hash_: str = None, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str\n\n# Create a script job by path and return its job id.\ndef run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str\n\n# Create a script job by hash and return its job id.\ndef run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str\n\n# Create a flow job and return its job id.\ndef run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str\n\n# Run script synchronously and return its result.\n# \n# .. deprecated:: Use run_script_by_path or run_script_by_hash instead.\ndef run_script(path: str = None, hash_: str = None, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any\n\n# Run script by path synchronously and return its result.\ndef run_script_by_path(path: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any\n\n# Run script by hash synchronously and return its result.\ndef run_script_by_hash(hash_: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any\n\n# Run a script on the current worker without creating a job.\n# \n# On agent workers (no internal server), falls back to running a normal\n# preview job and waiting for the result.\ndef run_inline_script_preview(content: str, language: str, args: dict = None) -> Any\n\n# Wait for a job to complete and return its result.\n# \n# Args:\n# job_id: ID of the job to wait for\n# timeout: Maximum time to wait (seconds or timedelta)\n# verbose: Enable verbose logging\n# cleanup: Register cleanup handler to cancel job on exit\n# assert_result_is_not_none: Raise exception if result is None\n# \n# Returns:\n# Job result when completed\n# \n# Raises:\n# TimeoutError: If timeout is reached\n# Exception: If job fails\ndef wait_job(job_id, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False)\n\n# Cancel a specific job by ID.\n# \n# Args:\n# job_id: UUID of the job to cancel\n# reason: Optional reason for cancellation\n# \n# Returns:\n# Response message from the cancel endpoint\ndef cancel_job(job_id: str, reason: str = None) -> str\n\n# Cancel currently running executions of the same script.\ndef cancel_running() -> dict\n\n# Get job details by ID.\n# \n# Args:\n# job_id: UUID of the job\n# \n# Returns:\n# Job details dictionary\ndef get_job(job_id: str) -> dict\n\n# Get the root job ID for a flow hierarchy.\n# \n# Args:\n# job_id: Job ID (defaults to current WM_JOB_ID)\n# \n# Returns:\n# Root job ID\ndef get_root_job_id(job_id: str | None = None) -> dict\n\n# Get an OIDC JWT token for authentication to external services.\n# \n# Args:\n# audience: Token audience (e.g., \"vault\", \"aws\")\n# expires_in: Optional expiration time in seconds\n# \n# Returns:\n# JWT token string\ndef get_id_token(audience: str, expires_in: int | None = None) -> str\n\n# Get the status of a job.\n# \n# Args:\n# job_id: UUID of the job\n# \n# Returns:\n# Job status: \"RUNNING\", \"WAITING\", or \"COMPLETED\"\ndef get_job_status(job_id: str) -> JobStatus\n\n# Get the result of a completed job.\n# \n# Args:\n# job_id: UUID of the completed job\n# assert_result_is_not_none: Raise exception if result is None\n# \n# Returns:\n# Job result\ndef get_result(job_id: str, assert_result_is_not_none: bool = True) -> Any\n\n# Get a variable value by path.\n# \n# Args:\n# path: Variable path in Windmill\n# \n# Returns:\n# Variable value as string\ndef get_variable(path: str) -> str\n\n# Set a variable value by path, creating it if it doesn't exist.\n# \n# Args:\n# path: Variable path in Windmill\n# value: Variable value to set\n# is_secret: Whether the variable should be secret (default: False)\ndef set_variable(path: str, value: str, is_secret: bool = False) -> None\n\n# Get a resource value by path.\n# \n# Args:\n# path: Resource path in Windmill\n# none_if_undefined: Return None instead of raising if not found\n# interpolated: if variables and resources are fully unrolled\n# \n# Returns:\n# Resource value dictionary or None\ndef get_resource(path: str, none_if_undefined: bool = False, interpolated: bool = True) -> dict | None\n\n# Set a resource value by path, creating it if it doesn't exist.\n# \n# Args:\n# value: Resource value to set\n# path: Resource path in Windmill\n# resource_type: Resource type for creation\ndef set_resource(value: Any, path: str, resource_type: str)\n\n# List resources from Windmill workspace.\n# \n# Args:\n# resource_type: Optional resource type to filter by (e.g., \"postgresql\", \"mysql\", \"s3\")\n# page: Optional page number for pagination\n# per_page: Optional number of results per page\n# \n# Returns:\n# List of resource dictionaries\ndef list_resources(resource_type: str = None, page: int = None, per_page: int = None) -> list[dict]\n\n# Set the workflow state.\n# \n# Args:\n# value: State value to set\n# path: Optional state resource path override.\ndef set_state(value: Any, path: str | None = None) -> None\n\n# Get the workflow state.\n# \n# Args:\n# path: Optional state resource path override.\n# \n# Returns:\n# State value or None if not set\ndef get_state(path: str | None = None) -> Any\n\n# Set job progress percentage (0-99).\n# \n# Args:\n# value: Progress percentage\n# job_id: Job ID (defaults to current WM_JOB_ID)\ndef set_progress(value: int, job_id: Optional[str] = None)\n\n# Get job progress percentage.\n# \n# Args:\n# job_id: Job ID (defaults to current WM_JOB_ID)\n# \n# Returns:\n# Progress value (0-100) or None if not set\ndef get_progress(job_id: Optional[str] = None) -> Any\n\n# Set the user state of a flow at a given key\ndef set_flow_user_state(key: str, value: Any) -> None\n\n# Get the user state of a flow at a given key\ndef get_flow_user_state(key: str) -> Any\n\n# Get the Windmill server version.\n# \n# Returns:\n# Version string\ndef version()\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from DuckDB\ndef get_duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings | None\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from Polars\ndef get_polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection using boto3\ndef get_boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings\n\n# Load a file from the workspace s3 bucket and returns its content as bytes.\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# my_obj_content = client.load_s3_file(s3_obj)\n# file_content = my_obj_content.decode(\"utf-8\")\n# '''\ndef load_s3_file(s3object: S3Object | str, s3_resource_path: str | None) -> bytes\n\n# Load a file from the workspace s3 bucket and returns the bytes stream.\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# with wmill.load_s3_file_reader(s3object, s3_resource_path) as file_reader:\n# print(file_reader.read())\n# '''\ndef load_s3_file_reader(s3object: S3Object | str, s3_resource_path: str | None) -> BufferedReader\n\n# Write a file to the workspace S3 bucket\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# \n# # for an in memory bytes array:\n# file_content = b'Hello Windmill!'\n# client.write_s3_file(s3_obj, file_content)\n# \n# # for a file:\n# with open(\"my_file.txt\", \"rb\") as my_file:\n# client.write_s3_file(s3_obj, my_file)\n# '''\ndef write_s3_file(s3object: S3Object | str | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, content_type: str | None = None, content_disposition: str | None = None) -> S3Object\n\n# Permanently delete a file from the workspace S3 bucket.\n# \n# '''python\n# from wmill import S3Object\n# \n# s3_obj = S3Object(s3=\"/path/to/my_file.txt\")\n# client.delete_s3_object(s3_obj)\n# '''\ndef delete_s3_object(s3object: S3Object | str, s3_resource_path: str | None = None) -> None\n\n# Sign S3 objects for use by anonymous users in public apps.\n# \n# Args:\n# s3_objects: List of S3 objects to sign\n# \n# Returns:\n# List of signed S3 objects\ndef sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object]\n\n# Sign a single S3 object for use by anonymous users in public apps.\n# \n# Args:\n# s3_object: S3 object to sign\n# \n# Returns:\n# Signed S3 object\ndef sign_s3_object(s3_object: S3Object | str) -> S3Object\n\n# Generate presigned public URLs for an array of S3 objects.\n# If an S3 object is not signed yet, it will be signed first.\n# \n# Args:\n# s3_objects: List of S3 objects to sign\n# base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL)\n# \n# Returns:\n# List of signed public URLs\n# \n# Example:\n# >>> s3_objs = [S3Object(s3=\"/path/to/file1.txt\"), S3Object(s3=\"/path/to/file2.txt\")]\n# >>> urls = client.get_presigned_s3_public_urls(s3_objs)\ndef get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None) -> list[str]\n\n# Generate a presigned public URL for an S3 object.\n# If the S3 object is not signed yet, it will be signed first.\n# \n# Args:\n# s3_object: S3 object to sign\n# base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL)\n# \n# Returns:\n# Signed public URL\n# \n# Example:\n# >>> s3_obj = S3Object(s3=\"/path/to/file.txt\")\n# >>> url = client.get_presigned_s3_public_url(s3_obj)\ndef get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None) -> str\n\n# Get the current user information.\n# \n# Returns:\n# User details dictionary\ndef whoami() -> dict\n\n# Get the current user information (alias for whoami).\n# \n# Returns:\n# User details dictionary\ndef user() -> dict\n\n# Get the state resource path from environment.\n# \n# Returns:\n# State path string\ndef state_path() -> str\n\n# Get the workflow state.\n# \n# Returns:\n# State value or None if not set\ndef state() -> Any\n\n# Set the state in the shared folder using pickle\ndef set_shared_state_pickle(value: Any, path: str = 'state.pickle') -> None\n\n# Get the state in the shared folder using pickle\ndef get_shared_state_pickle(path: str = 'state.pickle') -> Any\n\n# Set the state in the shared folder using pickle\ndef set_shared_state(value: Any, path: str = 'state.json') -> None\n\n# Get the state in the shared folder using pickle\ndef get_shared_state(path: str = 'state.json') -> None\n\n# Get URLs needed for resuming a flow after suspension.\n# \n# Args:\n# approver: Optional approver name\n# flow_level: If True, generate resume URLs for the parent flow instead of the\n# specific step. This allows pre-approvals that can be consumed by any later\n# suspend step in the same flow.\n# \n# Returns:\n# Dictionary with approvalPage, resume, and cancel URLs\ndef get_resume_urls(approver: str = None, flow_level: bool = None) -> dict\n\n# Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields.\n# \n# **[Enterprise Edition Only]** To include form fields in the Slack approval request, use the \"Advanced -> Suspend -> Form\" functionality.\n# Learn more at: https://www.windmill.dev/docs/flows/flow_approval#form\n# \n# :param slack_resource_path: The path to the Slack resource in Windmill.\n# :type slack_resource_path: str\n# :param channel_id: The Slack channel ID where the approval request will be sent.\n# :type channel_id: str\n# :param message: Optional custom message to include in the Slack approval request.\n# :type message: str, optional\n# :param approver: Optional user ID or name of the approver for the request.\n# :type approver: str, optional\n# :param default_args_json: Optional dictionary defining or overriding the default arguments for form fields.\n# :type default_args_json: dict, optional\n# :param dynamic_enums_json: Optional dictionary overriding the enum default values of enum form fields.\n# :type dynamic_enums_json: dict, optional\n# \n# :raises Exception: If the function is not called within a flow or flow preview.\n# :raises Exception: If the required flow job or flow step environment variables are not set.\n# \n# :return: None\n# \n# **Usage Example:**\n# >>> client.request_interactive_slack_approval(\n# ... slack_resource_path=\"/u/alex/my_slack_resource\",\n# ... channel_id=\"admins-slack-channel\",\n# ... message=\"Please approve this request\",\n# ... approver=\"approver123\",\n# ... default_args_json={\"key1\": \"value1\", \"key2\": 42},\n# ... dynamic_enums_json={\"foo\": [\"choice1\", \"choice2\"], \"bar\": [\"optionA\", \"optionB\"]},\n# ... )\n# \n# **Notes:**\n# - This function must be executed within a Windmill flow or flow preview.\n# - The function checks for required environment variables (`WM_FLOW_JOB_ID`, `WM_FLOW_STEP_ID`) to ensure it is run in the appropriate context.\ndef request_interactive_slack_approval(slack_resource_path: str, channel_id: str, message: str = None, approver: str = None, default_args_json: dict = None, dynamic_enums_json: dict = None) -> None\n\n# Get email from workspace username\n# This method is particularly useful for apps that require the email address of the viewer.\n# Indeed, in the viewer context WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app.\ndef username_to_email(username: str) -> str\n\n# Send a message to a Microsoft Teams conversation with conversation_id, where success is used to style the message\ndef send_teams_message(conversation_id: str, text: str, success: bool = True, card_block: dict = None)\n\n# Get a DataTable client for SQL queries.\n# \n# Args:\n# name: Database name (default: \"main\")\n# \n# Returns:\n# DataTableClient instance\ndef datatable(name: str = 'main')\n\n# Get a DuckLake client for DuckDB queries.\n# \n# Args:\n# name: Database name (default: \"main\")\n# \n# Returns:\n# DucklakeClient instance\ndef ducklake(name: str = 'main')\n\ndef init_global_client(f)\n\ndef deprecate(in_favor_of: str)\n\n# Get the current workspace ID.\n# \n# Returns:\n# Workspace ID string\ndef get_workspace() -> str\n\ndef get_version() -> str\n\n# Run a script synchronously by hash and return its result.\n# \n# Args:\n# hash: Script hash\n# args: Script arguments\n# verbose: Enable verbose logging\n# assert_result_is_not_none: Raise exception if result is None\n# cleanup: Register cleanup handler to cancel job on exit\n# timeout: Maximum time to wait\n# tag: Override the worker tag the job runs on\n# \n# Returns:\n# Script result\ndef run_script_sync(hash: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None, tag: str = None) -> Any\n\n# Run a script synchronously by path and return its result.\n# \n# Args:\n# path: Script path\n# args: Script arguments\n# verbose: Enable verbose logging\n# assert_result_is_not_none: Raise exception if result is None\n# cleanup: Register cleanup handler to cancel job on exit\n# timeout: Maximum time to wait\n# tag: Override the worker tag the job runs on\n# \n# Returns:\n# Script result\ndef run_script_by_path_sync(path: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None, tag: str = None) -> Any\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from DuckDB\ndef duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection from Polars\ndef polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings\n\n# Convenient helpers that takes an S3 resource as input and returns the settings necessary to\n# initiate an S3 connection using boto3\ndef boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings\n\n# Get the state resource path from environment.\n# \n# Returns:\n# State path string\ndef get_state_path() -> str\n\n# Parse resource syntax from string.\ndef parse_resource_syntax(s: str) -> Optional[str]\n\n# Parse S3 object from a `s3:///` URI string (`s3:///`\n# for the default storage) or S3Object format. Any other string raises\n# rather than falling back to an auto-generated key: an auto key is\n# requested by omitting the object, and a fallback would silently misplace\n# the upload on any typo.\ndef parse_s3_object(s3_object: S3Object | str) -> S3Object\n\n# Parse variable syntax from string.\ndef parse_variable_syntax(s: str) -> Optional[str]\n\n# Append a text to the result stream.\n# \n# Args:\n# text: text to append to the result stream\ndef append_to_result_stream(text: str) -> None\n\n# Stream to the result stream.\n# \n# Args:\n# stream: stream to stream to the result stream\ndef stream_result(stream) -> None\n\n# Execute a SQL query against the DataTable.\n# \n# Args:\n# sql: SQL query string with $1, $2, etc. placeholders\n# *args: Positional arguments to bind to query placeholders\n# \n# Returns:\n# SqlQuery instance for fetching results\ndef query(sql: str, *args) -> SqlQuery\n\n# Idempotently materialize the rows of `select_sql` into ducklake\n# `table` for one `partition` (or the whole table when `partition` is\n# None). Client-side equivalent of the `// materialize` engine: with\n# `unique_key` it upserts within the slice (delete-by-key + insert);\n# without it, it replaces (whole table \u2192 CREATE OR REPLACE; partition \u2192\n# delete the partition + insert). Re-running the same slice is safe \u2014 the\n# backfill / failure-recovery contract.\n# \n# The partition value is bound as a DuckDB arg (never string-interpolated)\n# so it cannot inject SQL. `select_sql` is trusted (your own query).\ndef upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)\n\n# INSERT-only materialization (no dedup / no replace) for an immutable\n# event-log table \u2014 for one `partition`, or the whole table when\n# `partition` is None. NOTE: unlike `upsert_partition`, re-running the same\n# slice duplicates rows \u2014 use only for append-only sources.\ndef append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)\n\n# Read a materialized ducklake table, optionally a single partition.\ndef read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)\n\n# Execute query and fetch results.\n# \n# Args:\n# result_collection: Optional result collection mode\n# \n# Returns:\n# Query results\ndef fetch(result_collection: str | None = None)\n\n# Execute query and fetch first row of results.\n# \n# Returns:\n# First row of query results\ndef fetch_one()\n\n# Execute query and fetch first row of results. Return result as a scalar value.\n# \n# Returns:\n# First row of query result as a scalar value\ndef fetch_one_scalar()\n\n# Execute query and don't return any results.\n# \ndef execute()\n\n# DuckDB executor requires explicit argument types at declaration\n# These types exist in both DuckDB and Postgres\n# Check that the types exist if you plan to extend this function for other SQL engines.\ndef infer_sql_type(value) -> str\n\ndef parse_sql_client_name(name: str) -> tuple[str, Optional[str]]\n\n# Decorator that marks a function as a workflow task.\n# \n# Works in both WAC v1 (sync, HTTP-based dispatch) and WAC v2\n# (async, checkpoint/replay) modes:\n# \n# - **v2 (inside @workflow)**: dispatches as a checkpoint step.\n# - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API.\n# - **Standalone**: executes the function body directly.\n# \n# Usage::\n# \n# @task\n# async def extract_data(url: str): ...\n# \n# @task(path=\"f/external_script\", timeout=600, tag=\"gpu\")\n# async def run_external(x: int): ...\ndef task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill script.\n# \n# Usage::\n# \n# extract = task_script(\"f/data/extract\", timeout=600)\n# \n# @workflow\n# async def main():\n# data = await extract(url=\"https://...\")\ndef task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill flow.\n# \n# Usage::\n# \n# pipeline = task_flow(\"f/etl/pipeline\", priority=10)\n# \n# @workflow\n# async def main():\n# result = await pipeline(input=data)\ndef task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Decorator marking an async function as a workflow-as-code entry point.\n# \n# The function must be **deterministic**: given the same inputs it must call\n# tasks in the same order on every replay. Branching on task results is fine\n# (results are replayed from checkpoint), but branching on external state\n# (current time, random values, external API calls) must use ``step()`` to\n# checkpoint the value so replays see the same result.\ndef workflow(func)\n\n# Execute ``fn`` inline and checkpoint the result.\n# \n# On replay the cached value is returned without re-executing ``fn``.\n# Use for lightweight deterministic operations (timestamps, random IDs,\n# config reads) that should not incur the overhead of a child job.\nasync def step(name: str, fn)\n\n# Server-side sleep \u2014 suspend the workflow for the given duration without holding a worker.\n# \n# Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``.\n# Outside a workflow, falls back to ``asyncio.sleep``.\nasync def sleep(seconds: int)\n\n# Suspend the workflow and wait for an external approval.\n# \n# Use ``get_resume_urls()`` (wrapped in ``step()``) to obtain\n# resume/cancel/approval URLs before calling this function.\n# \n# Returns a dict with ``value`` (form data), ``approver``, and ``approved``.\n# \n# Args:\n# timeout: Approval timeout in seconds (default 1800).\n# form: Optional form schema for the approval page.\n# self_approval: Whether the user who triggered the flow can approve it (default True).\n# \n# Example::\n# \n# urls = await step(\"urls\", lambda: get_resume_urls())\n# await step(\"notify\", lambda: send_email(urls[\"approvalPage\"]))\n# result = await wait_for_approval(timeout=3600)\nasync def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict\n\n# Process items in parallel with optional concurrency control.\n# \n# Each item is processed by calling ``fn(item)``, which should be a @task.\n# Items are dispatched in batches of ``concurrency`` (default: all at once).\n# \n# Example::\n# \n# @task\n# async def process(item: str):\n# ...\n# \n# results = await parallel(items, process, concurrency=5)\nasync def parallel(items, fn, concurrency: Optional[int] = None)\n\n# Commit Kafka offsets for a trigger with auto_commit disabled.\n# \n# Args:\n# trigger_path: Path to the Kafka trigger (from event['wm_trigger']['trigger_path'])\n# topic: Kafka topic name (from event['topic'])\n# partition: Partition number (from event['partition'])\n# offset: Message offset to commit (from event['offset'])\ndef commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None\n\n"; -export declare const WAC_SDK_TYPESCRIPT = "## TypeScript Workflow-as-Code API (windmill-client)\n\nImport: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getResumeUrls, parallel } from \"windmill-client\"`\n\n```typescript\nexport interface TaskOptions {\n timeout?: number;\n tag?: string;\n cache_ttl?: number;\n priority?: number;\n concurrency_limit?: number;\n concurrency_key?: string;\n concurrency_time_window_s?: number;\n}\n\n/**\n * Get URLs needed for resuming a flow after this step\n * @param approver approver name\n * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step.\n * This allows pre-approvals that can be consumed by any later suspend step in the same flow.\n * @returns approval page UI URL, resume and cancel API URLs for resuming the flow\n */\nexport async function getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ approvalPage: string; resume: string; cancel: string; }>\n\n/**\n * Wrap an async function as a workflow task.\n *\n * @example\n * const extract_data = task(async (url: string) => { ... });\n * const run_external = task(\"f/external_script\", async (x: number) => { ... });\n *\n * Inside a `workflow()`, calling a task dispatches it as a step.\n * Outside a workflow, the function body executes directly.\n */\nexport function task Promise>(fnOrPath: T | string, maybeFnOrOptions?: T | TaskOptions, maybeOptions?: TaskOptions,): T\n\n/**\n * Create a task that dispatches to a separate Windmill script.\n *\n * @example\n * const extract = taskScript(\"f/data/extract\");\n * // inside workflow: await extract({ url: \"https://...\" })\n */\nexport function taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Create a task that dispatches to a separate Windmill flow.\n *\n * @example\n * const pipeline = taskFlow(\"f/etl/pipeline\");\n * // inside workflow: await pipeline({ input: data })\n */\nexport function taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike\n\n/**\n * Mark an async function as a workflow-as-code entry point.\n *\n * The function must be **deterministic**: given the same inputs it must call\n * tasks in the same order on every replay. Branching on task results is fine\n * (results are replayed from checkpoint), but branching on external state\n * (current time, random values, external API calls) must use `step()` to\n * checkpoint the value so replays see the same result.\n */\nexport function workflow(fn: (...args: any[]) => Promise)\n\nexport async function step(name: string, fn: () => T | Promise): Promise\n\nexport async function sleep(seconds: number): Promise\n\n/**\n * Suspend the workflow and wait for an external approval.\n *\n * Use `getResumeUrls()` (wrapped in `step()`) to obtain resume/cancel/approvalPage\n * URLs before calling this function.\n *\n * @example\n * const urls = await step(\"urls\", () => getResumeUrls());\n * await step(\"notify\", () => sendEmail(urls.approvalPage));\n * const { value, approver } = await waitForApproval({ timeout: 3600 });\n */\nexport function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }>\n\n/**\n * Process items in parallel with optional concurrency control.\n *\n * Each item is processed by calling `fn(item)`, which should be a task().\n * Items are dispatched in batches of `concurrency` (default: all at once).\n *\n * @example\n * const process = task(async (item: string) => { ... });\n * const results = await parallel(items, process, { concurrency: 5 });\n */\nexport async function parallel(items: T[], fn: (item: T) => PromiseLike | R, options?: { concurrency?: number },): Promise\n```\n"; -export declare const WAC_SDK_PYTHON = "## Python Workflow-as-Code API (wmill)\n\nImport: `from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_resume_urls, parallel, TaskError`\n\n```python\n# Raised when a WAC task step failed.\n#\n# Attributes:\n# step_key: The checkpoint key of the failed step.\n# child_job_id: The UUID of the failed child job.\n# result: The error result from the child job.\nclass TaskError(Exception):\n def __init__(self, message: str, *, step_key: str = '', child_job_id: str = '', result = None)\n\n# Get URLs needed for resuming a flow after suspension.\n#\n# Args:\n# approver: Optional approver name\n# flow_level: If True, generate resume URLs for the parent flow instead of the\n# specific step. This allows pre-approvals that can be consumed by any later\n# suspend step in the same flow.\n#\n# Returns:\n# Dictionary with approvalPage, resume, and cancel URLs\ndef get_resume_urls(approver: str = None, flow_level: bool = None) -> dict\n\n# Decorator that marks a function as a workflow task.\n#\n# Works in both WAC v1 (sync, HTTP-based dispatch) and WAC v2\n# (async, checkpoint/replay) modes:\n#\n# - **v2 (inside @workflow)**: dispatches as a checkpoint step.\n# - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API.\n# - **Standalone**: executes the function body directly.\n#\n# Usage::\n#\n# @task\n# async def extract_data(url: str): ...\n#\n# @task(path=\"f/external_script\", timeout=600, tag=\"gpu\")\n# async def run_external(x: int): ...\ndef task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill script.\n#\n# Usage::\n#\n# extract = task_script(\"f/data/extract\", timeout=600)\n#\n# @workflow\n# async def main():\n# data = await extract(url=\"https://...\")\ndef task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Create a task that dispatches to a separate Windmill flow.\n#\n# Usage::\n#\n# pipeline = task_flow(\"f/etl/pipeline\", priority=10)\n#\n# @workflow\n# async def main():\n# result = await pipeline(input=data)\ndef task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None)\n\n# Decorator marking an async function as a workflow-as-code entry point.\n#\n# The function must be **deterministic**: given the same inputs it must call\n# tasks in the same order on every replay. Branching on task results is fine\n# (results are replayed from checkpoint), but branching on external state\n# (current time, random values, external API calls) must use ``step()`` to\n# checkpoint the value so replays see the same result.\ndef workflow(func)\n\n# Execute ``fn`` inline and checkpoint the result.\n#\n# On replay the cached value is returned without re-executing ``fn``.\n# Use for lightweight deterministic operations (timestamps, random IDs,\n# config reads) that should not incur the overhead of a child job.\nasync def step(name: str, fn)\n\n# Server-side sleep \u2014 suspend the workflow for the given duration without holding a worker.\n#\n# Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``.\n# Outside a workflow, falls back to ``asyncio.sleep``.\nasync def sleep(seconds: int)\n\n# Suspend the workflow and wait for an external approval.\n#\n# Use ``get_resume_urls()`` (wrapped in ``step()``) to obtain\n# resume/cancel/approval URLs before calling this function.\n#\n# Returns a dict with ``value`` (form data), ``approver``, and ``approved``.\n#\n# Args:\n# timeout: Approval timeout in seconds (default 1800).\n# form: Optional form schema for the approval page.\n# self_approval: Whether the user who triggered the flow can approve it (default True).\n#\n# Example::\n#\n# urls = await step(\"urls\", lambda: get_resume_urls())\n# await step(\"notify\", lambda: send_email(urls[\"approvalPage\"]))\n# result = await wait_for_approval(timeout=3600)\nasync def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict\n\n# Process items in parallel with optional concurrency control.\n#\n# Each item is processed by calling ``fn(item)``, which should be a @task.\n# Items are dispatched in batches of ``concurrency`` (default: all at once).\n#\n# Example::\n#\n# @task\n# async def process(item: str):\n# ...\n#\n# results = await parallel(items, process, concurrency=5)\nasync def parallel(items, fn, *, concurrency: Optional[int] = None)\n```\n"; -export declare const DATATABLE_SDK_TYPESCRIPT = "## TypeScript Datatable API (windmill-client)\n\nImport: `import * as wmill from 'windmill-client'`\n\nSQL statement object with query content, arguments, and execution methods\n```typescript\ntype SqlStatement = {\n /** Raw SQL content with formatted arguments */\n content: string;\n\n /** Argument values keyed by parameter name */\n args: Record;\n\n /**\n * Execute the SQL query and return results\n * @param params - Optional parameters including result collection mode\n * @returns Query results based on the result collection mode\n */\n fetch(\n params?: FetchParams // The union is for auto-completion\n ): Promise>;\n\n /**\n * Execute the SQL query and return only the first row\n * @param params - Optional parameters\n * @returns First row of the query result\n */\n fetchOne(\n params?: Omit, \"resultCollection\">\n ): Promise>;\n\n /**\n * Execute the SQL query and return only the first row as a scalar value\n * @param params - Optional parameters\n * @returns First row of the query result\n */\n fetchOneScalar(\n params?: Omit<\n FetchParams<\"last_statement_first_row_scalar\">,\n \"resultCollection\"\n >\n ): Promise>;\n\n /**\n * Execute the SQL query without fetching rows\n * @param params - Optional parameters\n */\n execute(\n params?: Omit, \"resultCollection\">\n ): Promise;\n};\n```\n\n```typescript\n// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch()\ninterface DatatableSqlTemplateFunction {\n // Tagged template usage:\n (strings: TemplateStringsArray, ...values: any[]): SqlStatement;\n query(sql: string, ...params: any[]): SqlStatement;\n};\n```\n\nCreate a SQL template function for PostgreSQL/datatable queries\n@param name - Database/datatable name (default: \"main\")\n@returns SQL template function for building parameterized queries\n@example\nlet sql = wmill.datatable()\nlet name = 'Robin'\nlet age = 21\nawait sql`\n SELECT * FROM friends\n WHERE name = ${name} AND age = ${age}::int\n`.fetch()\n```typescript\nfunction datatable(name: string = \"main\"): DatatableSqlTemplateFunction\n```\n"; -export declare const DATATABLE_SDK_PYTHON = "## Python Datatable API (wmill)\n\nImport: `import wmill`\n\n# Get a DataTable client for SQL queries.\n# \n# Args:\n# name: Database name (default: \"main\")\n# \n# Returns:\n# DataTableClient instance\ndef datatable(name: str = 'main') -> DataTableClient\n\n# Client for executing SQL queries against Windmill DataTables.\nclass DataTableClient:\n # Initialize DataTableClient.\n # \n # Args:\n # client: Windmill client instance\n # name: DataTable name\n def __init__(client: Windmill, name: str)\n\n # Execute a SQL query against the DataTable.\n # \n # Args:\n # sql: SQL query string with $1, $2, etc. placeholders\n # *args: Positional arguments to bind to query placeholders\n # \n # Returns:\n # SqlQuery instance for fetching results\n def query(sql: str, *args) -> SqlQuery\n\n\n# Query result handler for DataTable and DuckLake queries.\nclass SqlQuery:\n # Initialize SqlQuery.\n # \n # Args:\n # sql: SQL query string\n # fetch_fn: Function to execute the query\n def __init__(sql: str, fetch_fn)\n\n # Execute query and fetch results.\n # \n # Args:\n # result_collection: Optional result collection mode\n # \n # Returns:\n # Query results\n def fetch(result_collection: str | None = None)\n\n # Execute query and fetch first row of results.\n # \n # Returns:\n # First row of query results\n def fetch_one()\n\n # Execute query and fetch first row of results. Return result as a scalar value.\n # \n # Returns:\n # First row of query result as a scalar value\n def fetch_one_scalar()\n\n # Execute query and don't return any results.\n # \n def execute()\n\n\n"; -export declare const OPENFLOW_SCHEMA = "## OpenFlow Schema\n\n{\"OpenFlow\":{\"type\":\"object\",\"description\":\"Top-level flow definition containing metadata, configuration, and the flow structure\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Short description of what this flow does\"},\"description\":{\"type\":\"string\",\"description\":\"Detailed documentation for this flow\"},\"value\":{\"$ref\":\"#/components/schemas/FlowValue\"},\"schema\":{\"type\":\"object\",\"description\":\"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')\"},\"on_behalf_of_email\":{\"type\":\"string\",\"description\":\"The flow will be run with the permissions of the user with this email.\"}},\"required\":[\"summary\",\"value\"]},\"FlowValue\":{\"type\":\"object\",\"description\":\"The flow structure containing modules and optional preprocessor/failure handlers\",\"properties\":{\"modules\":{\"type\":\"array\",\"description\":\"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"failure_module\":{\"description\":\"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types\",\"$ref\":\"#/components/schemas/FlowModule\"},\"preprocessor_module\":{\"description\":\"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results\",\"$ref\":\"#/components/schemas/FlowModule\"},\"same_worker\":{\"type\":\"boolean\",\"description\":\"If true, all steps run on the same worker for better performance\"},\"preserve_step_tags\":{\"type\":\"boolean\",\"description\":\"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag.\"},\"concurrent_limit\":{\"type\":\"number\",\"description\":\"Maximum number of concurrent executions of this flow\"},\"concurrency_key\":{\"type\":\"string\",\"description\":\"Expression to group concurrent executions (e.g., by user ID)\"},\"concurrency_time_window_s\":{\"type\":\"number\",\"description\":\"Time window in seconds for concurrent_limit\"},\"debounce_delay_s\":{\"type\":\"integer\",\"description\":\"Delay in seconds to debounce flow executions\"},\"debounce_key\":{\"type\":\"string\",\"description\":\"Expression to group debounced executions\"},\"debounce_args_to_accumulate\":{\"type\":\"array\",\"description\":\"Arguments to accumulate across debounced executions\",\"items\":{\"type\":\"string\"}},\"max_total_debouncing_time\":{\"type\":\"integer\",\"description\":\"Maximum total time in seconds that a job can be debounced\"},\"max_total_debounces_amount\":{\"type\":\"integer\",\"description\":\"Maximum number of times a job can be debounced\"},\"skip_expr\":{\"type\":\"string\",\"description\":\"JavaScript expression to conditionally skip the entire flow\"},\"cache_ttl\":{\"type\":\"number\",\"description\":\"Cache duration in seconds for flow results\"},\"cache_ignore_s3_path\":{\"type\":\"boolean\"},\"delete_after_secs\":{\"type\":\"integer\",\"description\":\"If set, delete the flow job's args, result and logs after this many seconds following job completion\"},\"flow_env\":{\"type\":\"object\",\"description\":\"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).\",\"additionalProperties\":{}},\"priority\":{\"type\":\"number\",\"description\":\"Execution priority (higher numbers run first)\"},\"early_return\":{\"type\":\"string\",\"description\":\"JavaScript expression to return early from the flow\"},\"chat_input_enabled\":{\"type\":\"boolean\",\"description\":\"Whether this flow accepts chat-style input\"},\"notes\":{\"type\":\"array\",\"description\":\"Sticky notes attached to the flow\",\"items\":{\"$ref\":\"#/components/schemas/FlowNote\"}},\"groups\":{\"type\":\"array\",\"description\":\"Semantic groups of modules for organizational purposes\",\"items\":{\"$ref\":\"#/components/schemas/FlowGroup\"}}},\"required\":[\"modules\"]},\"Retry\":{\"type\":\"object\",\"description\":\"Retry configuration for failed module executions\",\"properties\":{\"constant\":{\"type\":\"object\",\"description\":\"Retry with constant delay between attempts\",\"properties\":{\"attempts\":{\"type\":\"integer\",\"description\":\"Number of retry attempts\"},\"seconds\":{\"type\":\"integer\",\"description\":\"Seconds to wait between retries\"}}},\"exponential\":{\"type\":\"object\",\"description\":\"Retry with exponential backoff (delay doubles each time)\",\"properties\":{\"attempts\":{\"type\":\"integer\",\"description\":\"Number of retry attempts\"},\"multiplier\":{\"type\":\"integer\",\"description\":\"Multiplier for exponential backoff\"},\"seconds\":{\"type\":\"integer\",\"minimum\":1,\"description\":\"Initial delay in seconds\"},\"random_factor\":{\"type\":\"integer\",\"minimum\":0,\"maximum\":100,\"description\":\"Random jitter percentage (0-100) to avoid thundering herd\"}}},\"retry_if\":{\"$ref\":\"#/components/schemas/RetryIf\"}}},\"FlowNote\":{\"type\":\"object\",\"description\":\"A sticky note attached to a flow for documentation and annotation\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"Unique identifier for the note\"},\"text\":{\"type\":\"string\",\"description\":\"Content of the note\"},\"position\":{\"type\":\"object\",\"description\":\"Position of the note in the flow editor\",\"properties\":{\"x\":{\"type\":\"number\",\"description\":\"X coordinate\"},\"y\":{\"type\":\"number\",\"description\":\"Y coordinate\"}},\"required\":[\"x\",\"y\"]},\"size\":{\"type\":\"object\",\"description\":\"Size of the note in the flow editor\",\"properties\":{\"width\":{\"type\":\"number\",\"description\":\"Width in pixels\"},\"height\":{\"type\":\"number\",\"description\":\"Height in pixels\"}},\"required\":[\"width\",\"height\"]},\"color\":{\"type\":\"string\",\"description\":\"Color of the note (e.g., \\\"yellow\\\", \\\"#ffff00\\\")\"},\"type\":{\"type\":\"string\",\"enum\":[\"free\",\"group\"],\"description\":\"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes\"},\"locked\":{\"type\":\"boolean\",\"default\":false,\"description\":\"Whether the note is locked and cannot be edited or moved\"},\"contained_node_ids\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"For group notes, the IDs of nodes contained within this group\"}},\"required\":[\"id\",\"text\",\"color\",\"type\"]},\"FlowGroup\":{\"type\":\"object\",\"description\":\"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Display name for this group\"},\"note\":{\"type\":\"string\",\"description\":\"Markdown note shown below the group header\"},\"autocollapse\":{\"type\":\"boolean\",\"default\":false,\"description\":\"If true, this group is collapsed by default in the flow editor. UI hint only.\"},\"start_id\":{\"type\":\"string\",\"description\":\"ID of the first flow module in this group (topological entry point)\"},\"end_id\":{\"type\":\"string\",\"description\":\"ID of the last flow module in this group (topological exit point)\"},\"color\":{\"type\":\"string\",\"description\":\"Color for the group in the flow editor\"}},\"required\":[\"start_id\",\"end_id\"]},\"RetryIf\":{\"type\":\"object\",\"description\":\"Conditional retry based on error or result\",\"properties\":{\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables\"}},\"required\":[\"expr\"]},\"StopAfterIf\":{\"type\":\"object\",\"description\":\"Early termination condition for a module\",\"properties\":{\"skip_if_stopped\":{\"type\":\"boolean\",\"description\":\"If true, following steps are skipped when this condition triggers\"},\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop\"},\"error_message\":{\"type\":\"string\",\"nullable\":true,\"description\":\"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.\"},\"error_include_result\":{\"type\":\"boolean\",\"description\":\"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.\"}},\"required\":[\"expr\"]},\"FlowModule\":{\"type\":\"object\",\"description\":\"A single step in a flow. Can be a script, subflow, loop, or branch\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)\"},\"value\":{\"$ref\":\"#/components/schemas/FlowModuleValue\"},\"stop_after_if\":{\"description\":\"Early termination condition evaluated after this step completes\",\"$ref\":\"#/components/schemas/StopAfterIf\"},\"stop_after_all_iters_if\":{\"description\":\"For loops only - early termination condition evaluated after all iterations complete\",\"$ref\":\"#/components/schemas/StopAfterIf\"},\"skip_if\":{\"type\":\"object\",\"description\":\"Conditionally skip this step based on previous results or flow inputs\",\"properties\":{\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'\"}},\"required\":[\"expr\"]},\"sleep\":{\"description\":\"Delay before executing this step (in seconds or as expression)\",\"$ref\":\"#/components/schemas/InputTransform\"},\"cache_ttl\":{\"type\":\"number\",\"description\":\"Cache duration in seconds for this step's results\"},\"cache_ignore_s3_path\":{\"type\":\"boolean\"},\"timeout\":{\"description\":\"Maximum execution time in seconds (static value or expression)\",\"$ref\":\"#/components/schemas/InputTransform\"},\"delete_after_secs\":{\"type\":\"integer\",\"description\":\"If set, delete the step's args, result and logs after this many seconds following job completion\"},\"summary\":{\"type\":\"string\",\"description\":\"Short description of what this step does\"},\"mock\":{\"type\":\"object\",\"description\":\"Mock configuration for testing without executing the actual step\",\"properties\":{\"enabled\":{\"type\":\"boolean\",\"description\":\"If true, return mock value instead of executing\"},\"return_value\":{\"description\":\"Value to return when mocked\"}}},\"suspend\":{\"type\":\"object\",\"description\":\"Configuration for approval/resume steps that wait for user input\",\"properties\":{\"required_events\":{\"type\":\"integer\",\"description\":\"Number of approvals required before continuing\"},\"timeout\":{\"type\":\"integer\",\"description\":\"Timeout in seconds before auto-continuing or canceling\"},\"resume_form\":{\"type\":\"object\",\"description\":\"Form schema for collecting input when resuming\",\"properties\":{\"schema\":{\"type\":\"object\",\"description\":\"JSON Schema for the resume form\"}}},\"user_auth_required\":{\"type\":\"boolean\",\"description\":\"If true, only authenticated users can approve\"},\"user_groups_required\":{\"description\":\"Expression or list of groups that can approve\",\"$ref\":\"#/components/schemas/InputTransform\"},\"self_approval_disabled\":{\"type\":\"boolean\",\"description\":\"If true, the user who started the flow cannot approve\"},\"hide_cancel\":{\"type\":\"boolean\",\"description\":\"If true, hide the cancel button on the approval form\"},\"continue_on_disapprove_timeout\":{\"type\":\"boolean\",\"description\":\"If true, continue flow on timeout instead of canceling\"}}},\"priority\":{\"type\":\"number\",\"description\":\"Execution priority for this step (higher numbers run first)\"},\"continue_on_error\":{\"type\":\"boolean\",\"description\":\"If true, flow continues even if this step fails\"},\"retry\":{\"description\":\"Retry configuration if this step fails\",\"$ref\":\"#/components/schemas/Retry\"},\"debouncing\":{\"description\":\"Debounce configuration for this step (EE only)\",\"type\":\"object\",\"properties\":{\"debounce_delay_s\":{\"type\":\"integer\",\"description\":\"Delay in seconds to debounce this step's executions across flow runs\"},\"debounce_key\":{\"type\":\"string\",\"description\":\"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-\"},\"debounce_args_to_accumulate\":{\"type\":\"array\",\"description\":\"Array-type arguments to accumulate across debounced executions\",\"items\":{\"type\":\"string\"}},\"max_total_debouncing_time\":{\"type\":\"integer\",\"description\":\"Maximum total time in seconds before forced execution\"},\"max_total_debounces_amount\":{\"type\":\"integer\",\"description\":\"Maximum number of debounces before forced execution\"}}}},\"required\":[\"value\",\"id\"]},\"InputTransform\":{\"description\":\"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs\",\"oneOf\":[{\"$ref\":\"#/components/schemas/StaticTransform\"},{\"$ref\":\"#/components/schemas/JavascriptTransform\"},{\"$ref\":\"#/components/schemas/AiTransform\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"static\":\"#/components/schemas/StaticTransform\",\"javascript\":\"#/components/schemas/JavascriptTransform\",\"ai\":\"#/components/schemas/AiTransform\"}}},\"StaticTransform\":{\"type\":\"object\",\"description\":\"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'\",\"properties\":{\"value\":{\"description\":\"The static value. For resources, use format '$res:path/to/resource'\"},\"type\":{\"type\":\"string\",\"enum\":[\"static\"]}},\"required\":[\"type\"]},\"JavascriptTransform\":{\"type\":\"object\",\"description\":\"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value\",\"properties\":{\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)\"},\"type\":{\"type\":\"string\",\"enum\":[\"javascript\"]}},\"required\":[\"expr\",\"type\"]},\"AiTransform\":{\"type\":\"object\",\"description\":\"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"ai\"]}},\"required\":[\"type\"]},\"AIProviderKind\":{\"type\":\"string\",\"description\":\"Supported AI provider types\",\"enum\":[\"openai\",\"azure_openai\",\"azure_foundry\",\"anthropic\",\"mistral\",\"deepseek\",\"googleai\",\"groq\",\"openrouter\",\"togetherai\",\"customai\",\"aws_bedrock\"]},\"ProviderConfig\":{\"type\":\"object\",\"description\":\"Complete AI provider configuration with resource reference and model selection\",\"properties\":{\"kind\":{\"$ref\":\"#/components/schemas/AIProviderKind\"},\"resource\":{\"type\":\"string\",\"description\":\"Resource reference in format '$res:{resource_path}' pointing to provider credentials\"},\"model\":{\"type\":\"string\",\"description\":\"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')\"},\"reasoning_effort\":{\"type\":\"string\",\"description\":\"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.\"}},\"required\":[\"kind\",\"resource\",\"model\"]},\"StaticProviderTransform\":{\"type\":\"object\",\"description\":\"Static provider configuration passed directly to the AI agent\",\"properties\":{\"value\":{\"$ref\":\"#/components/schemas/ProviderConfig\"},\"type\":{\"type\":\"string\",\"enum\":[\"static\"]}},\"required\":[\"type\",\"value\"]},\"ProviderTransform\":{\"description\":\"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined\",\"oneOf\":[{\"$ref\":\"#/components/schemas/StaticProviderTransform\"},{\"$ref\":\"#/components/schemas/JavascriptTransform\"},{\"$ref\":\"#/components/schemas/AiTransform\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"static\":\"#/components/schemas/StaticProviderTransform\",\"javascript\":\"#/components/schemas/JavascriptTransform\",\"ai\":\"#/components/schemas/AiTransform\"}}},\"MemoryOff\":{\"type\":\"object\",\"description\":\"No conversation memory/context\",\"properties\":{\"kind\":{\"type\":\"string\",\"enum\":[\"off\"]}},\"required\":[\"kind\"]},\"MemoryAuto\":{\"type\":\"object\",\"description\":\"Automatic context management\",\"properties\":{\"kind\":{\"type\":\"string\",\"enum\":[\"auto\"]},\"context_length\":{\"type\":\"integer\",\"description\":\"Maximum number of messages to retain in context\"},\"memory_id\":{\"type\":\"string\",\"description\":\"Identifier for persistent memory across agent invocations\"}},\"required\":[\"kind\"]},\"MemoryMessage\":{\"type\":\"object\",\"description\":\"A single message in conversation history\",\"properties\":{\"role\":{\"type\":\"string\",\"enum\":[\"user\",\"assistant\",\"system\"]},\"content\":{\"type\":\"string\"}},\"required\":[\"role\",\"content\"]},\"MemoryManual\":{\"type\":\"object\",\"description\":\"Explicit message history\",\"properties\":{\"kind\":{\"type\":\"string\",\"enum\":[\"manual\"]},\"messages\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/MemoryMessage\"}}},\"required\":[\"kind\",\"messages\"]},\"MemoryConfig\":{\"description\":\"Conversation memory configuration\",\"oneOf\":[{\"$ref\":\"#/components/schemas/MemoryOff\"},{\"$ref\":\"#/components/schemas/MemoryAuto\"},{\"$ref\":\"#/components/schemas/MemoryManual\"}],\"discriminator\":{\"propertyName\":\"kind\",\"mapping\":{\"off\":\"#/components/schemas/MemoryOff\",\"auto\":\"#/components/schemas/MemoryAuto\",\"manual\":\"#/components/schemas/MemoryManual\"}}},\"StaticMemoryTransform\":{\"type\":\"object\",\"description\":\"Static memory configuration passed directly to the AI agent\",\"properties\":{\"value\":{\"$ref\":\"#/components/schemas/MemoryConfig\"},\"type\":{\"type\":\"string\",\"enum\":[\"static\"]}},\"required\":[\"type\",\"value\"]},\"MemoryTransform\":{\"description\":\"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined\",\"oneOf\":[{\"$ref\":\"#/components/schemas/StaticMemoryTransform\"},{\"$ref\":\"#/components/schemas/JavascriptTransform\"},{\"$ref\":\"#/components/schemas/AiTransform\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"static\":\"#/components/schemas/StaticMemoryTransform\",\"javascript\":\"#/components/schemas/JavascriptTransform\",\"ai\":\"#/components/schemas/AiTransform\"}}},\"FlowModuleValue\":{\"description\":\"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type\",\"oneOf\":[{\"$ref\":\"#/components/schemas/RawScript\"},{\"$ref\":\"#/components/schemas/PathScript\"},{\"$ref\":\"#/components/schemas/PathFlow\"},{\"$ref\":\"#/components/schemas/ForloopFlow\"},{\"$ref\":\"#/components/schemas/WhileloopFlow\"},{\"$ref\":\"#/components/schemas/BranchOne\"},{\"$ref\":\"#/components/schemas/BranchAll\"},{\"$ref\":\"#/components/schemas/Identity\"},{\"$ref\":\"#/components/schemas/AiAgent\"}],\"discriminator\":{\"propertyName\":\"type\",\"mapping\":{\"rawscript\":\"#/components/schemas/RawScript\",\"script\":\"#/components/schemas/PathScript\",\"flow\":\"#/components/schemas/PathFlow\",\"forloopflow\":\"#/components/schemas/ForloopFlow\",\"whileloopflow\":\"#/components/schemas/WhileloopFlow\",\"branchone\":\"#/components/schemas/BranchOne\",\"branchall\":\"#/components/schemas/BranchAll\",\"identity\":\"#/components/schemas/Identity\",\"aiagent\":\"#/components/schemas/AiAgent\"}}},\"RawScript\":{\"type\":\"object\",\"description\":\"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments\",\"additionalProperties\":{\"$ref\":\"#/components/schemas/InputTransform\"}},\"content\":{\"type\":\"string\",\"description\":\"The script source code. Should export a 'main' function\"},\"language\":{\"type\":\"string\",\"description\":\"Programming language for this script\",\"enum\":[\"deno\",\"bun\",\"python3\",\"go\",\"bash\",\"powershell\",\"postgresql\",\"mysql\",\"bigquery\",\"snowflake\",\"mssql\",\"oracledb\",\"graphql\",\"nativets\",\"php\",\"rust\",\"ansible\",\"csharp\",\"nu\",\"java\",\"ruby\",\"rlang\",\"duckdb\"]},\"path\":{\"type\":\"string\",\"description\":\"Optional path for saving this script\"},\"lock\":{\"type\":\"string\",\"description\":\"Lock file content for dependencies\"},\"type\":{\"type\":\"string\",\"enum\":[\"rawscript\"]},\"tag\":{\"type\":\"string\",\"description\":\"Worker group tag for execution routing\"},\"concurrent_limit\":{\"type\":\"number\",\"description\":\"Maximum concurrent executions of this script\"},\"concurrency_time_window_s\":{\"type\":\"number\",\"description\":\"Time window for concurrent_limit\"},\"custom_concurrency_key\":{\"type\":\"string\",\"description\":\"Custom key for grouping concurrent executions\"},\"is_trigger\":{\"type\":\"boolean\",\"description\":\"If true, this script is a trigger that can start the flow\"},\"assets\":{\"type\":\"array\",\"description\":\"External resources this script accesses (S3 objects, resources, etc.)\",\"items\":{\"type\":\"object\",\"required\":[\"path\",\"kind\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the asset\"},\"kind\":{\"type\":\"string\",\"description\":\"Type of asset\",\"enum\":[\"s3object\",\"resource\",\"ducklake\",\"datatable\",\"volume\"]},\"access_type\":{\"type\":\"string\",\"nullable\":true,\"description\":\"Access level for this asset\",\"enum\":[\"r\",\"w\",\"rw\"]},\"alt_access_type\":{\"type\":\"string\",\"nullable\":true,\"description\":\"Alternative access level\",\"enum\":[\"r\",\"w\",\"rw\"]}}}}},\"required\":[\"type\",\"content\",\"language\",\"input_transforms\"]},\"PathScript\":{\"type\":\"object\",\"description\":\"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments\",\"additionalProperties\":{\"$ref\":\"#/components/schemas/InputTransform\"}},\"path\":{\"type\":\"string\",\"description\":\"Path to the script in the workspace (e.g., 'f/scripts/send_email')\"},\"hash\":{\"type\":\"string\",\"description\":\"Optional specific version hash of the script to use\"},\"type\":{\"type\":\"string\",\"enum\":[\"script\"]},\"tag_override\":{\"type\":\"string\",\"description\":\"Override the script's default worker group tag\"},\"is_trigger\":{\"type\":\"boolean\",\"description\":\"If true, this script is a trigger that can start the flow\"}},\"required\":[\"type\",\"path\",\"input_transforms\"]},\"PathFlow\":{\"type\":\"object\",\"description\":\"Reference to an existing flow by path. Use this to call another flow as a subflow\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments\",\"additionalProperties\":{\"$ref\":\"#/components/schemas/InputTransform\"}},\"path\":{\"type\":\"string\",\"description\":\"Path to the flow in the workspace (e.g., 'f/flows/process_user')\"},\"type\":{\"type\":\"string\",\"enum\":[\"flow\"]}},\"required\":[\"type\",\"path\",\"input_transforms\"]},\"ForloopFlow\":{\"type\":\"object\",\"description\":\"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations\",\"properties\":{\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"iterator\":{\"description\":\"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'\",\"$ref\":\"#/components/schemas/InputTransform\"},\"skip_failures\":{\"type\":\"boolean\",\"description\":\"If true, iteration failures don't stop the loop. Failed iterations return null\"},\"type\":{\"type\":\"string\",\"enum\":[\"forloopflow\"]},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency\"},\"parallelism\":{\"description\":\"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression\",\"$ref\":\"#/components/schemas/InputTransform\"},\"squash\":{\"type\":\"boolean\"}},\"required\":[\"modules\",\"iterator\",\"skip_failures\",\"type\"]},\"WhileloopFlow\":{\"type\":\"object\",\"description\":\"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination\",\"properties\":{\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute in each iteration. Use stop_after_if to control when the loop ends\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"skip_failures\":{\"type\":\"boolean\",\"description\":\"If true, iteration failures don't stop the loop. Failed iterations return null\"},\"type\":{\"type\":\"string\",\"enum\":[\"whileloopflow\"]},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, iterations run concurrently (use with caution in while loops)\"},\"parallelism\":{\"description\":\"Maximum number of concurrent iterations when parallel=true\",\"$ref\":\"#/components/schemas/InputTransform\"},\"squash\":{\"type\":\"boolean\"}},\"required\":[\"modules\",\"skip_failures\",\"type\"]},\"BranchOne\":{\"type\":\"object\",\"description\":\"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes\",\"properties\":{\"branches\":{\"type\":\"array\",\"description\":\"Array of branches to evaluate in order. The first branch with expr evaluating to true executes\",\"items\":{\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Short description of this branch condition\"},\"expr\":{\"type\":\"string\",\"description\":\"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins\"},\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute if this branch's expr is true\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}}},\"required\":[\"modules\",\"expr\"]}},\"default\":{\"type\":\"array\",\"description\":\"Steps to execute if no branch expressions match\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}},\"type\":{\"type\":\"string\",\"enum\":[\"branchone\"]}},\"required\":[\"branches\",\"default\",\"type\"]},\"BranchAll\":{\"type\":\"object\",\"description\":\"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently\",\"properties\":{\"branches\":{\"type\":\"array\",\"description\":\"Array of branches that all execute (either in parallel or sequentially)\",\"items\":{\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"string\",\"description\":\"Short description of this branch's purpose\"},\"skip_failure\":{\"type\":\"boolean\",\"description\":\"If true, failure in this branch doesn't fail the entire flow\"},\"modules\":{\"type\":\"array\",\"description\":\"Steps to execute in this branch\",\"items\":{\"$ref\":\"#/components/schemas/FlowModule\"}}},\"required\":[\"modules\"]}},\"type\":{\"type\":\"string\",\"enum\":[\"branchall\"]},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, all branches execute concurrently. If false, they execute sequentially\"}},\"required\":[\"branches\",\"type\"]},\"AgentTool\":{\"type\":\"object\",\"description\":\"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')\"},\"summary\":{\"type\":\"string\",\"description\":\"Short description of what this tool does (shown to the AI)\"},\"value\":{\"$ref\":\"#/components/schemas/ToolValue\"}},\"required\":[\"id\",\"value\"]},\"ToolValue\":{\"description\":\"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference\",\"oneOf\":[{\"$ref\":\"#/components/schemas/FlowModuleTool\"},{\"$ref\":\"#/components/schemas/McpToolValue\"},{\"$ref\":\"#/components/schemas/WebsearchToolValue\"}],\"discriminator\":{\"propertyName\":\"tool_type\",\"mapping\":{\"flowmodule\":\"#/components/schemas/FlowModuleTool\",\"mcp\":\"#/components/schemas/McpToolValue\",\"websearch\":\"#/components/schemas/WebsearchToolValue\"}}},\"FlowModuleTool\":{\"description\":\"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module\",\"allOf\":[{\"type\":\"object\",\"properties\":{\"tool_type\":{\"type\":\"string\",\"enum\":[\"flowmodule\"]}},\"required\":[\"tool_type\"]},{\"$ref\":\"#/components/schemas/FlowModuleValue\"}]},\"WebsearchToolValue\":{\"type\":\"object\",\"description\":\"A tool implemented as a websearch tool. The AI can call this like any other websearch tool\",\"properties\":{\"tool_type\":{\"type\":\"string\",\"enum\":[\"websearch\"]}},\"required\":[\"tool_type\"]},\"McpToolValue\":{\"type\":\"object\",\"description\":\"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers\",\"properties\":{\"tool_type\":{\"type\":\"string\",\"enum\":[\"mcp\"]},\"resource_path\":{\"type\":\"string\",\"description\":\"Path to the MCP resource/server configuration\"},\"include_tools\":{\"type\":\"array\",\"description\":\"Whitelist of specific tools to include from this MCP server\",\"items\":{\"type\":\"string\"}},\"exclude_tools\":{\"type\":\"array\",\"description\":\"Blacklist of tools to exclude from this MCP server\",\"items\":{\"type\":\"string\"}}},\"required\":[\"tool_type\",\"resource_path\"]},\"AiAgent\":{\"type\":\"object\",\"description\":\"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task\",\"properties\":{\"input_transforms\":{\"type\":\"object\",\"description\":\"Input parameters for the AI agent mapped to their values\",\"properties\":{\"provider\":{\"$ref\":\"#/components/schemas/ProviderTransform\"},\"output_type\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n\"},\"user_message\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax.\"},\"system_prompt\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"System instructions that guide the AI's behavior, persona, and response style. Optional.\"},\"streaming\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n\"},\"memory\":{\"$ref\":\"#/components/schemas/MemoryTransform\"},\"output_schema\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n\"},\"user_attachments\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n\"},\"max_completion_tokens\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n\"},\"temperature\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n\"},\"max_iterations\":{\"allOf\":[{\"$ref\":\"#/components/schemas/InputTransform\"}],\"description\":\"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n\"}},\"required\":[\"provider\",\"user_message\",\"output_type\"]},\"tools\":{\"type\":\"array\",\"description\":\"Array of tools the agent can use. The agent decides which tools to call based on the task\",\"items\":{\"$ref\":\"#/components/schemas/AgentTool\"}},\"type\":{\"type\":\"string\",\"enum\":[\"aiagent\"]},\"tag\":{\"type\":\"string\",\"description\":\"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)\"},\"omit_output_from_conversation\":{\"type\":\"boolean\",\"default\":false,\"description\":\"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.\"},\"parallel\":{\"type\":\"boolean\",\"description\":\"If true, the agent can execute multiple tool calls in parallel\"}},\"required\":[\"tools\",\"type\",\"input_transforms\"]},\"Identity\":{\"type\":\"object\",\"description\":\"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"identity\"]},\"flow\":{\"type\":\"boolean\",\"description\":\"If true, marks this as a flow identity (special handling)\"}},\"required\":[\"type\"]},\"FlowStatus\":{\"type\":\"object\",\"properties\":{\"step\":{\"type\":\"integer\"},\"modules\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/FlowStatusModule\"}},\"user_states\":{\"additionalProperties\":true},\"preprocessor_module\":{\"allOf\":[{\"$ref\":\"#/components/schemas/FlowStatusModule\"}]},\"failure_module\":{\"allOf\":[{\"$ref\":\"#/components/schemas/FlowStatusModule\"},{\"type\":\"object\",\"properties\":{\"parent_module\":{\"type\":\"string\"}}}]},\"retry\":{\"type\":\"object\",\"properties\":{\"fail_count\":{\"type\":\"integer\"},\"failed_jobs\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"format\":\"uuid\"}}}}},\"required\":[\"step\",\"modules\",\"failure_module\"]},\"FlowStatusModule\":{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"WaitingForPriorSteps\",\"WaitingForEvents\",\"WaitingForExecutor\",\"InProgress\",\"Success\",\"Failure\"]},\"id\":{\"type\":\"string\"},\"job\":{\"type\":\"string\",\"format\":\"uuid\"},\"count\":{\"type\":\"integer\"},\"progress\":{\"type\":\"integer\"},\"iterator\":{\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"integer\"},\"itered\":{\"type\":\"array\",\"items\":{}},\"itered_len\":{\"type\":\"integer\"},\"args\":{}}},\"flow_jobs\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"flow_jobs_success\":{\"type\":\"array\",\"items\":{\"type\":\"boolean\"}},\"flow_jobs_duration\":{\"type\":\"object\",\"properties\":{\"started_at\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"duration_ms\":{\"type\":\"array\",\"items\":{\"type\":\"integer\"}}}},\"branch_chosen\":{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"branch\",\"default\"]},\"branch\":{\"type\":\"integer\"}},\"required\":[\"type\"]},\"branchall\":{\"type\":\"object\",\"properties\":{\"branch\":{\"type\":\"integer\"},\"len\":{\"type\":\"integer\"}},\"required\":[\"branch\",\"len\"]},\"approvers\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"resume_id\":{\"type\":\"integer\"},\"approver\":{\"type\":\"string\"}},\"required\":[\"resume_id\",\"approver\"]}},\"failed_retries\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"format\":\"uuid\"}},\"skipped\":{\"type\":\"boolean\"},\"agent_actions\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"oneOf\":[{\"type\":\"object\",\"properties\":{\"job_id\":{\"type\":\"string\",\"format\":\"uuid\"},\"function_name\":{\"type\":\"string\"},\"type\":{\"type\":\"string\",\"enum\":[\"tool_call\"]},\"module_id\":{\"type\":\"string\"}},\"required\":[\"job_id\",\"function_name\",\"type\",\"module_id\"]},{\"type\":\"object\",\"properties\":{\"call_id\":{\"type\":\"string\",\"format\":\"uuid\"},\"function_name\":{\"type\":\"string\"},\"resource_path\":{\"type\":\"string\"},\"type\":{\"type\":\"string\",\"enum\":[\"mcp_tool_call\"]},\"arguments\":{\"type\":\"object\"}},\"required\":[\"call_id\",\"function_name\",\"resource_path\",\"type\"]},{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"web_search\"]}},\"required\":[\"type\"]},{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\",\"enum\":[\"message\"]}},\"required\":[\"content\",\"type\"]}]}},\"agent_actions_success\":{\"type\":\"array\",\"items\":{\"type\":\"boolean\"}}},\"required\":[\"type\"]}}"; -export declare const CLI_COMMANDS = "# Windmill CLI Commands\n\nThe Windmill CLI (`wmill`) provides commands for managing scripts, flows, apps, and other resources.\n\n## Global Options\n\n- `--workspace ` - Specify the target workspace. This overrides the default workspace.\n- `--debug --verbose` - Show debug/verbose logs\n- `--show-diffs` - Show diff informations when syncing (may show sensitive informations)\n- `--token ` - Specify an API token. This will override any stored token.\n- `--base-url ` - Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used.\n- `--config-dir ` - Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location.\n\n## Commands\n\n### app\n\napp related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `app list` - list all apps\n - `--json` - Output as JSON (for piping to jq)\n- `app get ` - get an app's details\n - `--json` - Output as JSON (for piping to jq)\n- `app push [file_path:string] [remote_path:string]` - push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml.\n- `app dev [app_folder:string]` - Start a development server for building apps with live reload and hot module replacement\n - `--port ` - Port to run the dev server on (will find next available port if occupied)\n - `--host ` - Host to bind the dev server to\n - `--entry ` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)\n - `--no-open` - Don't automatically open the browser\n- `app lint [app_folder:string]` - Lint a raw app folder to validate structure and buildability\n - `--fix` - Attempt to fix common issues (not implemented yet)\n- `app new` - create a new raw app from a template\n - `--summary ` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode.\n - `--path ` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode.\n - `--framework ` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode.\n - `--datatable ` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured.\n - `--schema ` - Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist.\n - `--overwrite` - Overwrite the target directory if it already exists, without prompting.\n - `--no-open-in-desktop` - Do not prompt to open the new app in Claude Desktop.\n- `app generate-agents [app_folder:string]` - regenerate AGENTS.md and DATATABLES.md from remote workspace\n- `app set-permissioned-as ` - Set the on_behalf_of_email for an app (requires admin or wm_deployers group)\n\n### audit\n\nView audit logs (requires admin)\n\n**Subcommands:**\n\n- `audit list` - List audit log entries\n- `audit get ` - Get a specific audit log entry\n - `--json` - Output as JSON (for piping to jq)\n\n### config\n\nShow all available wmill.yaml configuration options\n\n**Options:**\n- `--json` - Output as JSON for programmatic consumption\n\n**Subcommands:**\n\n- `config migrate` - Migrate wmill.yaml from gitBranches/environments to workspaces format\n\n### datatable\n\ndatatable related commands\n\n**Subcommands:**\n\n- `datatable list` - list all datatables in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `datatable run ` - run a SQL query on a datatable\n - `-n --name ` - Datatable name (default: main)\n - `-s --silent` - Output only the final result as JSON. Useful for scripting.\n- `datatable migrate` - manage datatable migrations\n - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files)\n - `-d --datatable ` - Target datatable (default: main)\n - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable)\n - `-d --datatable ` - Target datatable (default: main)\n - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable)\n - `-d --datatable ` - Target datatable (default: main)\n- `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://\n - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database\n - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved)\n- `datatable serve` - Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string\n - `--port ` - Port to listen on (default: first free port in 5433-5500)\n - `--host ` - Bind address (default: 127.0.0.1)\n - `--password ` - Password for Postgres clients (default: generate a random password at startup)\n- `datatable psql` - Start a serve listener and launch psql connected to it\n - `-n --name ` - Datatable to connect psql to (default: main)\n - `--port ` - Port the proxy listens on (default: first free port in 5433-5500)\n - `--host ` - Bind address for the proxy (default: 127.0.0.1)\n - `--password ` - Password for the temporary Postgres proxy (default: generate a random password at startup)\n\n### dependencies\n\nworkspace dependencies related commands\n\n**Alias:** `deps`\n\n**Subcommands:**\n\n- `dependencies push ` - Push workspace dependencies from a local file\n\n### dev\n\nWatch local file changes and live-reload the dev page for preview. Does NOT deploy to the remote workspace \u2014 use wmill sync push for that.\n\n**Options:**\n- `--includes ` - Filter paths given a glob pattern or path\n- `--proxy-port ` - Port for a localhost reverse proxy to the remote Windmill server\n- `--path ` - Watch a specific windmill path (e.g., u/admin/my_script or f/my_flow)\n- `--no-open` - Do not open the browser automatically\n\n### docs\n\nSearch Windmill documentation.\n\n**Arguments:** ``\n\n**Options:**\n- `--json` - Output results as JSON.\n\n### ducklake\n\nducklake related commands\n\n**Subcommands:**\n\n- `ducklake list` - list all ducklakes in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `ducklake run ` - run a SQL query on a ducklake\n - `-n --name ` - Ducklake name (default: main)\n - `-s --silent` - Output only the final result as JSON. Useful for scripting.\n\n### flow\n\nflow related commands\n\n**Options:**\n- `--show-archived` - Enable archived flows in output\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `flow list` - list all flows\n - `--show-archived` - Enable archived flows in output\n - `--json` - Output as JSON (for piping to jq)\n- `flow get ` - get a flow's details\n - `--json` - Output as JSON (for piping to jq)\n- `flow push ` - push a local flow spec. This overrides any remote versions.\n - `--message ` - Deployment message\n- `flow run ` - run a flow by path.\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting.\n- `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow).\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not output anything other then the final output. Useful for scripting.\n - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files.\n - `--step ` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does.\n- `flow new ` - create a new empty flow\n - `--summary ` - flow summary\n - `--description ` - flow description\n- `flow bootstrap ` - create a new empty flow (alias for new)\n - `--summary ` - flow summary\n - `--description ` - flow description\n- `flow history ` - Show version history for a flow\n - `--json` - Output as JSON (for piping to jq)\n- `flow show-version ` - Show a specific version of a flow\n - `--json` - Output as JSON (for piping to jq)\n- `flow set-permissioned-as ` - Set the on_behalf_of_email for a flow (requires admin or wm_deployers group)\n\n### folder\n\nfolder related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `folder list` - list all folders\n - `--json` - Output as JSON (for piping to jq)\n- `folder get ` - get a folder's details\n - `--json` - Output as JSON (for piping to jq)\n- `folder new ` - create a new folder locally\n - `--summary ` - folder summary\n- `folder push ` - push a local folder to the remote by name. This overrides any remote versions.\n- `folder add-missing` - create default folder.meta.yaml for all subdirectories of f/ that are missing one\n - `-y, --yes` - skip confirmation prompt\n- `folder show-rules ` - Show default_permissioned_as rules for a folder. Use --test-path to see which rule matches a given item path.\n - `--test-path ` - Test which rule matches this item path (e.g. f/prod/jobs/my_script)\n - `--json` - Output as JSON\n\n### generate-metadata\n\nRegenerate stale local locks and script schemas and refresh wmill-lock.yaml content hashes (scripts, flows, apps). Writes local files only, not a deploy. Run it after edits that add or remove imports or change a script's arguments, so the lock, the auto-generated UI schema, and wmill-lock.yaml stay in sync.\n\n**Arguments:** `[folder:string]`\n\n**Options:**\n- `--yes` - Skip confirmation prompt\n- `--dry-run` - Show what would be updated without making changes\n- `--lock-only` - Re-generate only the lock files\n- `--schema-only` - Re-generate only script schemas (skips flows and apps)\n- `--skip-scripts` - Skip processing scripts\n- `--skip-flows` - Skip processing flows\n- `--skip-apps` - Skip processing apps\n- `--strict-folder-boundaries` - Only update items inside the specified folder (requires folder argument)\n- `--parallel ` - Number of items to process in parallel\n- `-i --includes ` - Comma separated patterns to specify which files to include\n- `-e --excludes ` - Comma separated patterns to specify which files to exclude\n\n**Subcommands:**\n\n- `generate-metadata rehash [folder:string]` - Refresh wmill-lock.yaml content hashes from the on-disk .lock and .script.yaml without re-resolving dependencies or hitting the backend. Use when those files are already correct and only the hashes need updating: bootstrapping missing entries or recovering from hash drift.\n - `--skip-scripts` - Skip processing scripts\n - `--skip-flows` - Skip processing flows\n - `--skip-apps` - Skip processing apps\n - `--parallel ` - Number of items to process in parallel\n - `-i --includes ` - Comma separated patterns to specify which files to include\n - `-e --excludes ` - Comma separated patterns to specify which files to exclude\n\n### gitsync-settings\n\nManage git-sync settings between local wmill.yaml and Windmill backend\n\n**Subcommands:**\n\n- `gitsync-settings pull` - Pull git-sync settings from Windmill backend to local wmill.yaml\n - `--repository ` - Specify repository path (e.g., u/user/repo)\n - `--default` - Write settings to top-level defaults instead of overrides\n - `--replace` - Replace existing settings (non-interactive mode)\n - `--override` - Add branch-specific override (non-interactive mode)\n - `--diff` - Show differences without applying changes\n - `--json-output` - Output in JSON format\n - `--with-backend-settings ` - Use provided JSON settings instead of querying backend (for testing)\n - `--yes` - Skip interactive prompts and use default behavior\n - `--promotion ` - Use promotionOverrides from the specified branch instead of regular overrides\n- `gitsync-settings push` - Push git-sync settings from local wmill.yaml to Windmill backend\n - `--repository ` - Specify repository path (e.g., u/user/repo)\n - `--diff` - Show what would be pushed without applying changes\n - `--json-output` - Output in JSON format\n - `--with-backend-settings ` - Use provided JSON settings instead of querying backend (for testing)\n - `--yes` - Skip interactive prompts and use default behavior\n - `--promotion ` - Use promotionOverrides from the specified branch instead of regular overrides\n\n### group\n\nManage workspace groups\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `group list` - List all groups in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `group get ` - Get group details and members\n - `--json` - Output as JSON (for piping to jq)\n- `group create ` - Create a new group\n - `--summary ` - Group summary/description\n- `group delete ` - Delete a group\n- `group add-user ` - Add a user to a group\n- `group remove-user ` - Remove a user from a group\n\n### hub\n\nHub related commands. EXPERIMENTAL. INTERNAL USE ONLY.\n\n**Subcommands:**\n\n- `hub pull` - pull any supported definitions. EXPERIMENTAL.\n\n### init\n\nBootstrap a windmill project with a wmill.yaml file\n\n**Options:**\n- `--use-default` - Use default settings without checking backend\n- `--use-backend` - Use backend git-sync settings if available\n- `--repository ` - Specify repository path (e.g., u/user/repo) when using backend settings\n- `--bind-profile` - Automatically bind active workspace profile to current Git branch\n- `--no-bind-profile` - Skip workspace profile binding prompt\n\n### instance\n\nsync local with a remote instance or the opposite (push or pull)\n\n**Subcommands:**\n\n- `instance add [instance_name:string] [remote:string] [token:string]` - Add a new instance\n- `instance remove ` - Remove an instance\n- `instance switch ` - Switch the current instance\n- `instance pull` - Pull instance settings, users, configs, instance groups and overwrite local\n - `--yes` - Pull without needing confirmation\n - `--dry-run` - Perform a dry run without making changes\n - `--skip-users` - Skip pulling users\n - `--skip-settings` - Skip pulling settings\n - `--skip-configs` - Skip pulling configs (worker groups)\n - `--skip-groups` - Skip pulling instance groups\n - `--include-workspaces` - Also pull workspaces\n - `--folder-per-instance` - Create a folder per instance\n - `--instance ` - Name of the instance to pull from, override the active instance\n - `--prefix ` - Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces\n - `--prefix-settings` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance\n- `instance push` - Push instance settings, users, configs, group and overwrite remote\n - `--yes` - Push without needing confirmation\n - `--dry-run` - Perform a dry run without making changes\n - `--skip-users` - Skip pushing users\n - `--skip-settings` - Skip pushing settings\n - `--skip-configs` - Skip pushing configs (worker groups)\n - `--skip-groups` - Skip pushing instance groups\n - `--include-workspaces` - Also push workspaces\n - `--folder-per-instance` - Create a folder per instance\n - `--instance ` - Name of the instance to push to, override the active instance\n - `--prefix ` - Prefix of the local workspaces folders to push\n - `--prefix-settings` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance\n- `instance whoami` - Display information about the currently logged-in user\n- `instance get-config` - Dump the current instance config (global settings + worker configs) as YAML\n - `-o, --output-file ` - Write YAML to a file instead of stdout\n - `--show-secrets` - Include sensitive fields (license key, JWT secret) without prompting\n - `--instance ` - Name of the instance, override the active instance\n- `instance connect-slack` - Non-interactively connect Slack at the instance level using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: global_settings 'slack' row + encrypted f/slack_bot/global_bot_token variable and resource in the admins workspace.\n - `--bot-token ` - Slack bot token (xoxb-...)\n - `--team-id ` - Slack team id\n - `--team-name ` - Slack team name\n - `--instance ` - Instance profile to connect against (defaults to the active instance)\n\n### job\n\nManage jobs (list, inspect, cancel)\n\n**Subcommands:**\n\n- `job list` - List recent jobs\n- `job get ` - Get job details. For flows: shows step tree with sub-job IDs\n - `--json` - Output as JSON (for piping to jq)\n- `job result ` - Get the result of a completed job (machine-friendly)\n- `job logs ` - Get job logs. For flows: aggregates all step logs\n- `job cancel ` - Cancel a running or queued job\n - `--reason ` - Reason for cancellation\n- `job rerun ` - Re-run a completed job with the same args. Prints the new job UUID on stdout.\n- `job restart ` - Restart a completed flow at a given top-level step. Prints the new flow job UUID on stdout.\n - `--step ` - Top-level step id to restart the flow from\n - `--iteration ` - For a top-level branchall or for-loop step, the iteration to restart at\n\n### jobs\n\nManage jobs (import/export)\n\n**Subcommands:**\n\n- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace\n - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json)\n - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json)\n - `--skip-worker-check` - Skip checking for active workers before export\n- `jobs push [workspace:string]` - Push completed and queued jobs to workspace\n - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json)\n - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json)\n - `--skip-worker-check` - Skip checking for active workers before import\n\n### lint\n\nValidate Windmill flow, schedule, and trigger YAML files in a directory\n\n**Arguments:** `[directory:string]`\n\n**Options:**\n- `--json` - Output results in JSON format\n- `--fail-on-warn` - Exit with code 1 when warnings are emitted\n- `--locks-required` - Fail if scripts or flow inline scripts that need locks have no locks\n- `-w, --watch` - Watch for file changes and re-lint automatically\n\n### object-storage\n\nObject storage (S3) related commands. Operates on the workspace's default object storage; use --storage to target a configured secondary storage.\n\n**Alias:** `s3`\n\n**Subcommands:**\n\n- `object-storage list` - List configured object storages for the workspace (default + secondary).\n - `--json` - Output as JSON (for piping to jq)\n- `object-storage files [prefix:string]` - List files in an object storage. Optionally filter by prefix.\n - `--json` - Output as JSON (for piping to jq)\n - `--max-keys ` - Page size (default 100)\n - `--marker ` - Pagination marker from a previous response\n - `--storage ` - Secondary storage name (omit for the workspace default)\n- `object-storage upload ` - Upload a local file to object storage at the given file key.\n - `--storage ` - Secondary storage name\n - `--content-type ` - Content-Type header to set on the object\n - `--content-disposition ` - Content-Disposition header to set on the object\n- `object-storage download [output_path:string]` - Download an object to a local file (or stdout). Default output path is the basename of the file key in the current directory.\n - `--storage ` - Secondary storage name\n - `--stdout` - Write file contents to stdout instead of a file\n- `object-storage delete ` - Delete an object from object storage. Prompts for confirmation unless --yes is set.\n - `--storage ` - Secondary storage name\n - `--yes` - Skip the confirmation prompt\n- `object-storage move ` - Move an object within the same storage (rename or relocate by key).\n - `--storage ` - Secondary storage name\n- `object-storage info ` - Show metadata (size, mime, last-modified) for an object.\n - `--json` - Output as JSON (for piping to jq)\n - `--storage ` - Secondary storage name\n- `object-storage preview ` - Preview the contents of an object (text/CSV). Use --bytes-from / --bytes-length to peek at a slice of binary files.\n - `--storage ` - Secondary storage name\n - `--mime ` - Override the detected mime type (e.g. text/csv)\n - `--bytes-from ` - Start offset in bytes\n - `--bytes-length ` - Number of bytes to read\n - `--csv-separator ` - CSV column separator (default ,)\n - `--csv-header` - Treat the first CSV row as a header\n\n### pipeline\n\ninspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on ` annotations)\n\n**Subcommands:**\n\n- `pipeline list` - list pipeline folders in the workspace\n - `--json` - Output as JSON (for piping to jq)\n- `pipeline show ` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal\n - `--json` - Output the raw asset graph as JSON\n - `--local` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace \u2014 no deploy needed.\n- `pipeline run ` - run a cascade: from --from (a root OR any mid-DAG model), fan downstream up to the --to end node(s)\n - `--from ` - Start script (short name or path). May be any node, including a mid-DAG model \u2014 that node plus its transitive downstream runs, upstream is NOT re-run (dbt `--select model+`). Defaults to the folder's sole schedule/manual root.\n - `--to ` - End node(s) to stop at \u2014 script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream.\n - `--dry-run` - Print the topological run plan without executing.\n - `--json` - Output the plan as JSON (for piping to jq).\n - `--local` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files.\n - `--upload ` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable.\n - `--arg ` - Pass a plain run arg to a script in the cascade, as SCRIPT:PARAM=VALUE (VALUE is parsed as JSON when possible, else taken as a string \u2014 e.g. daily_report:partition=2026-07-02). Repeatable.\n - `--partition ` - Partition value for `// partitioned` scripts in the run (e.g. 2026-06-30) \u2014 use it to backfill a past slice. With --local, time kinds (daily/hourly/weekly/monthly) default to the current UTC period when omitted; `dynamic` always needs it. Deployed runs without it defer to backend run-start resolution.\n- `pipeline docs ` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop\n - `--local` - Build the graph from local working-tree files instead of the deployed workspace.\n- `pipeline dev [folder:string]` - Live-preview a data pipeline from local files: watch an `f/` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).\n - `--port ` - Port for the dev WebSocket server.\n - `--no-open` - Do not open the browser automatically.\n - `--frontend ` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.\n\n### protection-rules\n\nSync workspace protection rules between protection-rules.yaml and Windmill. The file is keyed by workspace name; keys must match wmill.yaml 'workspaces'.\n\n**Subcommands:**\n\n- `protection-rules pull [workspace:string]` - Pull protection rules from Windmill into protection-rules.yaml for a workspace\n - `--all` - Pull every workspace defined in wmill.yaml\n - `--dry-run` - Show what would change without writing the file\n - `--json-output` - Output in JSON format\n- `protection-rules push [workspace:string]` - Push protection rules from protection-rules.yaml to Windmill for a workspace (full reconcile: creates, updates, and deletes)\n - `--all` - Push every workspace defined in protection-rules.yaml\n - `--dry-run` - Show what would change without applying\n - `--json-output` - Output in JSON format\n - `--yes` - Skip the confirmation prompt (including deletions)\n\n### queues\n\nList all queues with their metrics\n\n**Arguments:** `[workspace:string] the optional workspace to filter by (default to all workspaces)`\n\n**Options:**\n- `--instance [instance]` - Name of the instance to push to, override the active instance\n- `--base-url [baseUrl]` - If used with --token, will be used as the base url for the instance\n\n### refresh\n\nRefresh wmill-managed project files (AGENTS.wmill.md, skills, tsconfig.wmill.json)\n\n**Subcommands:**\n\n- `refresh prompts` - Refresh AGENTS.wmill.md and managed skills. User-owned AGENTS.md and CLAUDE.md are never overwritten unless you opt in.\n - `--yes` - Non-interactive: append the @AGENTS.wmill.md include to an existing AGENTS.md / CLAUDE.md without prompting. Without it, a non-interactive run leaves an unlinked file untouched.\n- `refresh tsconfig` - Refresh the wmill-managed tsconfig.wmill.json (and Deno import map for Deno projects)\n - `--yes` - Non-interactive: wire an existing custom tsconfig.json/deno.json to the managed file without prompting (a previously-generated config is always migrated automatically).\n\n### resource\n\nresource related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `resource list` - list all resources\n - `--json` - Output as JSON (for piping to jq)\n- `resource get ` - get a resource's details\n - `--json` - Output as JSON (for piping to jq)\n- `resource new ` - create a new resource locally\n- `resource push ` - push a local resource spec. This overrides any remote versions.\n\n### resource-type\n\nresource type related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `resource-type list` - list all resource types\n - `--schema` - Show schema in the output\n - `--json` - Output as JSON (for piping to jq)\n- `resource-type get ` - get a resource type's details\n - `--json` - Output as JSON (for piping to jq)\n- `resource-type new ` - create a new resource type locally\n- `resource-type push ` - push a local resource spec. This overrides any remote versions.\n- `resource-type generate-namespace` - Create a TypeScript definition file with the RT namespace generated from the resource types\n\n### schedule\n\nschedule related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `schedule list` - list all schedules\n - `--json` - Output as JSON (for piping to jq)\n- `schedule get ` - get a schedule's details\n - `--json` - Output as JSON (for piping to jq)\n- `schedule new ` - create a new schedule locally\n- `schedule push ` - push a local schedule spec. This overrides any remote versions.\n- `schedule enable ` - Enable a schedule\n - `--force` - Bypass the fork-conflict warning when the parent workspace has the same schedule (acknowledges that both crons will fire)\n- `schedule disable ` - Disable a schedule\n- `schedule set-permissioned-as ` - Set the email (run-as user) for a schedule (requires admin or wm_deployers group)\n\n### script\n\nscript related commands\n\n**Options:**\n- `--show-archived` - Show archived scripts instead of active ones\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `script list` - list all scripts\n - `--show-archived` - Show archived scripts instead of active ones\n - `--json` - Output as JSON (for piping to jq)\n- `script push ` - push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh)\n - `--message ` - Deployment message\n- `script get ` - get a script's details\n - `--json` - Output as JSON (for piping to jq)\n- `script show ` - show a script's content (alias for get)\n- `script run ` - run a script by path\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not output anything other then the final output. Useful for scripting.\n- `script preview ` - preview a local script without deploying it. Supports both regular and codebase scripts.\n - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-.\n - `-s --silent` - Do not output anything other than the final output. Useful for scripting.\n- `script new ` - create a new script\n - `--summary ` - script summary\n - `--description ` - script description\n- `script bootstrap ` - create a new script (alias for new)\n - `--summary ` - script summary\n - `--description ` - script description\n- `script set-permissioned-as ` - Set the on_behalf_of_email for a script (requires admin or wm_deployers group)\n- `script history ` - show version history for a script\n - `--json` - Output as JSON (for piping to jq)\n\n### sync\n\nsync local with a remote workspaces or the opposite (push or pull)\n\n**Subcommands:**\n\n- `sync pull` - Pull any remote changes and apply them locally.\n - `--yes` - Pull without needing confirmation\n - `--dry-run` - Show changes that would be pulled without actually pushing\n - `--plain-secrets` - Pull secrets as plain text\n - `--json` - Use JSON instead of YAML\n - `--skip-variables` - Skip syncing variables (including secrets)\n - `--skip-secrets` - Skip syncing only secrets variables\n - `--include-secrets` - Include secrets in sync (overrides skipSecrets in wmill.yaml)\n - `--skip-resources` - Skip syncing resources\n - `--skip-resource-types` - Skip syncing resource types\n - `--skip-scripts` - Skip syncing scripts\n - `--skip-flows` - Skip syncing flows\n - `--skip-apps` - Skip syncing apps\n - `--skip-folders` - Skip syncing folders\n - `--skip-workspace-dependencies` - Skip syncing workspace dependencies\n - `--skip-scripts-metadata` - Skip syncing scripts metadata, focus solely on logic\n - `--include-schedules` - Include syncing schedules\n - `--include-triggers` - Include syncing triggers\n - `--include-users` - Include syncing users\n - `--include-groups` - Include syncing groups\n - `--include-settings` - Include syncing workspace settings\n - `--include-key` - Include workspace encryption key\n - `--skip-branch-validation` - Skip git branch validation and prompts\n - `--json-output` - Output results in JSON format\n - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes\n - `-e --excludes ` - Comma separated patterns to specify which file to NOT take into account. Overrides wmill.yaml excludes\n - `--extra-includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy\n - `--repository ` - Specify repository path (e.g., u/user/repo) when multiple repositories exist\n - `--promotion ` - Use promotionOverrides from the specified branch instead of regular overrides\n - `--branch, --env ` - [Deprecated: use --workspace] Override the current git branch/environment\n- `sync push` - Push any local changes and apply them remotely.\n - `--yes` - Push without needing confirmation\n - `--dry-run` - Show changes that would be pushed without actually pushing\n - `--plain-secrets` - Push secrets as plain text\n - `--json` - Use JSON instead of YAML\n - `--skip-variables` - Skip syncing variables (including secrets)\n - `--skip-secrets` - Skip syncing only secrets variables\n - `--include-secrets` - Include secrets in sync (overrides skipSecrets in wmill.yaml)\n - `--skip-resources` - Skip syncing resources\n - `--skip-resource-types` - Skip syncing resource types\n - `--skip-scripts` - Skip syncing scripts\n - `--skip-flows` - Skip syncing flows\n - `--skip-apps` - Skip syncing apps\n - `--skip-folders` - Skip syncing folders\n - `--skip-workspace-dependencies` - Skip syncing workspace dependencies\n - `--skip-scripts-metadata` - Skip syncing scripts metadata, focus solely on logic\n - `--include-schedules` - Include syncing schedules\n - `--include-triggers` - Include syncing triggers\n - `--include-users` - Include syncing users\n - `--include-groups` - Include syncing groups\n - `--include-settings` - Include syncing workspace settings\n - `--include-key` - Include workspace encryption key\n - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.\n - `--skip-branch-validation` - Skip git branch validation and prompts\n - `--json-output` - Output results in JSON format\n - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)\n - `-e --excludes ` - Comma separated patterns to specify which file to NOT take into account.\n - `--extra-includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy\n - `--message ` - Include a message that will be added to all scripts/flows/apps updated during this push\n - `--parallel ` - Number of changes to process in parallel\n - `--repository ` - Specify repository path (e.g., u/user/repo) when multiple repositories exist\n - `--branch, --env ` - [Deprecated: use --workspace] Override the current git branch/environment\n - `--lint` - Run lint validation before pushing\n - `--locks-required` - Fail if scripts or flow inline scripts that need locks have no locks\n - `--auto-metadata` - Automatically regenerate stale metadata (locks and schemas) before pushing\n - `--accept-overriding-permissioned-as-with-self` - Accept that items with a different permissioned_as will be updated with your own user\n\n### token\n\nManage API tokens\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `token list` - List API tokens\n - `--json` - Output as JSON (for piping to jq)\n- `token create` - Create a new API token\n - `--label ` - Token label\n - `--expiration ` - Token expiration (ISO 8601 timestamp)\n- `token delete ` - Delete a token by its prefix\n\n### trigger\n\ntrigger related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `trigger list` - list all triggers\n - `--json` - Output as JSON (for piping to jq)\n- `trigger get ` - get a trigger's details\n - `--json` - Output as JSON (for piping to jq)\n - `--kind ` - Trigger kind (http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email). Recommended for faster lookup\n- `trigger new ` - create a new trigger locally\n - `--kind ` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email)\n- `trigger push ` - push a local trigger spec. This overrides any remote versions.\n- `trigger set-permissioned-as ` - Set the email (run-as user) for a trigger (requires admin or wm_deployers group)\n - `--kind ` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email)\n\n### user\n\nuser related commands\n\n**Subcommands:**\n\n- `user add [password:string]` - Create a user\n - `--superadmin` - Specify to make the new user superadmin.\n - `--company ` - Specify to set the company of the new user.\n - `--name ` - Specify to set the name of the new user.\n- `user remove ` - Delete a user\n- `user create-token` - Create a new API token for the authenticated user\n - `--email ` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.\n - `--password ` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.\n\n### variable\n\nvariable related commands\n\n**Options:**\n- `--json` - Output as JSON (for piping to jq)\n\n**Subcommands:**\n\n- `variable list` - list all variables\n - `--json` - Output as JSON (for piping to jq)\n- `variable get ` - get a variable's details\n - `--json` - Output as JSON (for piping to jq)\n- `variable new ` - create a new variable locally\n- `variable push ` - Push a local variable spec. This overrides any remote versions.\n - `--plain-secrets` - Push secrets as plain text\n- `variable add ` - Create a new variable on the remote. This will update the variable if it already exists.\n - `--yes` - Skip confirmation prompt when updating an existing variable\n - `--secret` - Mark the variable as secret (default when creating a new variable)\n - `--no-secret` - Mark the variable as non-secret (when updating, the existing setting is preserved if neither --secret nor --no-secret is passed)\n - `--description ` - Set the variable description (when updating, the existing description is preserved if not passed)\n - `--plain-secrets` - Push secrets as plain text\n - `--public` - Legacy option, use --no-secret instead\n\n### version\n\nShow version information\n\n### worker-groups\n\ndisplay worker groups, pull and push worker groups configs\n\n**Subcommands:**\n\n- `worker-groups pull` - Pull worker groups (similar to `wmill instance pull --skip-users --skip-settings --skip-groups`)\n - `--instance` - Name of the instance to push to, override the active instance\n - `--base-url` - Base url to be passed to the instance settings instead of the local one\n - `--yes` - Pull without needing confirmation\n- `worker-groups push` - Push worker groups (similar to `wmill instance push --skip-users --skip-settings --skip-groups`)\n - `--instance [instance]` - Name of the instance to push to, override the active instance\n - `--base-url [baseUrl]` - If used with --token, will be used as the base url for the instance\n - `--yes` - Push without needing confirmation\n\n### workers\n\nList all workers grouped by worker groups\n\n**Options:**\n- `--instance [instance]` - Name of the instance to push to, override the active instance\n- `--base-url [baseUrl]` - If used with --token, will be used as the base url for the instance\n\n### workspace\n\nworkspace related commands\n\n**Alias:** `profile`\n\n**Subcommands:**\n\n- `workspace switch ` - Switch to another workspace\n- `workspace add [workspace_name:string] [workspace_id:string] [remote:string]` - Add a workspace\n - `-c --create` - Create the workspace if it does not exist\n - `--create-workspace-name ` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.\n - `--create-username ` - Specify your own username in the newly created workspace. Ignored if --create is not specified, the workspace already exists or automatic username creation is enabled on the instance.\n- `workspace remove ` - Remove a workspace\n- `workspace whoami` - Show the currently active user\n- `workspace list` - List local workspace profiles\n- `workspace list-remote` - List workspaces on the remote server that you have access to\n - `--as-superadmin` - List ALL workspaces on the instance (requires the token to belong to a superadmin/devops user)\n- `workspace list-forks` - List forked workspaces on the remote server\n- `workspace bind` - Create or update a workspace entry in wmill.yaml from the active profile\n - `--workspace ` - Workspace name (default: current branch or workspaceId)\n - `--branch ` - Git branch to associate (default: workspace name)\n- `workspace unbind` - Remove baseUrl and workspaceId from a workspace entry\n - `--workspace ` - Workspace to unbind\n- `workspace fork [workspace_name:string] [workspace_id:string]` - Create a forked workspace\n - `--create-workspace-name ` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.\n - `--color ` - Workspace color (hex code, e.g. #ff0000)\n - `--datatable-behavior ` - How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt)\n - `--from-branch ` - Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on (its bound workspace is the parent) and rename the current branch onto wm-fork//. Usually unneeded \u2014 from a working branch `wmill workspace fork` offers this interactively; from a base branch it creates a fresh fork branch.\n - `-y --yes` - Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for.\n- `workspace delete-fork ` - Delete a forked workspace\n - `-y --yes` - Skip confirmation prompt\n- `workspace merge` - Compare and deploy changes between a fork and its parent workspace\n - `--direction ` - Deploy direction: to-parent or to-fork\n - `--all` - Deploy all changed items including conflicts\n - `--skip-conflicts` - Skip items modified in both workspaces\n - `--include ` - Comma-separated kind:path items to include (e.g. script:f/test/main,flow:f/my/flow)\n - `--exclude ` - Comma-separated kind:path items to exclude\n - `--preserve-on-behalf-of` - Preserve original on_behalf_of/permissioned_as values\n - `-y --yes` - Non-interactive mode (deploy without prompts)\n- `workspace connect-slack` - Non-interactively connect Slack to the active workspace using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: workspace_settings fields, g/slack group, f/slack_bot folder, and the encrypted bot token variable + resource at f/slack_bot/bot_token.\n - `--bot-token ` - Slack bot token (xoxb-...)\n - `--team-id ` - Slack team id\n - `--team-name ` - Slack team name\n- `workspace disconnect-slack` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group \u2014 delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override \u2014 set slack_oauth_client_id/_secret to '' in settings.yaml and push.\n\n\n\n# Object Storage CLI\n\n`wmill object-storage` (alias `wmill s3`) exposes the workspace's object storage (S3-compatible: AWS S3, MinIO, GCS, R2, Azure Blob) over the per-workspace `/job_helpers/*` endpoints.\n\n## Key concepts (not obvious from per-command --help)\n\n- **`file_key` is the path inside the bucket** (e.g. `reports/2026-05/orders.csv`), not a Windmill path. Do NOT pass `u/...` or `f/...` here \u2014 those are Windmill paths to scripts/flows/resources, unrelated to objects in the bucket.\n- **Scope is the active workspace.** Object storage is configured per-workspace (default storage + optional secondary storages). Switching workspaces switches which bucket the commands target.\n- **`--storage ` targets a secondary storage** configured on the workspace. Omit it to use the workspace's default object storage. Use `wmill object-storage list` to discover configured storages.\n- **`preview` vs `download`**: `preview` returns a peek (CSV first rows, text content, or a byte slice via `--bytes-from`/`--bytes-length`) without writing to disk. Use `download` when you want the full file on disk.\n\n## Choosing a subcommand\n\n- Look at what's there: `wmill object-storage files [prefix]` (alias `ls`) \u2014 paginated, use `--marker` to continue.\n- Inspect one file: `wmill object-storage info ` for size/mime/last-modified, `wmill object-storage preview ` for content peek.\n- Move data in: `wmill object-storage upload ` \u2014 set `--content-type` if the receiver cares (e.g. `text/csv`).\n- Move data out: `wmill object-storage download [output_path]` \u2014 `--stdout` to pipe.\n- Reorganize: `wmill object-storage move ` (same storage), `wmill object-storage delete ` (interactive confirm unless `--yes`).\n"; -export declare const LANG_ANSIBLE = "# Ansible\n\nWindmill runs Ansible playbooks with `ansible-playbook`. A script is a single YAML\ndocument made of two parts separated by a `---` line: a Windmill **header** and one or\nmore standard Ansible **plays**.\n\n## Structure\n\n```yaml\n---\n# Windmill header: configures inventories, file resources, arguments and dependencies\nextra_vars:\n world_qualifier:\n type: string\ndependencies:\n galaxy:\n collections:\n - name: community.general\n python:\n - jmespath\n---\n# Standard Ansible plays\n- name: Echo\n hosts: 127.0.0.1\n connection: local\n tasks:\n - name: Print debug message\n debug:\n msg: \"Hello, {{ world_qualifier }} world!\"\n```\n\n## Header\n\nThe header is **not** standard Ansible \u2014 it is parsed by Windmill to build the script's\ninputs and runtime environment. Supported keys:\n\n- `extra_vars`: defines the script arguments. Each entry is passed to the playbook via\n `--extra-vars` and becomes a Jinja variable usable as `{{ name }}` in the plays. Give\n each argument a `type` (`string`, `number`, `boolean`, `object`, ...) so Windmill can\n generate the input form.\n- `inventory`: lists inventories. Use `resource_type: ansible_inventory` (optionally\n pinned with `resource: u/user/your_resource`) or `resource_type: dynamic_inventory`.\n- `files`: writes Windmill resources/variables to files before the run, e.g.\n `- resource: u/user/template` with `target: ./config.j2`, or\n `- variable: u/user/ssh_key` with `target: ./ssh_key` and `mode: '0600'`.\n- `dependencies`: `galaxy` collections/roles (installed with `ansible-galaxy`) and\n `python` pip packages available to the playbook.\n- `options`: extra `ansible-playbook` flags such as `- verbosity: vvv`.\n- `vault_password`: a Windmill variable path to use as the Ansible Vault password.\n\n## Arguments\n\nReference header `extra_vars` directly as Jinja variables in the plays:\n\n```yaml\nextra_vars:\n name:\n type: string\n count:\n type: number\n---\n- hosts: localhost\n tasks:\n - debug:\n msg: \"{{ name }} x {{ count }}\"\n```\n\n## Environment variables\n\nWindmill contextual variables are available as environment variables and read with the\n`env` lookup:\n\n```yaml\n- debug:\n msg: \"Running in workspace {{ lookup('env', 'WM_WORKSPACE') }}\"\n```\n\n## Output\n\nTo return a result, write JSON to a `result.json` file in the job directory:\n\n```yaml\n- hosts: localhost\n tasks:\n - name: Write result\n copy:\n content: \"{{ { 'ok': true, 'value': 42 } | to_json }}\"\n dest: result.json\n```\n"; -export declare const LANG_BASH = "# Bash\n\n## Structure\n\nDo not include `#!/bin/bash`. Arguments are obtained as positional parameters:\n\n```bash\n# Get arguments\nvar1=\"$1\"\nvar2=\"$2\"\n\necho \"Processing $var1 and $var2\"\n\n# Return JSON by echoing to stdout\necho \"{\\\"result\\\": \\\"$var1\\\", \\\"count\\\": $var2}\"\n```\n\n**Important:**\n- Do not include shebang (`#!/bin/bash`)\n- Arguments are always strings\n- Access with `$1`, `$2`, etc.\n\n## Output\n\nThe script output is captured as the result. For structured data, output valid JSON:\n\n```bash\nname=\"$1\"\ncount=\"$2\"\n\n# Output JSON result\ncat << EOF\n{\n \"name\": \"$name\",\n \"count\": $count,\n \"timestamp\": \"$(date -Iseconds)\"\n}\nEOF\n```\n\n## Environment Variables\n\nEnvironment variables set in Windmill are available:\n\n```bash\n# Access environment variable\necho \"Workspace: $WM_WORKSPACE\"\necho \"Job ID: $WM_JOB_ID\"\n```\n"; -export declare const LANG_BIGQUERY = "# BigQuery\n\nArguments use `@name` syntax.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- @name1 (string)\n-- @name2 (int64) = 0\nSELECT * FROM users WHERE name = @name1 AND age > @name2;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as a `STRING` JSON parameter \u2014 Parquet/CSV\nfiles are decoded server-side into a JSON array of records, JSON/JSONL pass\nthrough. Consume with `JSON_EXTRACT_ARRAY` / `JSON_VALUE`:\n\n```sql\n-- @file (s3object)\nSELECT\n CAST(JSON_VALUE(row, '$.id') AS INT64) AS id,\n JSON_VALUE(row, '$.name') AS name\nFROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row;\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered, bypassing the 10000-row return cap.\n"; -export declare const LANG_BUN = "# TypeScript (Bun)\n\nBun runtime with full npm ecosystem and fastest execution.\n\n## Structure\n\nExport a single **async** function called `main`:\n\n```typescript\nexport async function main(param1: string, param2: number) {\n // Your code here\n return { result: param1, count: param2 };\n}\n```\n\nDo not call the main function. Libraries are installed automatically.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nUse the `RT` namespace for resource types:\n\n```typescript\nexport async function main(stripe: RT.Stripe) {\n // stripe contains API key and config from the resource\n}\n```\n\nOnly use resource types if you need them to satisfy the instructions. Always use the RT namespace.\n\nBefore using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`.\n\n## Imports\n\n```typescript\nimport Stripe from \"stripe\";\nimport { someFunction } from \"some-package\";\n```\n\n## Prefer `//native` when the runtime allows it\n\nIf a script only needs `fetch` and the JavaScript standard library \u2014 including when it uses `windmill-client` \u2014 prefer making it a **native** script: add `//native` as the first line and write it with the `write-script-bunnative` skill. Native scripts run on a lightweight V8 isolate, start faster, and parallelize heavily. `windmill-client` works on the native worker (its calls go over `fetch`), so needing the Windmill client is **not** a reason to avoid `//native`. Use the regular `bun` language only when the code (or a dependency) needs Node/Bun runtime APIs \u2014 `node:*` modules, the filesystem, child processes, or native addons.\n\n## Windmill Client\n\nImport the windmill client for platform interactions:\n\n```typescript\nimport * as wmill from \"windmill-client\";\n```\n\n**Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** \u2014 reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill.\n\nThe full `windmill-client` API reference (every exported function and its signature) is included in this skill below \u2014 consult it for the exact method to use instead of guessing or falling back to `fetch`.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```typescript\ntype Event = {\n kind:\n | \"webhook\"\n | \"http\"\n | \"websocket\"\n | \"kafka\"\n | \"email\"\n | \"nats\"\n | \"postgres\"\n | \"sqs\"\n | \"mqtt\"\n | \"gcp\";\n body: any;\n headers: Record;\n query: Record;\n};\n\nexport async function preprocessor(event: Event) {\n return {\n param1: event.body.field1,\n param2: event.query.id,\n };\n}\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form \u2014 always use it instead of redefining your own.\n\n### Receiving an S3Object as a script parameter\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\nexport async function main(file: wmill.S3Object) {\n const content = await wmill.loadS3File(file);\n // ...\n}\n```\n\n### S3 operations\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\n// Load file content from S3\nconst content: Uint8Array = await wmill.loadS3File(s3object);\n\n// Load file as stream\nconst blob: Blob = await wmill.loadS3FileStream(s3object);\n\n// Write file to S3\nconst result: wmill.S3Object = await wmill.writeS3File(\n s3object, // Target path (or undefined to auto-generate)\n fileContent, // string or Blob\n s3ResourcePath // Optional: specific S3 resource to use\n);\n```\n"; -export declare const LANG_BUNNATIVE = "# TypeScript (Bun Native)\n\nNative TypeScript execution. Native scripts are Bun scripts that run on the native worker \u2014 a lightweight V8 isolate that exposes `fetch` and the JavaScript standard library \u2014 and can be heavily parallelized. Every script MUST start with `//native` on its first line so Windmill routes it to the native worker; without it the exact same script runs on the regular Bun worker. You may import npm packages and other Windmill scripts (e.g. `./helper.ts`) \u2014 imports are resolved and bundled just like a regular Bun script \u2014 as long as everything (your code and its dependencies) relies only on `fetch` and the standard library. Libraries that need Node/Bun runtime APIs (filesystem, `node:*` modules, child processes, native addons) will not work on the native worker; use the regular `bun` language for those.\n\n## Structure\n\nExport a single **async** function called `main`:\n\n```typescript\n//native\nexport async function main(param1: string, param2: number) {\n // Your code here\n return { result: param1, count: param2 };\n}\n```\n\nDo not call the main function.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nUse the `RT` namespace for resource types:\n\n```typescript\n//native\nexport async function main(stripe: RT.Stripe) {\n // stripe contains API key and config from the resource\n}\n```\n\nOnly use resource types if you need them to satisfy the instructions. Always use the RT namespace.\n\nBefore using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`.\n\n## Imports\n\n**The constraint is the runtime, not the import list.** You may import npm packages and relative Windmill scripts; they are resolved and bundled exactly like a regular Bun script. But the native worker only provides `fetch` and the JavaScript standard library, so any imported code must work using only those. Anything requiring Node/Bun built-ins (`node:fs`, `child_process`, the `Bun` API, native modules) belongs in a regular `bun` script instead. Use the globally available `fetch` for HTTP:\n\n```typescript\n//native\nexport async function main(url: string) {\n const response = await fetch(url);\n return await response.json();\n}\n```\n\n## Windmill Client\n\n`windmill-client` works on the native worker (its calls go over `fetch`), so use it as the **preferred way to talk to Windmill** \u2014 reading resources/variables/states, running scripts and flows, and the S3 helpers below (`loadS3File`, `loadS3FileStream`, `writeS3File`, `S3Object`). It handles auth, the workspace, and the base URL for you. Reserve raw `fetch` for calling *external* HTTP APIs that aren't Windmill.\n\nThe full `windmill-client` API reference (every exported function and its signature) is included in this skill below \u2014 consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```typescript\n//native\ntype Event = {\n kind:\n | \"webhook\"\n | \"http\"\n | \"websocket\"\n | \"kafka\"\n | \"email\"\n | \"nats\"\n | \"postgres\"\n | \"sqs\"\n | \"mqtt\"\n | \"gcp\";\n body: any;\n headers: Record;\n query: Record;\n};\n\nexport async function preprocessor(event: Event) {\n return {\n param1: event.body.field1,\n param2: event.query.id,\n };\n}\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form \u2014 always use it instead of redefining your own.\n\n### Receiving an S3Object as a script parameter\n\n```typescript\n//native\nimport * as wmill from \"windmill-client\";\n\nexport async function main(file: wmill.S3Object) {\n const content = await wmill.loadS3File(file);\n // ...\n}\n```\n\n### S3 operations\n\n```typescript\n//native\nimport * as wmill from \"windmill-client\";\n\n// Load file content from S3\nconst content: Uint8Array = await wmill.loadS3File(s3object);\n\n// Load file as stream\nconst blob: Blob = await wmill.loadS3FileStream(s3object);\n\n// Write file to S3\nconst result: wmill.S3Object = await wmill.writeS3File(\n s3object, // Target path (or undefined to auto-generate)\n fileContent, // string or Blob\n s3ResourcePath // Optional: specific S3 resource to use\n);\n```\n"; -export declare const LANG_CSHARP = "# C#\n\nThe script must contain a public static `Main` method inside a class:\n\n```csharp\npublic class Script\n{\n public static object Main(string name, int count)\n {\n return new { Name = name, Count = count };\n }\n}\n```\n\n**Important:**\n- Class name is irrelevant\n- Method must be `public static`\n- Return type can be `object` or specific type\n\n## NuGet Packages\n\nAdd packages using the `#r` directive at the top:\n\n```csharp\n#r \"nuget: Newtonsoft.Json, 13.0.3\"\n#r \"nuget: RestSharp, 110.2.0\"\n\nusing Newtonsoft.Json;\nusing RestSharp;\n\npublic class Script\n{\n public static object Main(string url)\n {\n var client = new RestClient(url);\n var request = new RestRequest();\n var response = client.Get(request);\n return JsonConvert.DeserializeObject(response.Content);\n }\n}\n```\n"; -export declare const LANG_DENO = "# TypeScript (Deno)\n\nDeno runtime with npm support via `npm:` prefix and native Deno libraries.\n\n## Structure\n\nExport a single **async** function called `main`:\n\n```typescript\nexport async function main(param1: string, param2: number) {\n // Your code here\n return { result: param1, count: param2 };\n}\n```\n\nDo not call the main function. Libraries are installed automatically.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nUse the `RT` namespace for resource types:\n\n```typescript\nexport async function main(stripe: RT.Stripe) {\n // stripe contains API key and config from the resource\n}\n```\n\nOnly use resource types if you need them to satisfy the instructions. Always use the RT namespace.\n\nBefore using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`.\n\n## Imports\n\n```typescript\n// npm packages use npm: prefix\nimport Stripe from \"npm:stripe\";\nimport { someFunction } from \"npm:some-package\";\n\n// Deno standard library\nimport { serve } from \"https://deno.land/std/http/server.ts\";\n```\n\n## Windmill Client\n\nImport the windmill client for platform interactions:\n\n```typescript\nimport * as wmill from \"windmill-client\";\n```\n\n**Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** \u2014 reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill.\n\nThe full `windmill-client` API reference (every exported function and its signature) is included in this skill below \u2014 consult it for the exact method instead of guessing or falling back to `fetch`.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```typescript\ntype Event = {\n kind:\n | \"webhook\"\n | \"http\"\n | \"websocket\"\n | \"kafka\"\n | \"email\"\n | \"nats\"\n | \"postgres\"\n | \"sqs\"\n | \"mqtt\"\n | \"gcp\";\n body: any;\n headers: Record;\n query: Record;\n};\n\nexport async function preprocessor(event: Event) {\n return {\n param1: event.body.field1,\n param2: event.query.id,\n };\n}\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form \u2014 always use it instead of redefining your own.\n\n### Receiving an S3Object as a script parameter\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\nexport async function main(file: wmill.S3Object) {\n const content = await wmill.loadS3File(file);\n // ...\n}\n```\n\n### S3 operations\n\n```typescript\nimport * as wmill from \"windmill-client\";\n\n// Load file content from S3\nconst content: Uint8Array = await wmill.loadS3File(s3object);\n\n// Load file as stream\nconst blob: Blob = await wmill.loadS3FileStream(s3object);\n\n// Write file to S3\nconst result: wmill.S3Object = await wmill.writeS3File(\n s3object, // Target path (or undefined to auto-generate)\n fileContent, // string or Blob\n s3ResourcePath // Optional: specific S3 resource to use\n);\n```\n"; -export declare const LANG_DUCKDB = "# DuckDB\n\nArguments are defined with comments and used with `$name` syntax:\n\n```sql\n-- $name (text) = default\n-- $age (integer)\nSELECT * FROM users WHERE name = $name AND age > $age;\n```\n\n## Ducklake Integration\n\nAttach Ducklake for data lake operations:\n\n```sql\n-- Main ducklake\nATTACH 'ducklake' AS dl;\n\n-- Named ducklake\nATTACH 'ducklake://my_lake' AS dl;\n\n-- Then query\nSELECT * FROM dl.schema.table;\n```\n\n## External Database Connections\n\nConnect to external databases using resources:\n\n```sql\nATTACH '$res:path/to/resource' AS db (TYPE postgres);\nSELECT * FROM db.schema.table;\n```\n\n## S3 File Operations\n\nRead files from S3 storage:\n\n```sql\n-- Default storage\nSELECT * FROM read_csv('s3:///path/to/file.csv');\n\n-- Named storage\nSELECT * FROM read_csv('s3://storage_name/path/to/file.csv');\n\n-- Parquet files\nSELECT * FROM read_parquet('s3:///path/to/file.parquet');\n\n-- JSON files\nSELECT * FROM read_json('s3:///path/to/file.json');\n```\n\n### Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for it\nand binds the arg as the bare `s3://storage/key` URI, which DuckDB's reader\nfunctions consume directly:\n\n```sql\n-- $file (s3object)\nSELECT * FROM read_parquet($file);\n```\n\nWorks with any DuckDB reader: `read_csv($file)`, `read_json($file)`, etc.\n\n### Writing query results to S3\n\nDuckDB writes to S3 natively via `COPY ... TO`:\n\n```sql\nCOPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET);\n```\n\nUse this instead of the `-- s3` streaming directive supported by the other SQL\ndialects \u2014 that directive is not available in DuckDB.\n"; -export declare const LANG_GO = "# Go\n\n## Structure\n\nThe file package must be `inner` and export a function called `main`:\n\n```go\npackage inner\n\nfunc main(param1 string, param2 int) (map[string]interface{}, error) {\n return map[string]interface{}{\n \"result\": param1,\n \"count\": param2,\n }, nil\n}\n```\n\n**Important:**\n- Package must be `inner`\n- Return type must be `({return_type}, error)`\n- Function name is `main` (lowercase)\n\n## Return Types\n\nThe return type can be any Go type that can be serialized to JSON:\n\n```go\npackage inner\n\ntype Result struct {\n Name string `json:\"name\"`\n Count int `json:\"count\"`\n}\n\nfunc main(name string, count int) (Result, error) {\n return Result{\n Name: name,\n Count: count,\n }, nil\n}\n```\n\n## Error Handling\n\nReturn errors as the second return value:\n\n```go\npackage inner\n\nimport \"errors\"\n\nfunc main(value int) (string, error) {\n if value < 0 {\n return \"\", errors.New(\"value must be positive\")\n }\n return \"success\", nil\n}\n```\n"; -export declare const LANG_GRAPHQL = "# GraphQL\n\n## Structure\n\nWrite GraphQL queries or mutations. Arguments can be added as query parameters:\n\n```graphql\nquery GetUser($id: ID!) {\n user(id: $id) {\n id\n name\n email\n }\n}\n```\n\n## Variables\n\nVariables are passed as script arguments and automatically bound to the query:\n\n```graphql\nquery SearchProducts($query: String!, $limit: Int = 10) {\n products(search: $query, first: $limit) {\n edges {\n node {\n id\n name\n price\n }\n }\n }\n}\n```\n\n## Mutations\n\n```graphql\nmutation CreateUser($input: CreateUserInput!) {\n createUser(input: $input) {\n id\n name\n createdAt\n }\n}\n```\n"; -export declare const LANG_JAVA = "# Java\n\nThe script must contain a Main public class with a `public static main()` method:\n\n```java\npublic class Main {\n public static Object main(String name, int count) {\n java.util.Map result = new java.util.HashMap<>();\n result.put(\"name\", name);\n result.put(\"count\", count);\n return result;\n }\n}\n```\n\n**Important:**\n- Class must be named `Main`\n- Method must be `public static Object main(...)`\n- Return type is `Object` or `void`\n\n## Maven Dependencies\n\nAdd dependencies using comments at the top:\n\n```java\n//requirements:\n//com.google.code.gson:gson:2.10.1\n//org.apache.httpcomponents:httpclient:4.5.14\n\nimport com.google.gson.Gson;\n\npublic class Main {\n public static Object main(String input) {\n Gson gson = new Gson();\n return gson.fromJson(input, Object.class);\n }\n}\n```\n"; -export declare const LANG_MSSQL = "# Microsoft SQL Server (MSSQL)\n\nArguments use `@P1`, `@P2`, etc.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- @P1 name1 (varchar)\n-- @P2 name2 (int) = 0\nSELECT * FROM users WHERE name = @P1 AND age > @P2;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as `nvarchar(max)` JSON text \u2014 Parquet/CSV\nfiles are decoded server-side into a JSON array of records, JSON/JSONL pass\nthrough. Consume with `OPENJSON`:\n\n```sql\n-- @P1 file (s3object)\nSELECT id, name\nFROM OPENJSON(@P1)\nWITH (id INT, name NVARCHAR(200));\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered as the script return value.\n"; -export declare const LANG_MYSQL = "# MySQL\n\nArguments use `?` placeholders.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- ? name1 (text)\n-- ? name2 (int) = 0\nSELECT * FROM users WHERE name = ? AND age > ?;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as JSON text \u2014 Parquet/CSV files are\ndecoded server-side into a JSON array of records, JSON/JSONL pass through.\nConsume with `JSON_TABLE`:\n\n```sql\n-- ? file (s3object)\nSELECT id, name\nFROM JSON_TABLE(?, '$[*]'\n COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name')\n) AS r;\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered as the script return value.\n"; -export declare const LANG_PHP = "# PHP\n\n## Structure\n\nThe script must start with ` $param1, \"count\" => $param2];\n}\n```\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nYou need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using `class_exists`:\n\n```php\n $2::INT;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as a `jsonb` parameter \u2014 Parquet/CSV files\nare decoded server-side into a JSON array of records, JSON/JSONL pass through.\nConsume with `jsonb_to_recordset` (or any `jsonb` API):\n\n```sql\n-- $1 file (s3object)\nSELECT *\nFROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT);\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered as the script return value.\n"; -export declare const LANG_POWERSHELL = "# PowerShell\n\n## Structure\n\nArguments are obtained by calling the `param` function on the first line:\n\n```powershell\nparam($Name, $Count = 0, [int]$Age)\n\n# Your code here\nWrite-Output \"Processing $Name, count: $Count, age: $Age\"\n\n# Return object\n@{\n name = $Name\n count = $Count\n age = $Age\n}\n```\n\n## Parameter Types\n\nYou can specify types for parameters:\n\n```powershell\nparam(\n [string]$Name,\n [int]$Count = 0,\n [bool]$Enabled = $true,\n [array]$Items\n)\n\n@{\n name = $Name\n count = $Count\n enabled = $Enabled\n items = $Items\n}\n```\n\n## Return Values\n\nReturn values by outputting them at the end of the script:\n\n```powershell\nparam($Input)\n\n$result = @{\n processed = $true\n data = $Input\n timestamp = Get-Date -Format \"o\"\n}\n\n$result\n```\n"; -export declare const LANG_PYTHON3 = "# Python\n\n## Structure\n\nThe script must contain at least one function called `main`:\n\n```python\ndef main(param1: str, param2: int):\n # Your code here\n return {\"result\": param1, \"count\": param2}\n```\n\nDo not call the main function. Libraries are installed automatically.\n\n## Resource Types\n\nOn Windmill, credentials and configuration are stored in resources and passed as parameters to main.\n\nYou need to **redefine** the type of the resources that are needed before the main function as TypedDict:\n\n```python\nfrom typing import TypedDict\n\nclass postgresql(TypedDict):\n host: str\n port: int\n user: str\n password: str\n dbname: str\n\ndef main(db: postgresql):\n # db contains the database connection details\n pass\n```\n\n**Important rules:**\n\n- The resource type name must be **IN LOWERCASE**\n- Only include resource types if they are actually needed\n- If an import conflicts with a resource type name, **rename the imported object, not the type name**\n- Make sure to import TypedDict from typing **if you're using it**\n\n## Imports\n\nLibraries are installed automatically. Do not show installation instructions.\n\n```python\nimport requests\nimport pandas as pd\nfrom datetime import datetime\n```\n\nIf an import name conflicts with a resource type:\n\n```python\n# Wrong - don't rename the type\nimport stripe as stripe_lib\nclass stripe_type(TypedDict): ...\n\n# Correct - rename the import\nimport stripe as stripe_sdk\nclass stripe(TypedDict):\n api_key: str\n```\n\n## Windmill Client\n\nImport the windmill client for platform interactions:\n\n```python\nimport wmill\n```\n\nSee the SDK documentation for available methods.\n\n## Preprocessor Scripts\n\nFor preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter:\n\n```python\nfrom typing import TypedDict, Literal, Any\n\nclass Event(TypedDict):\n kind: Literal[\"webhook\", \"http\", \"websocket\", \"kafka\", \"email\", \"nats\", \"postgres\", \"sqs\", \"mqtt\", \"gcp\"]\n body: Any\n headers: dict[str, str]\n query: dict[str, str]\n\ndef preprocessor(event: Event):\n # Transform the event into flow input parameters\n return {\n \"param1\": event[\"body\"][\"field1\"],\n \"param2\": event[\"query\"][\"id\"]\n }\n```\n\n## S3 Object Operations\n\nWindmill provides built-in support for S3-compatible storage operations.\n\n### Receiving an S3Object as a script parameter\n\nTo accept a file from S3 as input to a script, type the parameter with `S3Object` (imported from `wmill`):\n\n```python\nimport wmill\nfrom wmill import S3Object\n\ndef main(file: S3Object):\n content = wmill.load_s3_file(file)\n # ...\n```\n\n### S3 operations\n\n```python\nimport wmill\n\n# Load file content from S3\ncontent: bytes = wmill.load_s3_file(s3object)\n\n# Load file as stream reader\nreader: BufferedReader = wmill.load_s3_file_reader(s3object)\n\n# Write file to S3\nresult: S3Object = wmill.write_s3_file(\n s3object, # Target path (or None to auto-generate)\n file_content, # bytes or BufferedReader\n s3_resource_path, # Optional: specific S3 resource\n content_type, # Optional: MIME type\n content_disposition # Optional: Content-Disposition header\n)\n```\n"; -export declare const LANG_RLANG = "# R\n\n## Structure\n\nDefine a `main` function using `<-` or `=` assignment. Parameters become the script inputs:\n\n```r\nlibrary(dplyr)\nlibrary(jsonlite)\n\nmain <- function(x, name = \"default\", flag = TRUE) {\n df <- tibble(x = x, name = name)\n result <- df %>% mutate(greeting = paste(\"Hello\", name))\n return(toJSON(result, auto_unbox = TRUE))\n}\n```\n\n**Important:**\n- The `main` function is required\n- Use `library()` to load packages \u2014 they are resolved and installed automatically\n- `jsonlite` is always available (used internally for argument parsing)\n- Return values must be JSON-serializable\n\n## Parameters\n\nR types map to Windmill types:\n- `numeric` \u2192 float/int\n- `character` \u2192 string\n- `logical` \u2192 bool (use `TRUE`/`FALSE`)\n- `list` \u2192 object/dict\n- `NULL` \u2192 null\n\nDefault values are inferred from the function signature:\n\n```r\nmain <- function(\n name, # required string\n count = 10, # optional int, default 10\n verbose = FALSE # optional bool, default FALSE\n) {\n # ...\n}\n```\n\n## Resources and Variables\n\nUse the built-in Windmill helpers (no import needed):\n\n```r\nmain <- function() {\n # Get a variable\n api_key <- get_variable(\"f/my_folder/api_key\")\n\n # Get a resource (returns a list)\n db <- get_resource(\"f/my_folder/postgres_config\")\n host <- db$host\n port <- db$port\n\n return(list(host = host, port = port))\n}\n```\n\n## Output\n\nReturn any JSON-serializable value from `main`. The return value becomes the step result:\n\n```r\nmain <- function(x) {\n # Return a scalar\n return(x + 1)\n\n # Or a list (becomes JSON object)\n return(list(result = x + 1, status = \"ok\"))\n}\n```\n\n## Annotations\n\nControl execution behavior with comment annotations:\n\n```r\n#renv_verbose = true # Show verbose renv output during resolution\n#renv_install_verbose = true # Show verbose output during package installation\n#sandbox = true # Run in nsjail sandbox (requires nsjail)\n```\n"; -export declare const LANG_RUST = "# Rust\n\n## Structure\n\nThe script must contain a function called `main` with proper return type:\n\n```rust\nuse anyhow::anyhow;\nuse serde::Serialize;\n\n#[derive(Serialize, Debug)]\nstruct ReturnType {\n result: String,\n count: i32,\n}\n\nfn main(param1: String, param2: i32) -> anyhow::Result {\n Ok(ReturnType {\n result: param1,\n count: param2,\n })\n}\n```\n\n**Important:**\n- Arguments should be owned types\n- Return type must be serializable (`#[derive(Serialize)]`)\n- Return type is `anyhow::Result`\n\n## Dependencies\n\nPackages must be specified with a partial cargo.toml at the beginning of the script:\n\n```rust\n//! ```cargo\n//! [dependencies]\n//! anyhow = \"1.0.86\"\n//! reqwest = { version = \"0.11\", features = [\"json\"] }\n//! tokio = { version = \"1\", features = [\"full\"] }\n//! ```\n\nuse anyhow::anyhow;\n// ... rest of the code\n```\n\n**Note:** Serde is already included, no need to add it again.\n\n## Async Functions\n\nIf you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside:\n\n```rust\n//! ```cargo\n//! [dependencies]\n//! anyhow = \"1.0.86\"\n//! tokio = { version = \"1\", features = [\"full\"] }\n//! reqwest = { version = \"0.11\", features = [\"json\"] }\n//! ```\n\nuse anyhow::anyhow;\nuse serde::Serialize;\n\n#[derive(Serialize, Debug)]\nstruct Response {\n data: String,\n}\n\nfn main(url: String) -> anyhow::Result {\n let rt = tokio::runtime::Runtime::new()?;\n rt.block_on(async {\n let resp = reqwest::get(&url).await?.text().await?;\n Ok(Response { data: resp })\n })\n}\n```\n"; -export declare const LANG_SNOWFLAKE = "# Snowflake\n\nArguments use `?` placeholders.\n\nName the parameters by adding comments before the statement:\n\n```sql\n-- ? name1 (text)\n-- ? name2 (number) = 0\nSELECT * FROM users WHERE name = ? AND age > ?;\n```\n\n## Receiving an S3Object as a script parameter\n\nDeclare the arg with type `(s3object)`. Windmill renders an S3 file picker for\nit, downloads the file, and binds it as JSON text \u2014 Parquet/CSV files are\ndecoded server-side into a JSON array of records, JSON/JSONL pass through.\nWrap the bind with `PARSE_JSON(?)` and walk it with `LATERAL FLATTEN`:\n\n```sql\n-- ? file (s3object)\nSELECT\n v.value:id::NUMBER AS id,\n v.value:name::STRING AS name\nFROM LATERAL FLATTEN(input => PARSE_JSON(?)) v;\n```\n\n## Streaming query results to S3\n\nAdd a `-- s3` directive at the top of the script to stream the result set to S3\ninstead of returning rows. Windmill writes the file and returns its `S3Object`\nas the script result.\n\n```sql\n-- s3 prefix=exports/users format=parquet\nSELECT id, name FROM users;\n```\n\nAll keys are optional: `prefix` (object key prefix), `storage` (named storage \u2014\nomit to use the workspace default), `format` (`json` (default), `parquet`, or\n`csv`). Use this for large result sets \u2014 rows stream directly to S3 instead of\nbeing buffered, bypassing the 10000-row return cap.\n"; +// Auto-generated by generate.py - DO NOT EDIT + +export declare const SCRIPT_BASE: string; +export declare const FLOW_BASE: string; +export declare const RESOURCES_BASE: string; +export declare const RAW_APP_BASE: string; +export declare const PIPELINE_BASE: string; +export declare const WORKFLOW_AS_CODE_BASE: string; +export declare const FLOW_CHAT_SPECIAL_MODULES: string; +export declare const SDK_TYPESCRIPT: string; +export declare const SDK_PYTHON: string; +export declare const WAC_SDK_TYPESCRIPT: string; +export declare const WAC_SDK_PYTHON: string; +export declare const DATATABLE_SDK_TYPESCRIPT: string; +export declare const DATATABLE_SDK_PYTHON: string; +export declare const OPENFLOW_SCHEMA: string; +export declare const CLI_COMMANDS: string; +export declare const LANG_ANSIBLE: string; +export declare const LANG_BASH: string; +export declare const LANG_BIGQUERY: string; +export declare const LANG_BUN: string; +export declare const LANG_BUNNATIVE: string; +export declare const LANG_CSHARP: string; +export declare const LANG_DENO: string; +export declare const LANG_DUCKDB: string; +export declare const LANG_GO: string; +export declare const LANG_GRAPHQL: string; +export declare const LANG_JAVA: string; +export declare const LANG_MSSQL: string; +export declare const LANG_MYSQL: string; +export declare const LANG_PHP: string; +export declare const LANG_POSTGRESQL: string; +export declare const LANG_POWERSHELL: string; +export declare const LANG_PYTHON3: string; +export declare const LANG_RLANG: string; +export declare const LANG_RUST: string; +export declare const LANG_SNOWFLAKE: string; From 7046dc6dfb474ef49313377855bb2bd60294e25a Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 7 Jul 2026 19:21:11 +0200 Subject: [PATCH 250/273] fix(sessions): scope fork session Edits to session-edited items only (#9989) * fix(sessions): scope fork session Edits to session-edited items only A session chat with an undefined modified-items mask fell back to showing every draft in its (possibly forked) workspace, so the Edits bar/diff drawer listed all fork drafts instead of just what the session edited. Always track session chats: seed an empty mask for legacy chats in loadPastChat and guard the not-yet-persisted-chat case in initRuntime. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: clarify session chats always persist their modified-items mask Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/chat/AIChatManager.svelte.ts | 20 +++++---- .../copilot/chat/AIChatManager.test.ts | 41 +++++++++++++++++++ .../copilot/chat/HistoryManager.svelte.ts | 13 +++--- .../sessions/sessionRuntime.svelte.ts | 5 +++ 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 9cb6106a5f..d000b8a966 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -367,10 +367,11 @@ export class AIChatManager { // Workspace items the CURRENT chat modified via AI tool calls, as // `${UserDraftItemKind}:${storagePath}` keys (see modifiedItemsMask.ts). - // undefined = untracked: the global side-panel chat (never initialised) and - // loaded legacy chats with no stored mask, both of which fall back to the - // show-all bar. A SvelteSet (even empty) = tracked. Reactive so the session - // bar updates as tools record mid-turn. + // undefined = untracked: only the global side-panel chat (never initialised), + // which falls back to the show-all bar. Session chats are always tracked (a + // SvelteSet, even empty) — see loadPastChat/initRuntime — so their Edits + // surface never claims drafts the session didn't touch. Reactive so the + // session bar updates as tools record mid-turn. modifiedItems = $state | undefined>(undefined) // Start tracking for a brand-new session chat (empty = "tracked, nothing yet"). @@ -2159,13 +2160,14 @@ export class AIChatManager { this.displayMessages = chat.displayMessages this.messages = chat.actualMessages this.contextUsage = normalizeContextUsage(chat.contextUsage) - // Seed the modified-items mask from the stored chat. A stored array - // (even empty) → tracked; a legacy chat with no field stays untracked - // (undefined) so the session bar falls back to showing all drafts. The - // global side-panel chat never tracks, so leave it untouched there. + // Seed the modified-items mask from the stored chat. A session's Edits + // surface is scoped strictly to what this session edited, so it must never + // fall back to showing every draft in the (possibly forked) workspace: a + // legacy chat with no stored mask seeds an empty tracked set, not undefined. + // The global side-panel chat never tracks, so leave it untouched there. if (this.isSessionChat) { const stored = this.historyManager.getModifiedItems(id) - this.modifiedItems = stored !== undefined ? new SvelteSet(stored) : undefined + this.modifiedItems = new SvelteSet(stored ?? []) } this.#automaticScroll = true this.onChatRotated?.(id) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index a415377a7b..4879035685 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -634,6 +634,47 @@ describe('AIChatManager queued messages', () => { await session.saveAndClear() expect(session.attachedFiles.count).toBe(1) }) + + it('tracks (empty mask) a session chat loaded with no stored modified-items', async () => { + // A legacy session chat has no persisted mask. It must NOT stay untracked + // (undefined) — that makes the Edits surface fall back to showing every + // draft in the (possibly forked) workspace. Seed an empty tracked set so the + // session only ever surfaces what it actually edited. + const manager = createManager(createInputMock()) + manager.isSessionChat = true + vi.spyOn(manager.historyManager, 'loadPastChat').mockReturnValue({ + id: 'legacy-session-chat', + title: 'Legacy', + displayMessages: [], + actualMessages: [], + lastModified: 0 + } as unknown as ReturnType) + vi.spyOn(manager.historyManager, 'getModifiedItems').mockReturnValue(undefined) + + await manager.loadPastChat('legacy-session-chat') + + expect(manager.modifiedItems).toBeInstanceOf(Set) + expect(manager.modifiedItems?.size).toBe(0) + }) + + it('seeds a session chat mask from its stored modified-items', async () => { + const manager = createManager(createInputMock()) + manager.isSessionChat = true + vi.spyOn(manager.historyManager, 'loadPastChat').mockReturnValue({ + id: 'tracked-session-chat', + title: 'Tracked', + displayMessages: [], + actualMessages: [], + lastModified: 0 + } as unknown as ReturnType) + vi.spyOn(manager.historyManager, 'getModifiedItems').mockReturnValue([ + 'script:u/admin/hello_world' + ]) + + await manager.loadPastChat('tracked-session-chat') + + expect([...(manager.modifiedItems ?? [])]).toEqual(['script:u/admin/hello_world']) + }) }) describe('AIChatManager context compaction', () => { diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 0db0751293..3d9c5d8004 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -239,12 +239,13 @@ export default class HistoryManager { lastModified: Date.now(), ...(this.sessionId ? { sessionId: this.sessionId } : {}), ...(contextUsage !== undefined ? { contextUsage } : {}), - // Only persist when the caller passes a defined array. Loaded legacy - // chats keep their accumulator undefined, so we never retroactively - // stamp them with [] (which would flip them to the filtered view). - // But since `put` replaces the whole record, a caller that omits the - // argument must not ERASE a tracked chat's stored mask — fall back to - // the previously saved field. + // Only persist when the caller passes a defined array — an untracked + // chat (the global side-panel chat, mask still undefined) must not be + // stamped with [], which would flip it to the filtered view. Session + // chats are always tracked (see AIChatManager.loadPastChat), so they do + // pass a defined array and persist it. Since `put` replaces the whole + // record, a caller that omits the argument must not ERASE a tracked + // chat's stored mask — fall back to the previously saved field. ...(modifiedItems !== undefined ? { modifiedItems } : this.savedChats[this.currentChatId]?.modifiedItems !== undefined diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index 3c6ea7cdb8..31ac8bf8e8 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -713,6 +713,11 @@ async function initRuntime(runtime: SessionRuntime, session: Session) { manager.historyManager.setCurrentChatId(session.chatId) await manager.historyManager.tagChatWithSession(session.chatId, session.id) await manager.loadPastChat(session.chatId) + // loadPastChat only seeds the mask when the chat exists in history; a chatId + // pointing at a chat not yet persisted (no turn saved) would leave it + // undefined, and the Edits surface would then show every workspace draft. + // Start tracking so this session is scoped to its own edits from the outset. + if (manager.modifiedItems === undefined) manager.initModifiedItemsTracking() } else { // Brand-new session chat: start tracking modified items now (empty mask) // so the session bar filters to this chat's changes from the first turn. From 4bb82ad6cdb62eae7b69b1054714333e54558632 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 7 Jul 2026 19:21:34 +0200 Subject: [PATCH 251/273] feat: open runs/schedules pages from AI chat in session preview tabs (#9976) * feat(copilot): open runs/schedules pages in session preview tabs Co-Authored-By: Claude Opus 4.8 (1M context) * fix(copilot): drop buggy in-place nav, always chip outside a session Co-Authored-By: Claude Opus 4.8 (1M context) * feat(copilot): open_page covers variables/resources/assets/audit-logs/settings, perm-gated Co-Authored-By: Claude Opus 4.8 (1M context) * feat(copilot): open_page adds folders, groups and all trigger kinds (EE-gated) Co-Authored-By: Claude Opus 4.8 (1M context) * feat(copilot): close_page tool to close session preview tabs Co-Authored-By: Claude Opus 4.8 (1M context) * fix(copilot): fail-closed on unavailable trigger_kind in open_page handler Co-Authored-By: Claude Opus 4.8 (1M context) * fix(copilot): gate open_page on operator_settings, keep open_preview mention preview-only Gate the open_page page set on the workspace operator_settings for operators (mirrors OperatorMenu) instead of hardcoding runs/assets, with an empty-enum guard. Also move the open_preview cross-reference out of the always-on prompt line into the preview-gated block so it isn't advertised when preview tools are off. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(copilot): gate open_page on the session's operating workspace A session chat targets its own (possibly forked) workspace while $workspaceStore stays on the navigation workspace, so operator_settings must be read for the operating workspace, not the global store. Thread it through GlobalToolHelpers so both setSchema (advertised enum) and the handler guard gate on the same workspace; the global side-panel chat still follows the live store. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ai_evals/cases/global.yaml | 201 ++++++++ .../copilot/chat/AIChatManager.svelte.ts | 7 +- .../copilot/chat/ToolMessageActions.svelte | 40 +- .../components/copilot/chat/global/core.ts | 443 +++++++++++++++++- .../chat/global/pageNavigation.test.ts | 66 +++ .../copilot/chat/global/pageNavigation.ts | 133 ++++++ .../src/lib/components/copilot/chat/shared.ts | 14 +- .../copilot/chat/workspaceItems.svelte.ts | 6 +- .../lib/components/sessions/previewRouter.ts | 50 ++ .../sessions/sessionPreviewTabs.svelte.ts | 21 + .../sessions/sessionPreviewTabs.test.ts | 32 ++ .../sessions/sessionRuntime.svelte.ts | 62 ++- frontend/src/lib/navigation.test.ts | 52 ++ frontend/src/lib/navigation.ts | 26 + frontend/src/lib/svelte5UtilsKit.svelte.ts | 2 +- .../src/routes/(root)/(logged)/+layout.svelte | 35 ++ .../(root)/(logged)/sessions/+page.svelte | 44 +- 17 files changed, 1204 insertions(+), 30 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/global/pageNavigation.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/global/pageNavigation.ts create mode 100644 frontend/src/lib/navigation.test.ts diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 66e5fd3cb7..0f7f2a4717 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -872,6 +872,207 @@ - fetches the logs for the requested job id - explains the failure from the returned logs (connection refused to the upstream API) +# --- Page navigation (open_page) --- +# The assistant should take the user to a Windmill page (Runs/Schedules) with the +# right filters via open_page, rather than describing where to click or dumping the +# data. No draft is produced, so the global judge is skipped and we validate the +# tool call and its arguments. + +- id: global-openpage1-runs-failed-of-script + prompt: |- + Take me to the failed runs of the script at f/evals/global/greet_user so I can see what's going wrong. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - open_page + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - runs + - tool: open_page + field: status + stringIncludesAnyOf: + - failure + - tool: open_page + field: path + stringIncludesAnyOf: + - f/evals/global/greet_user + skipJudge: true + judgeChecklist: + - opens the Runs page filtered to the failed runs of f/evals/global/greet_user + - applies both the failure status and the script path as filters + - does not write, deploy, or delete anything + +- id: global-openpage2-runs-of-schedule + prompt: |- + Open the runs page filtered to the jobs triggered by the schedule f/evals/global/nightly_digest. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - open_page + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - runs + - tool: open_page + field: schedule_path + stringIncludesAnyOf: + - f/evals/global/nightly_digest + skipJudge: true + judgeChecklist: + - opens the Runs page filtered to jobs triggered by the f/evals/global/nightly_digest schedule + - passes the schedule path as the filter + - does not write, deploy, or delete anything + +- id: global-openpage3-open-schedule + prompt: |- + Open the schedule f/evals/global/nightly_digest so I can review and edit it. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - open_page + forbiddenToolsUsed: + - write_schedule + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - schedules + - tool: open_page + field: open + stringIncludesAnyOf: + - f/evals/global/nightly_digest + skipJudge: true + judgeChecklist: + - opens the Schedules page and targets the f/evals/global/nightly_digest schedule for editing + - passes the schedule path so the editor opens on it + - does not write, deploy, or delete anything + +- id: global-openpage4-workspace-settings-tab + prompt: |- + Take me to the Git sync configuration for this workspace. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - workspace_settings + - tool: open_page + field: tab + stringIncludesAnyOf: + - git_sync + skipJudge: true + judgeChecklist: + - opens the Workspace settings page on the git_sync tab + - does not write, deploy, or delete anything + +- id: global-openpage5-audit-logs-user + prompt: |- + Open the audit logs filtered to actions performed by the user admin. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - audit_logs + - tool: open_page + field: username + stringIncludesAnyOf: + - admin + skipJudge: true + judgeChecklist: + - opens the Audit logs page filtered to the admin user + - does not write, deploy, or delete anything + +- id: global-openpage6-triggers-kind + prompt: |- + Take me to the Kafka triggers for this workspace. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - triggers + - tool: open_page + field: trigger_kind + stringIncludesAnyOf: + - kafka + skipJudge: true + judgeChecklist: + - opens the Kafka triggers page + - does not write, deploy, or delete anything + +- id: global-closepage1-close-runs-tab + prompt: |- + You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - close_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: close_page + field: match + stringIncludesAnyOf: + - runs + skipJudge: true + judgeChecklist: + - closes the runs preview tab in the side panel + - does not write, deploy, or delete anything + # --- Documentation search (search_docs) --- # Pure product-knowledge questions: the assistant should consult the docs via # search_docs and answer conversationally, not draft or mutate anything. No diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index d000b8a966..b455d0158e 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -1074,7 +1074,12 @@ export class AIChatManager { skills: this.globalSkills }) const baseHelpers: GlobalToolHelpers = { - ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), + // A session targets its own fixed (possibly forked) workspace, so capture it for + // permission gating. The global side-panel chat follows the live navigation + // workspace instead, so leave it unset there — allowedOpenPages reads the store. + ...(this.isSessionChat + ? { sessionId: this.sessionId, operatingWorkspace: this.operatingWorkspace } + : {}), testActiveFlow: async (args?: Record) => this.flowAiChatHelpers?.testFlow(args), attachedFiles: this.attachedFiles, getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', diff --git a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte index 0a2fd1ab79..afee03bcf2 100644 --- a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte @@ -1,15 +1,24 @@ {#snippet dock()} + badges (draft/deployed) so the bar reads at a glance. Display-only: the + enclosing Edits segment button opens the drawer. -->
{#if dockCounts.draft > 0} - + {dockCounts.draft} draft{dockCounts.draft === 1 ? '' : 's'} {/if} {#if dockCounts.deployed > 0} - + {dockCounts.deployed} deployed {/if} - {#if deletionOnly && compareHref} - - - Review deletions on compare page - - - {/if}
{/snippet} @@ -248,17 +240,59 @@ -{:else if showBar && committedId} - +{:else if committedId && (showBar || hasJobs)} +
-
- - Edits -
- {@render dock()} + {#if showBar} + {#if deletionOnly && compareHref} + + + + Edits + + Review deletions + + + {:else} + + + {/if} + {:else} + +
+ {/if} + {#if hasJobs} + + + {/if}
{/if} diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index b316d3d12b..261cc6aedc 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -1,5 +1,6 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity' import { get } from 'svelte/store' +import { base } from '$lib/base' import { AIChatManager, AIMode } from '$lib/components/copilot/chat/AIChatManager.svelte' import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' import { initFlow } from '$lib/components/flows/flowStore.svelte' @@ -352,6 +353,17 @@ function createRuntime(session: Session): SessionRuntime { setTarget: (target) => setSessionTarget(session.id, target) }) + // Let the jobs tray open a run in this session's preview panel (as an iframe + // tab over the run page). The global side-panel chat leaves this unset and + // falls back to a new browser tab. + manager.openRunInPreview = ({ jobId, workspace, label }) => { + previewTabs.open({ + type: 'page', + href: `${base}/run/${jobId}?workspace=${workspace}`, + label + }) + } + // Pipeline target state lives on the runtime (not the PipelineEditorView // component) so the in-session drafts survive hide/show of the editor pane — // the pane unmounts on hide, and a component-local store would be discarded. diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 7b7cc306c1..3989a447df 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -723,6 +723,11 @@ {@const rt = getRuntime(s.id)} {@const tabs = rt?.previewTabs} {#each tabs?.tabs ?? [] as tab (tab.id)} + + Date: Wed, 8 Jul 2026 14:01:05 +0200 Subject: [PATCH 261/273] feat: shared tab system, universal markdown code blocks, subtle scrollbars (#10003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: universal styled markdown code blocks with copy button and subtle scrollbar Co-Authored-By: Claude Opus 4.8 (1M context) * feat(sessions): use the shared DraggableTabs for the preview tab strip The session preview tabs were bespoke markup; converge them onto the same DraggableTabs component the raw-app editor uses, gaining drag-reorder and keyboard nav. The active tab keeps its breadcrumb/router picker via a new tabAccessory snippet, and tabs persist their new order. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sessions): keep the new-tab + button right after the last tab Add an afterTabs snippet to DraggableTabs that renders inside the scroll row after the tabs (unlike trailing, which stays pinned outside it), and move the session preview "+" there so it sits next to the last tab. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(tabs): use the subtle ScrollableX scrollbar for Tabs/TabsV2 headers Co-Authored-By: Claude Opus 4.8 (1M context) * style(sessions): use bg-surface for the preview tab strip Co-Authored-By: Claude Opus 4.8 (1M context) * style(tabs): add subtle shadow-sm to the selected DraggableTabs tab Co-Authored-By: Claude Opus 4.8 (1M context) * style(tabs): drop selected-tab shadow; session strip bg-surface-secondary/50 Co-Authored-By: Claude Opus 4.8 (1M context) * style(sessions): drop persistent bg on preview bar buttons, hover-only Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(scrollbar): share a .scrollbar-subtle utility across tabs and chat Extract ScrollableX's hover-revealed scrollbar styling into a global .scrollbar-subtle utility (both axes, size via --wm-scrollbar-size), have ScrollableX consume it, and apply it to the AI chat message list so the chat scrollbar matches the tabs. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: address review — scope HighlightCode copy button, plaintext unknown fences, Tailwind ScrollableX - HighlightCode: keep the subtle CopyButton + surface chip behind buttonsOnHover so the ~20 non-markdown callers keep the original light copy Button. - MarkdownCodeBlock: unlabeled/unknown fences render as plaintext instead of being mis-colored as TypeScript; added common language aliases (ts/js/py/...) so real languages still highlight. - ScrollableX: replace the custom diff --git a/frontend/src/lib/components/common/tabs/Tabs.svelte b/frontend/src/lib/components/common/tabs/Tabs.svelte index e8a98a21c2..65daedf622 100644 --- a/frontend/src/lib/components/common/tabs/Tabs.svelte +++ b/frontend/src/lib/components/common/tabs/Tabs.svelte @@ -3,6 +3,7 @@ import { writable } from 'svelte/store' import { createEventDispatcher } from 'svelte' import { twMerge } from 'tailwind-merge' + import ScrollableX from '../ScrollableX.svelte' import type { TabsContext } from '$lib/components/apps/editor/settingsPanel/inputEditor/tabs.svelte' const dispatch = createEventDispatcher<{ selected: string }>() @@ -75,10 +76,12 @@ {#if !hideTabs} -
+ +
{@render children?.({ selected })}
-
+ {/if} {@render content?.()} diff --git a/frontend/src/lib/components/common/tabs/TabsV2.svelte b/frontend/src/lib/components/common/tabs/TabsV2.svelte index 08616af90c..314bb9cd15 100644 --- a/frontend/src/lib/components/common/tabs/TabsV2.svelte +++ b/frontend/src/lib/components/common/tabs/TabsV2.svelte @@ -2,6 +2,7 @@ import { setContext, untrack } from 'svelte' import { writable } from 'svelte/store' import { twMerge } from 'tailwind-merge' + import ScrollableX from '../ScrollableX.svelte' import type { TabsContext } from '$lib/components/apps/editor/settingsPanel/inputEditor/tabs.svelte' interface Props { @@ -72,10 +73,10 @@ {#if !hideTabs} -
-
+ +
{@render children?.({ selected })}
-
+ {/if} {@render content?.()} diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index f4e2d73a44..3845c20435 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -592,7 +592,11 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#if messages.length > 0}
-
+
- diff --git a/frontend/src/lib/components/copilot/chat/script/CodeDisplay.svelte b/frontend/src/lib/components/copilot/chat/script/CodeDisplay.svelte index d0a4cba462..2be52a32d2 100644 --- a/frontend/src/lib/components/copilot/chat/script/CodeDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/script/CodeDisplay.svelte @@ -1,92 +1,17 @@ -
-
- {#if language === 'mermaid'} - - {:else} - - {/if} -
-
+ diff --git a/frontend/src/lib/components/markdownPlugins.ts b/frontend/src/lib/components/markdownPlugins.ts index 4495619868..da403aebe0 100644 --- a/frontend/src/lib/components/markdownPlugins.ts +++ b/frontend/src/lib/components/markdownPlugins.ts @@ -3,6 +3,7 @@ import { gfmPlugin } from 'svelte-exmarkdown/gfm' import rehypeRaw from 'rehype-raw' import rehypeSanitize from 'rehype-sanitize' import { rehypeGithubAlerts } from 'rehype-github-alerts' +import MarkdownCodeBlock from './MarkdownCodeBlock.svelte' /** * Shared plugin chain for rendering user-supplied Markdown (script/flow/resource @@ -19,6 +20,11 @@ import { rehypeGithubAlerts } from 'rehype-github-alerts' * injects (including inline SVG icons) is preserved * without having to allowlist SVG for user input. * + * The `pre` renderer gives fenced code blocks syntax highlighting, a copy + * button, and a subtle scrollbar everywhere this chain is used. It reads the + * `language-*` class rehypeSanitize preserves on ``; it highlights code as + * escaped text and never renders mermaid on untrusted input (chat opts in). + * * Any Markdown sink that renders user input MUST use this chain rather than * assembling its own `rehypeRaw` pipeline. */ @@ -26,5 +32,6 @@ export const markdownPlugins: Plugin[] = [ gfmPlugin(), { rehypePlugin: [rehypeRaw] }, { rehypePlugin: [rehypeSanitize] }, - { rehypePlugin: [rehypeGithubAlerts] } + { rehypePlugin: [rehypeGithubAlerts] }, + { renderer: { pre: MarkdownCodeBlock } } ] diff --git a/frontend/src/lib/components/sessions/SessionStatusDot.svelte b/frontend/src/lib/components/sessions/SessionStatusDot.svelte index 2a1b45c347..4c0ef4c408 100644 --- a/frontend/src/lib/components/sessions/SessionStatusDot.svelte +++ b/frontend/src/lib/components/sessions/SessionStatusDot.svelte @@ -47,9 +47,9 @@ {#if status === 'streaming'} - - - + + + {:else if status === 'needs-confirmation'} diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts index 10125de2ff..dbb4149472 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts @@ -240,6 +240,25 @@ export class SessionPreviewTabs { this.#flush() } + // Reorder the tabs to the given id order (drag-and-drop). Ids absent from the + // current set are ignored; any current tab the caller omitted is kept at the + // end so a stale/partial order can never drop a tab. No-op if unchanged. + reorder(orderedIds: string[]): void { + const byId = new Map(this.#tabs.map((t) => [t.id, t])) + const next: SessionPreviewTab[] = [] + for (const id of orderedIds) { + const t = byId.get(id) + if (t) { + next.push(t) + byId.delete(id) + } + } + for (const t of this.#tabs) if (byId.has(t.id)) next.push(t) + if (next.length === this.#tabs.length && next.every((t, i) => t === this.#tabs[i])) return + this.#tabs = next + this.#flush() + } + close(id: string): void { const idx = this.#tabs.findIndex((t) => t.id === id) if (idx < 0) return diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts index 352a60a439..08f139499e 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts @@ -343,6 +343,59 @@ describe('SessionPreviewTabs.select / close / setCollapsed', () => { }) }) +describe('SessionPreviewTabs.reorder', () => { + it('reorders tabs to the given id order and persists, keeping the active id', () => { + const { adapter, persisted } = makeAdapter() + const o = owner( + { + tabs: [ + { id: 'a', url: '/x', loc: '/x' }, + { id: 'b', url: '/y', loc: '/y' }, + { id: 'c', url: '/z', loc: '/z' } + ], + activeId: 'a' + }, + adapter + ) + o.reorder(['c', 'a', 'b']) + expect(o.tabs.map((t) => t.id)).toEqual(['c', 'a', 'b']) + expect(o.activeId).toBe('a') + vi.runAllTimers() + expect(persisted.at(-1)?.tabs.map((t) => t.id)).toEqual(['c', 'a', 'b']) + }) + + it('ignores unknown ids and keeps omitted tabs at the end', () => { + const o = owner({ + tabs: [ + { id: 'a', url: '/x', loc: '/x' }, + { id: 'b', url: '/y', loc: '/y' }, + { id: 'c', url: '/z', loc: '/z' } + ], + activeId: 'a' + }) + // 'zzz' doesn't exist (ignored); 'c' omitted from the order (kept at the end). + o.reorder(['b', 'zzz', 'a']) + expect(o.tabs.map((t) => t.id)).toEqual(['b', 'a', 'c']) + }) + + it('is a no-op (no persist) when the order is unchanged', () => { + const { adapter, persisted } = makeAdapter() + const o = owner( + { + tabs: [ + { id: 'a', url: '/x', loc: '/x' }, + { id: 'b', url: '/y', loc: '/y' } + ], + activeId: 'a' + }, + adapter + ) + o.reorder(['a', 'b']) + vi.runAllTimers() + expect(persisted).toHaveLength(0) + }) +}) + describe('SessionPreviewTabs.observeLocation', () => { it('updates loc without touching url', () => { const o = owner({ diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 3989a447df..c9155f1d03 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -11,11 +11,11 @@ PanelRightOpen, ChevronDown, MonitorPlay, - Loader2, - X + Loader2 } from 'lucide-svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' import { Button } from '$lib/components/common' + import DraggableTabs, { type TabItem } from '$lib/components/common/tabs/DraggableTabs.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' import PreviewRouterPicker, { type Scope @@ -245,6 +245,14 @@ const sid = activeRuntime?.sessionId if (sid) mountedTabKeys.delete(tabKey(sid, id)) } + function reorderTabs(next: TabItem[]) { + owner?.reorder(next.map((t) => t.id)) + } + // Adapt the session tab model to DraggableTabs items (labels derived from the + // observed location; every tab closable, none pinned). + const previewTabItems = $derived( + (owner?.tabs ?? []).map((t) => ({ id: t.id, label: tabLabel(t.loc) })) + ) let newTabOpen = $state(false) // Separate open flag for the empty-state launcher: it can be mounted at the // same time as the tab-strip "+" popover, so sharing one flag would open both @@ -587,7 +595,7 @@ onclick={() => owner?.setCollapsed(true)} title="Collapse preview" aria-label="Collapse preview" - class="absolute top-1 left-1 z-30 inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover bg-surface-secondary" + class="absolute top-1 left-1 z-30 inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover" > @@ -603,7 +611,7 @@ )} title="Open in workspace" aria-label="Open in workspace" - class="inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover bg-surface-secondary" + class="inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover" > @@ -612,7 +620,7 @@ onclick={() => (fullscreen = !fullscreen)} title={fullscreen ? 'Exit full screen' : 'Full screen'} aria-label={fullscreen ? 'Exit full screen' : 'Full screen'} - class="inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover bg-surface-secondary" + class="inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover" > {#if fullscreen} @@ -622,96 +630,78 @@
- -
+ - {#each owner?.tabs ?? [] as tab (tab.id)} -
- {#if tab.id === owner?.activeId} - - - {#snippet trigger()} - {tabLabel(tab.loc)} - - {/snippet} - {#snippet content()} - { - activeTabPickerOpen = false - navigatePreviewTo(t) - }} - /> - {/snippet} - - {:else} - - {/if} - -
- {/each} - - {#snippet trigger()} - - {/snippet} - {#snippet content()} - { - newTabOpen = false - openInNewTab(t) - }} - /> - {/snippet} - -
+ {#snippet trigger()} + + {/snippet} + {#snippet content()} + { + activeTabPickerOpen = false + navigatePreviewTo(t) + }} + /> + {/snippet} + + {/if} + {/snippet} + {#snippet afterTabs()} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + { + newTabOpen = false + openInNewTab(t) + }} + /> + {/snippet} + + {/snippet} +
-
+
diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 9e6f508b59..10015a06ec 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -1643,10 +1643,7 @@ gateJobIds={false} extraSourceWindow={() => externalPreviewWindow} /> -
+
{#snippet trigger()} @@ -685,6 +686,7 @@ bind:isOpen={newTabOpen} enableFlyTransition openFocus="[data-workspace-picker-search]" + contentClasses="flex flex-col overflow-hidden" class="shrink-0 inline-flex items-center justify-center w-6 h-6 rounded text-tertiary hover:text-primary hover:bg-surface-hover cursor-pointer" > {#snippet trigger()} @@ -753,6 +755,7 @@ bind:isOpen={emptyStateNewTabOpen} enableFlyTransition openFocus="[data-workspace-picker-search]" + contentClasses="flex flex-col overflow-hidden" > {#snippet trigger()} Date: Wed, 8 Jul 2026 16:34:13 +0200 Subject: [PATCH 263/273] feat(sessions): scoped preview refresh + multi-target live editors + pipeline preview (#10006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(sessions): scope preview-tab refresh to items a chat tool touched Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(sessions): drop dead editor pane, scope raw-app reload by path Multi-target migration P0. SessionWrapper's inline editor pane was dead (the sessions page always mounts it with hideEditor); remove it and the single-target machinery (setSessionTarget/pickEditorTarget/target-keyed editor views). Scope the raw-app file/runnable preview reload to args.path (the app's workspace path) instead of the session target. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(sessions): back editor state with per-(kind,path) cells Multi-target migration P1. Replace the three per-kind singleton stores/slots with per-(kind,path) cell maps, created on demand and kept (eviction deferred to P3). The runtime's public interface is unchanged: the flowStore/scriptStore/savedScript/rawApp/... getters and slot(kind) now forward to the 'active cell' per kind (a single-target shim, tracked by activePath, removed in P2 when the UI mounts one editor per tab). loadFlow/loadScript/loadRawApp and syncPreviewWithDeployed operate on the resolved cell; load logic and semantics are otherwise unchanged, so loading one item no longer clobbers another's state. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(sessions): mount every editable preview tab as its own live editor Multi-target migration P2 — the behavioral flip. resolvePreviewTab no longer takes a target: any editable route (script/flow/raw_app) resolves to an in-process editor, so several items are live at once (iframes remain only for real pages and regular non-raw apps). Each editor binds its own per-(kind,path) cell; the draft codecs close over that cell's store so two editors never cross-write. The single-target shim (activePath + the flowStore/scriptStore/... getters + slot(kind)) is removed; runtime exposes flowCell/scriptCell/rawAppCell(path). Tab open/navigate dedupe by (kind,path) and no longer setTarget. setLiveEditorDraft is gated on the visible tab (isActiveTab) so N editors don't clobber the one-per-(workspace,kind) live-draft slot (path re-key deferred to P4). Co-Authored-By: Claude Opus 4.8 (1M context) * perf(sessions): evict unreferenced editor cells; drop dead warm-editor LRU Multi-target migration P3. Bound the per-(kind,path) editor cell maps: pruneEditorCells drops every cell no open preview tab still references, wired to a new onTabsChanged adapter callback fired on each tab-set change — so closing or navigating a tab away from an item reclaims its cell (dedupe keeps <=1 editor tab per item, so a pruned item has no live editor to strand). Also remove the now-dead editorWarmIds/promoteEditorWarm/MAX_WARM_EDITORS warm-editor LRU: its only reader (SessionWrapper.mountEditor) was removed in P0, and mounted editors are already capped per-tab by mountedTabKeys. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(sessions): retire session.target; preview is fully tab-driven Multi-target migration P4 (final). Remove the session.target field and setSessionTarget: the preview is driven entirely by the tab model now (P2). hydratePreviewTabs no longer seeds a tab from target (saved previewTabs only); openEditorInSession seeds the preview via resetSessionPreviewTabs; normalizeLegacySession drops the retired target field from old records. The setLiveEditorDraft focus gate (isActiveTab, one-per-(workspace,kind)) is kept as-is; a per-path re-key is a possible future refinement, not needed for correctness. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(sessions): describe editor cells as-is, not by their refactor history Address standards review: AGENTS.md requires comments describe the code as it is, not its drafting history. Drop the 'used to be per-kind singletons' / 'pre-refactor empty editor' / 'now' phrasings from the cell comments. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(sessions): update stale runtime.rawApp.val comments to cell.store Address spec review: two comments still referenced the removed runtime.rawApp.val accessor; the live code uses the per-cell store now. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(sessions): fix editor-cell comments after main merge Main's #9993 added svelte-ignore comments describing the old runtime.savedFlow.val / runtime.rawApp.val singleton bindings. The multi-target refactor binds each tab's own editor cell (cell.store / cell.saved), so update the comment text to match; the ownership_invalid_binding directives themselves remain correct (the targets are still runtime-owned). Co-Authored-By: Claude Opus 4.8 (1M context) * feat(sessions): restore data-pipeline preview as a live editor tab The multi-target refactor removed the old single-target editor pane — PipelineEditorView's only mount point — so open_preview(kind="pipeline") opened nothing, even though the chat tool and system prompt still make it the first step of pipeline authoring. Route a /pipeline/ preview tab to the in-process graph editor: - previewRouter: parsePipelineRoute + resolvePreviewTab map the folder to a pipeline editor slot; PreviewSlot.editorKind gains 'pipeline'. - previewTargetForSessionTarget('pipeline') returns the folder route target (was undefined); open() keeps a single pipeline tab and retargets it to the requested folder, since all pipeline tabs share one runtime.pipelineEditorState. - PreviewTabHost mounts PipelineEditorView for the pipeline slot. - PipelineEditorView gains an `active` prop; AI-helper registration and the live-badge poll now gate on isActiveSession && active. Register the pipeline tools on the session's own chat, not the singleton: PreviewTabHost mounts the view outside the SessionWrapper subtree that provides the scoped aiChatManager context, so getAiChatManager() fell back to the app-wide singleton — build_pipeline_node / edit_pipeline_node never reached the session chat and the model fell back to write_script (whose draft never appears on the canvas). Use runtime.manager directly instead. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sessions): scope list-page preview refresh to the page each tool changes The scoped-refresh pass reloaded every open list-page preview tab on any workspace mutation (reloadPages: boolean), so creating a schedule also refreshed the Resources / Variables tabs. Replace the blanket flag with the specific page paths each tool can change: write_schedule → /schedules, write_resource → /resources, write_variable → /variables, create_folder → /folders, write_trigger → the trigger kind's page; delete/deploy/discard/rebase map their `type` to its page (none for script/flow/app). Item-editor writes now reload no pages — their live editor self-syncs. reloadTabs refreshes a list-page tab only when its own path is in the touched set. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(sessions): drop the inert item-reload path; extract a tested previewReload module Post multi-target, every editable item is a live editor whose reload() no-ops, and the one iframe item kind (legacy drag-drop apps) is never emitted as a scope — so the whole `scopes` half of the preview-reload machinery could never fire. Remove it (PreviewKind, PreviewScope, scopeKey, itemTypeToPreviewKind, pendingScopes, and the item-route branch of reloadTabs); the `pages` path already covers every real reload. Lift the surviving pure logic out of the 900-line route component into previewReload.ts — toolReloadEffect(name,args) -> {pages} and a new tabsToReload(tabs,pages) mirroring selectPreviewTabsToClose — and cover it with previewReload.test.ts (per-tool page mapping, item kinds reload nothing, the unknown/local-tool silent-stale guard, loc-over-url matching). Also clear session.target leftovers: delete the unread EDITOR_TARGET_KINDS export and rewrite five comments that still described the removed single-target pane / target-record write. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(sessions): state the preview-reload self-sync invariant once Consolidate the "live editors self-sync, only list pages reload" rationale to previewReload.ts and drop the drafting-history phrasings the review flagged: the update_user_instructions incident and the "(not the runtime)" contrast in sessionDraftCodecs. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sessions): follow the editor cell when a live tab retargets Address PR review findings on the multi-target preview. P1 (Codex) — draft sync stayed bound to the old cell after an in-place tab retarget. useUserDraftSync captured `codec` once, but navigate() re-points a live editor tab (script/flow/raw_app) to another item without remounting, so path/workspace/ready followed the new item while the codec still read/wrote the previous cell's store — cross-writing drafts. Make `codec` a reactive getter like the hook's other inputs; SessionEditorTarget rebuilds it per path. P2 (Claude) — navigate() now enforces the single-pipeline-tab invariant that open() does: retargeting to a /pipeline/ route focuses and re-points the existing pipeline tab instead of turning the active tab into a second editor racing the shared pipelineEditorState. P2 (Claude) — the deploy-in-session handler peeked an editor slot via the create-on-miss cell accessors, allocating an empty cell for items with no open tab. Add a non-creating runtime.loadedEditorPath(kind, path) and use it. P2 (Claude) — correct a SessionPicker comment left stale by the session.target removal (the preview no longer seeds from a target). Tests: two navigate() pipeline-invariant cases. npm run check 0 errors; 167 session unit tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../components/sessions/FlowEditorView.svelte | 31 +- .../sessions/PipelineEditorView.svelte | 43 +- .../components/sessions/PreviewTabHost.svelte | 31 +- .../sessions/RawAppEditorView.svelte | 41 +- .../sessions/ScriptEditorView.svelte | 24 +- .../sessions/SessionEditorTarget.svelte | 44 +- .../components/sessions/SessionPicker.svelte | 4 +- .../components/sessions/SessionWrapper.svelte | 170 +------- .../lib/components/sessions/appDraftCodec.ts | 6 +- .../components/sessions/previewReload.test.ts | 87 ++++ .../lib/components/sessions/previewReload.ts | 75 ++++ .../components/sessions/previewRouter.test.ts | 41 +- .../lib/components/sessions/previewRouter.ts | 34 +- .../sessions/sessionDraftCodecs.test.ts | 15 +- .../components/sessions/sessionDraftCodecs.ts | 36 +- .../sessions/sessionPreviewTabs.svelte.ts | 134 ++++--- .../sessions/sessionPreviewTabs.test.ts | 127 +++--- .../sessions/sessionRuntime.svelte.ts | 379 ++++++++++-------- .../sessions/sessionState.svelte.ts | 40 +- .../sessions/sessionStateIndexedDb.test.ts | 2 - .../sessions/sessionSwitch.svelte.ts | 17 +- .../sessions/useUserDraftSync.svelte.ts | 18 +- .../(root)/(logged)/sessions/+page.svelte | 78 ++-- 23 files changed, 789 insertions(+), 688 deletions(-) create mode 100644 frontend/src/lib/components/sessions/previewReload.test.ts create mode 100644 frontend/src/lib/components/sessions/previewReload.ts diff --git a/frontend/src/lib/components/sessions/FlowEditorView.svelte b/frontend/src/lib/components/sessions/FlowEditorView.svelte index 45163cc416..fe1ae268d7 100644 --- a/frontend/src/lib/components/sessions/FlowEditorView.svelte +++ b/frontend/src/lib/components/sessions/FlowEditorView.svelte @@ -15,7 +15,8 @@ path, workspaceId, onNavigate, - isActiveSession = true + isActiveSession = true, + active = true }: { runtime: SessionRuntime path: string @@ -24,8 +25,12 @@ /** Forwarded to SessionEditorTarget — only the visible session claims the * workspace's single live-editor slot. */ isActiveSession?: boolean + /** Whether this is the visible preview tab (forwarded as isActiveTab). */ + active?: boolean } = $props() + // This tab's own flow cell; each open flow editor binds its own store. + const cell = $derived(runtime.flowCell(path)) let selectedId = $state('settings-metadata') let diffDrawer: DiffDrawer | undefined = $state() @@ -35,7 +40,7 @@ // baseline — useUserDraftSync's inbound effect then syncs the editor preview. // Mirrors ScriptEditorView. async function restoreDeployed() { - const saved = runtime.savedFlow.val + const saved = cell.saved.val if (!saved) { sendUserToast('Could not restore to deployed', true) return @@ -61,7 +66,7 @@ } -{#if runtime.savedFlow.val} +{#if cell.saved.val} {/if} runtime.flowStore.val?.path ?? path} + isActiveTab={active} + effectivePath={() => cell.store.val?.path ?? path} > {#snippet editor()} - +
diff --git a/frontend/src/lib/components/sessions/appDraftCodec.ts b/frontend/src/lib/components/sessions/appDraftCodec.ts index 5f870be92d..10752b8d8f 100644 --- a/frontend/src/lib/components/sessions/appDraftCodec.ts +++ b/frontend/src/lib/components/sessions/appDraftCodec.ts @@ -18,9 +18,9 @@ export type RawAppDraft = { draft_path?: string } -// The shape `runtime.rawApp.val` actually holds (see SessionRuntime in -// sessionRuntime.svelte.ts). Adds `path` (a key, not a draft field) and -// makes `policy` required for the editor's live binding. +// The shape a raw-app cell's store (`RawAppRuntimeValue` in +// sessionRuntime.svelte.ts) actually holds. Adds `path` (a key, not a draft +// field) and makes `policy` required for the editor's live binding. export type RuntimeRawApp = { summary: string path: string diff --git a/frontend/src/lib/components/sessions/previewReload.test.ts b/frontend/src/lib/components/sessions/previewReload.test.ts new file mode 100644 index 0000000000..de11805b9a --- /dev/null +++ b/frontend/src/lib/components/sessions/previewReload.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect } from 'vitest' +import { toolReloadEffect, tabsToReload } from './previewReload' +import type { SessionPreviewTab } from './sessionState.svelte' + +describe('toolReloadEffect', () => { + it('maps a non-item mutation to its own list page only', () => { + expect(toolReloadEffect('write_schedule', { path: 'u/me/s' }).pages).toEqual(['/schedules']) + expect(toolReloadEffect('write_resource', {}).pages).toEqual(['/resources']) + expect(toolReloadEffect('write_variable', {}).pages).toEqual(['/variables']) + expect(toolReloadEffect('create_folder', { name: 'f' }).pages).toEqual(['/folders']) + }) + + it('maps a trigger write to its kind-specific page', () => { + expect(toolReloadEffect('write_trigger', { kind: 'kafka' }).pages).toEqual(['/kafka_triggers']) + expect(toolReloadEffect('write_trigger', { kind: 'http' }).pages).toEqual(['/routes']) + }) + + it('maps a generic item tool to the page for its type', () => { + expect(toolReloadEffect('deploy_workspace_item', { type: 'schedule' }).pages).toEqual([ + '/schedules' + ]) + expect(toolReloadEffect('delete_workspace_item', { type: 'resource' }).pages).toEqual([ + '/resources' + ]) + expect( + toolReloadEffect('discard_local_draft', { type: 'trigger', trigger_kind: 'nats' }).pages + ).toEqual(['/nats_triggers']) + }) + + it('reloads no page for item-editor kinds (they self-sync via their live editor)', () => { + for (const type of ['script', 'flow', 'app']) { + expect(toolReloadEffect('deploy_workspace_item', { type }).pages).toEqual([]) + } + for (const name of [ + 'write_script', + 'edit_script', + 'write_flow', + 'init_app', + 'write_app_file' + ]) { + expect(toolReloadEffect(name, { path: 'u/me/x' }).pages).toEqual([]) + } + }) + + it('reloads nothing for a purely local or unknown tool (the silent-stale guard)', () => { + expect(toolReloadEffect('update_user_instructions', {}).pages).toEqual([]) + expect(toolReloadEffect('some_future_tool', { path: 'p' }).pages).toEqual([]) + }) + + it('reloads nothing for a trigger of unknown kind rather than guessing', () => { + expect(toolReloadEffect('write_trigger', { kind: 'not_a_kind' }).pages).toEqual([]) + }) +}) + +describe('tabsToReload', () => { + const scheduleTab: SessionPreviewTab = { id: 's', url: '/schedules', loc: '/schedules' } + const resourceTab: SessionPreviewTab = { id: 'r', url: '/resources', loc: '/resources' } + const scriptTab: SessionPreviewTab = { + id: 'sc', + url: '/scripts/edit/f/foo/bar', + loc: '/scripts/edit/f/foo/bar' + } + const pipelineTab: SessionPreviewTab = { id: 'p', url: '/pipeline/crm', loc: '/pipeline/crm' } + const tabs = [scheduleTab, resourceTab, scriptTab, pipelineTab] + + it('returns only the tabs whose page is in the set', () => { + expect(tabsToReload(tabs, new Set(['/schedules']))).toEqual([scheduleTab]) + }) + + it('returns list-page tabs but never item-editor or pipeline tabs', () => { + // toolReloadEffect only ever emits list-page paths, so item/pipeline route + // paths are never in `pages` — those tabs self-sync and stay put. + expect(tabsToReload(tabs, new Set(['/schedules', '/resources']))).toEqual([ + scheduleTab, + resourceTab + ]) + }) + + it('is empty when no pages were touched', () => { + expect(tabsToReload(tabs, new Set())).toEqual([]) + }) + + it('matches on the observed loc (with query/hash stripped) over the seeded url', () => { + const navigated: SessionPreviewTab = { id: 'n', url: '/runs', loc: '/schedules?workspace=w' } + expect(tabsToReload([navigated], new Set(['/schedules']))).toEqual([navigated]) + }) +}) diff --git a/frontend/src/lib/components/sessions/previewReload.ts b/frontend/src/lib/components/sessions/previewReload.ts new file mode 100644 index 0000000000..82a87285e1 --- /dev/null +++ b/frontend/src/lib/components/sessions/previewReload.ts @@ -0,0 +1,75 @@ +import type { SessionPreviewTab } from './sessionState.svelte' +import { stripBase, TRIGGER_PAGES, type TriggerKind } from './previewRouter' + +// Which list pages a completed chat tool can change, as base-stripped paths +// (e.g. `/schedules`). This allowlist is the single source of truth for "does +// this tool change a list page a preview tab might show". A new mutating tool +// that surfaces on one of these pages must be added here or that tab silently +// goes stale — match by exact tool name, never a name regex, which mis-classifies +// purely-local tools (e.g. `update_user_instructions`) as page mutations. +// +// Item-editor writes (write_script / write_flow / init_app / write_app_*) are +// deliberately absent: every editable item is a live in-process editor that +// self-syncs from the store the chat mutates, so its tab needs no reload — and +// no list page we preview lists open drafts. They fall through to NO_RELOAD. +// This "live editors self-sync, only list pages reload" invariant is the reason +// the callers below and in the sessions page reload nothing for item tabs. +export type ToolReloadEffect = { pages: string[] } +const NO_RELOAD: ToolReloadEffect = { pages: [] } + +export function toolReloadEffect(name: string, args: any): ToolReloadEffect { + switch (name) { + case 'write_schedule': + return { pages: ['/schedules'] } + case 'write_trigger': + return { pages: triggerPages(args?.kind) } + case 'write_resource': + return { pages: ['/resources'] } + case 'write_variable': + return { pages: ['/variables'] } + case 'create_folder': + return { pages: ['/folders'] } + // Generic item tools carry a workspace-item `type`; refresh its list page + // when it lives on one (schedule/resource/variable/trigger). script/flow/app + // have their own live editor tab and no previewed list page → nothing. + case 'delete_workspace_item': + case 'discard_local_draft': + case 'deploy_workspace_item': + case 'rebase_draft': + return { pages: pagesForItemType(args?.type, args) } + default: + return NO_RELOAD + } +} + +function pagesForItemType(type: unknown, args: any): string[] { + switch (type) { + case 'schedule': + return ['/schedules'] + case 'resource': + return ['/resources'] + case 'variable': + return ['/variables'] + case 'trigger': + return triggerPages(args?.trigger_kind) + default: + return [] + } +} + +function triggerPages(kind: unknown): string[] { + const page = TRIGGER_PAGES[kind as TriggerKind] + return page ? [page.path] : [] +} + +// The open tabs a page-reload should refresh: those whose observed page path is +// in `pages`. Item-editor and pipeline tab routes are never list pages, so they +// never match (see the self-sync invariant above). Pure over a tab snapshot so +// the sessions page can reload by id and this stays unit-testable. +export function tabsToReload( + tabs: SessionPreviewTab[], + pages: ReadonlySet +): SessionPreviewTab[] { + if (pages.size === 0) return [] + return tabs.filter((t) => pages.has(stripBase(t.loc || t.url))) +} diff --git a/frontend/src/lib/components/sessions/previewRouter.test.ts b/frontend/src/lib/components/sessions/previewRouter.test.ts index a9a36ed462..0c8ee64d89 100644 --- a/frontend/src/lib/components/sessions/previewRouter.test.ts +++ b/frontend/src/lib/components/sessions/previewRouter.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect } from 'vitest' import { parsePreviewItemRoute, resolvePreviewTab } from './previewRouter' -import type { SessionTarget } from './sessionState.svelte' describe('parsePreviewItemRoute', () => { it('maps edit/get routes to item kinds', () => { @@ -34,33 +33,28 @@ describe('parsePreviewItemRoute', () => { }) describe('resolvePreviewTab', () => { - const scriptTarget: SessionTarget = { kind: 'script', path: 'f/foo/bar' } - it('routes a static page to the iframe fallback', () => { - expect(resolvePreviewTab('/runs', scriptTarget)).toEqual({ kind: 'iframe' }) + expect(resolvePreviewTab('/runs')).toEqual({ kind: 'iframe' }) }) - it('routes the matching target item to a live editor', () => { - expect(resolvePreviewTab('/scripts/edit/f/foo/bar', scriptTarget)).toEqual({ + it('routes any script item to a live editor', () => { + expect(resolvePreviewTab('/scripts/edit/f/foo/bar')).toEqual({ kind: 'editor', editorKind: 'script', path: 'f/foo/bar' }) }) - it('routes a same-kind but different item to the iframe (one editor per session)', () => { - expect(resolvePreviewTab('/scripts/edit/f/other/script', scriptTarget)).toEqual({ - kind: 'iframe' + it('routes any flow item to a live editor', () => { + expect(resolvePreviewTab('/flows/edit/f/foo/bar')).toEqual({ + kind: 'editor', + editorKind: 'flow', + path: 'f/foo/bar' }) }) - it('routes a different-kind item to the iframe even when it matches no target', () => { - expect(resolvePreviewTab('/flows/edit/f/foo/bar', scriptTarget)).toEqual({ kind: 'iframe' }) - }) - - it('maps a raw-app target to the raw_app editor kind', () => { - const target: SessionTarget = { kind: 'raw_app', path: 'f/a/b' } - expect(resolvePreviewTab('/apps_raw/edit/f/a/b', target)).toEqual({ + it('maps a raw app to the raw_app editor kind', () => { + expect(resolvePreviewTab('/apps_raw/edit/f/a/b')).toEqual({ kind: 'editor', editorKind: 'raw_app', path: 'f/a/b' @@ -68,11 +62,18 @@ describe('resolvePreviewTab', () => { }) it('never routes a regular drag-and-drop app to an editor (no wrapper exists)', () => { - const target = { kind: 'raw_app', path: 'f/a/b' } as SessionTarget - expect(resolvePreviewTab('/apps/edit/f/a/b', target)).toEqual({ kind: 'iframe' }) + expect(resolvePreviewTab('/apps/edit/f/a/b')).toEqual({ kind: 'iframe' }) }) - it('falls back to the iframe when the session has no target', () => { - expect(resolvePreviewTab('/scripts/edit/f/foo/bar', undefined)).toEqual({ kind: 'iframe' }) + it('routes a pipeline folder to the pipeline editor kind', () => { + expect(resolvePreviewTab('/pipeline/my_folder')).toEqual({ + kind: 'editor', + editorKind: 'pipeline', + path: 'my_folder' + }) + }) + + it('routes the bare pipeline list page to the iframe fallback', () => { + expect(resolvePreviewTab('/pipeline')).toEqual({ kind: 'iframe' }) }) }) diff --git a/frontend/src/lib/components/sessions/previewRouter.ts b/frontend/src/lib/components/sessions/previewRouter.ts index 37e8c9c595..8fcd038f79 100644 --- a/frontend/src/lib/components/sessions/previewRouter.ts +++ b/frontend/src/lib/components/sessions/previewRouter.ts @@ -13,7 +13,6 @@ import { } from 'lucide-svelte' import type { DrillIcon } from '$lib/components/drillPicker' import type { WorkspaceItem, WorkspaceItemKind } from '$lib/components/workspacePicker' -import type { SessionTarget } from './sessionState.svelte' import type { SessionTargetKind } from './sessionRuntime.svelte' /** What the preview breadcrumb picker can route to: a static workspace page @@ -104,6 +103,8 @@ export function previewLocationLabel(url: string): string { if (trigger) return trigger const run = stripBase(url).match(/^\/run\/([^/?#]+)/) if (run) return `Run ${decodeURIComponent(run[1]).slice(0, 8)}` + const pipelineFolder = parsePipelineRoute(url) + if (pipelineFolder) return pipelineFolder const parsed = parsePreviewItemRoute(url) if (parsed) return parsed.itemPath.split('/').pop() ?? parsed.itemPath return stripBase(url) @@ -125,16 +126,27 @@ export function parsePreviewItemRoute(fullPath: string): PreviewItemRoute | null return { kind: 'app', raw_app: false, itemPath } } -// How a preview tab should render: as an in-process live editor (sharing the -// session runtime's store) or as an iframe fallback. Only the three kinds with -// existing editor wrappers — and only the tab matching the session's target — -// resolve to 'editor'; everything else (static pages, regular drag-and-drop -// apps, any other item) stays an iframe. +// A `/pipeline/` route is the data-pipeline graph editor for that folder +// (the folder is a single path segment, not a workspace item path). The bare +// `/pipeline` list page is not an editor. Returns the folder name, or null. +export function parsePipelineRoute(fullPath: string): string | null { + const m = stripBase(fullPath).match(/^\/pipeline\/([^/?#]+)/) + return m ? decodeURIComponent(m[1]) : null +} + +// How a preview tab should render: as an in-process live editor or an iframe +// fallback. Any editable item of a wrappable kind (script, flow, raw app) mounts +// its per-(kind,path) cell editor; a `/pipeline/` route mounts the +// data-pipeline graph editor (single, shared runtime.pipelineEditorState — `path` +// is the folder); everything else (static pages, regular drag-and-drop apps, any +// other route) stays an iframe. export type PreviewSlot = - | { kind: 'editor'; editorKind: SessionTargetKind; path: string } + | { kind: 'editor'; editorKind: SessionTargetKind | 'pipeline'; path: string } | { kind: 'iframe' } -export function resolvePreviewTab(url: string, target: SessionTarget | undefined): PreviewSlot { +export function resolvePreviewTab(url: string): PreviewSlot { + const pipelineFolder = parsePipelineRoute(url) + if (pipelineFolder) return { kind: 'editor', editorKind: 'pipeline', path: pipelineFolder } const route = parsePreviewItemRoute(url) if (!route) return { kind: 'iframe' } const editorKind: SessionTargetKind | undefined = @@ -146,11 +158,5 @@ export function resolvePreviewTab(url: string, target: SessionTarget | undefined ? 'raw_app' : undefined if (!editorKind) return { kind: 'iframe' } - // SessionRuntime holds one load slot per kind, so only the tab pointing at the - // session's own target claims it as a live editor; any other item previews as - // an iframe (the "one live editor per session" rule). - if (!target || target.kind !== editorKind || target.path !== route.itemPath) { - return { kind: 'iframe' } - } return { kind: 'editor', editorKind, path: route.itemPath } } diff --git a/frontend/src/lib/components/sessions/sessionDraftCodecs.test.ts b/frontend/src/lib/components/sessions/sessionDraftCodecs.test.ts index fd20c4e2dc..dc2d8cd132 100644 --- a/frontend/src/lib/components/sessions/sessionDraftCodecs.test.ts +++ b/frontend/src/lib/components/sessions/sessionDraftCodecs.test.ts @@ -6,12 +6,11 @@ import { describe, it, expect, vi } from 'vitest' vi.mock('$lib/components/flows/flowState', () => ({ initFlowState: () => Promise.resolve() })) import { makeScriptCodec } from './sessionDraftCodecs' -import type { SessionRuntime } from './sessionRuntime.svelte' import type { NewScript } from '$lib/gen' -// Minimal runtime stub: the script codec only touches `runtime.scriptStore.val`. -function runtimeWith(script: Partial & { path: string }): SessionRuntime { - return { scriptStore: { val: script as NewScript } } as unknown as SessionRuntime +// The script codec closes over one cell's store — a plain `{ val }` object. +function storeWith(script: Partial & { path: string }): { val: NewScript } { + return { val: script as NewScript } } const STORAGE = 'u/admin/draft_abc' @@ -19,7 +18,7 @@ const STORAGE = 'u/admin/draft_abc' describe('makeScriptCodec — draft_path (path rename)', () => { it('writes draft_path when the typed path differs from the storage key', () => { const codec = makeScriptCodec( - runtimeWith({ path: 'u/admin/friendly', content: 'c', summary: 's' }), + storeWith({ path: 'u/admin/friendly', content: 'c', summary: 's' }), () => STORAGE ) const draft = codec.storeToDraft(undefined) as (NewScript & { draft_path?: string }) | undefined @@ -28,7 +27,7 @@ describe('makeScriptCodec — draft_path (path rename)', () => { it('drops draft_path when the typed path equals the storage key', () => { const codec = makeScriptCodec( - runtimeWith({ path: STORAGE, content: 'c', summary: 's' }), + storeWith({ path: STORAGE, content: 'c', summary: 's' }), () => STORAGE ) const draft = codec.storeToDraft(undefined) as (NewScript & { draft_path?: string }) | undefined @@ -36,9 +35,9 @@ describe('makeScriptCodec — draft_path (path rename)', () => { }) it('signature changes on a rename, so the outbound sync persists it', () => { - const before = makeScriptCodec(runtimeWith({ path: STORAGE, content: 'c' }), () => STORAGE) + const before = makeScriptCodec(storeWith({ path: STORAGE, content: 'c' }), () => STORAGE) const after = makeScriptCodec( - runtimeWith({ path: 'u/admin/renamed', content: 'c' }), + storeWith({ path: 'u/admin/renamed', content: 'c' }), () => STORAGE ) expect(before.sig(before.storeToDraft(undefined)!)).not.toBe( diff --git a/frontend/src/lib/components/sessions/sessionDraftCodecs.ts b/frontend/src/lib/components/sessions/sessionDraftCodecs.ts index 9222438534..2f531f2406 100644 --- a/frontend/src/lib/components/sessions/sessionDraftCodecs.ts +++ b/frontend/src/lib/components/sessions/sessionDraftCodecs.ts @@ -2,35 +2,41 @@ import type { Flow, NewScript } from '$lib/gen' import { initFlowState } from '$lib/components/flows/flowState' import { flowDraftSig } from './flowDraftSig' import { applyDraftToRuntimeRawApp, runtimeRawAppToDraft, type RawAppDraft } from './appDraftCodec' -import type { SessionRuntime } from './sessionRuntime.svelte' +import type { RawAppRuntimeValue } from './sessionRuntime.svelte' +import type { StateStore } from '$lib/utils' import type { DraftSyncCodec } from './useUserDraftSync.svelte' // Outbound debounce, uniform across kinds (script was previously immediate; // unified to 150ms so a typing burst coalesces into one persist like flow/raw_app). const DEBOUNCE_MS = 150 -export function makeFlowCodec(runtime: SessionRuntime): DraftSyncCodec { +// Each codec closes over one editor cell's store, so two live editors of the +// same kind sync to their own drafts without crossing. +export function makeFlowCodec( + store: StateStore, + stateStore: { val: Record } +): DraftSyncCodec { return { itemKind: 'flow', sig: flowDraftSig, debounceMs: DEBOUNCE_MS, applyDraftToStore(incoming) { - const current = runtime.flowStore.val + const current = store.val if (!current) return - runtime.flowStore.val = { + store.val = { ...current, value: incoming.value, schema: incoming.schema ?? current.schema, summary: incoming.summary ?? current.summary, description: incoming.description ?? current.description } - // flowStateStore is keyed by module_id; after an AI write the set of + // stateStore is keyed by module_id; after an AI write the set of // module ids may differ, so rebuild the UI state. This wipes per-module // test args / preview output — a known v1 trade-off. - void initFlowState(runtime.flowStore.val, runtime.flowStateStore) + void initFlowState(store.val, stateStore) }, storeToDraft() { - return runtime.flowStore.val + return store.val } } } @@ -40,7 +46,7 @@ export function makeFlowCodec(runtime: SessionRuntime): DraftSyncCodec { type ScriptDraft = NewScript & { draft_path?: string } export function makeScriptCodec( - runtime: SessionRuntime, + store: { val: NewScript | undefined }, // The draft's storage key (the URL path). A never-deployed script is parked // here at `…/draft_` while the user's typed name lives in `script.path`. storagePath: () => string @@ -63,7 +69,7 @@ export function makeScriptCodec( }), debounceMs: DEBOUNCE_MS, applyDraftToStore(incoming) { - const script = runtime.scriptStore.val + const script = store.val if (!script) return if (typeof incoming.content !== 'string') return script.content = incoming.content @@ -71,7 +77,7 @@ export function makeScriptCodec( if (incoming.summary !== undefined) script.summary = incoming.summary }, storeToDraft(current) { - const script = runtime.scriptStore.val + const script = store.val if (!script) return undefined // Merge over the existing entry so fields the preview doesn't edit // (set by the chat) survive a content-only save. @@ -89,18 +95,20 @@ export function makeScriptCodec( } } -export function makeRawAppCodec(runtime: SessionRuntime): DraftSyncCodec { +export function makeRawAppCodec(store: { + val: RawAppRuntimeValue | undefined +}): DraftSyncCodec { return { itemKind: 'raw_app', sig: (d) => JSON.stringify(d), debounceMs: DEBOUNCE_MS, applyDraftToStore(incoming) { - const current = runtime.rawApp.val + const current = store.val if (!current) return - runtime.rawApp.val = applyDraftToRuntimeRawApp(current, incoming) + store.val = applyDraftToRuntimeRawApp(current, incoming) }, storeToDraft() { - const raw = runtime.rawApp.val + const raw = store.val if (!raw) return undefined return runtimeRawAppToDraft(raw) } diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts index dbb4149472..be25a3a75d 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts @@ -3,21 +3,21 @@ import { randomUUID } from '$lib/utils/uuid' import { editPathFor, type WorkspaceItem } from '$lib/components/workspacePicker' import { matchPreviewPage, + parsePipelineRoute, parsePreviewItemRoute, previewLocationLabel, resolvePreviewTab, stripBase, type PreviewTarget } from './previewRouter' -import { sessionTargetHref } from './sessionMode.svelte' import type { SessionPreviewTab, SessionTarget } from './sessionState.svelte' // The single live owner of a session's preview tabs. Runs behind a small // interface both the sessions page (renderer) and the `open_preview` tool cross, // so there is exactly one live copy of the tab model instead of three drifting -// ones synced by effects. Persistence and the session-record `target` write are -// injected as an adapter, so the class is pure runes with no sessionState / IDB -// coupling (mirrors PipelineEditorState). Held on SessionRuntime.previewTabs. +// ones synced by effects. Persistence (and cell pruning) are injected as an +// adapter, so the class is pure runes with no sessionState / IDB coupling +// (mirrors PipelineEditorState). Held on SessionRuntime.previewTabs. export type PreviewTabsSnapshot = { tabs: SessionPreviewTab[] @@ -29,9 +29,17 @@ export type PreviewTabsAdapter = { // Write-behind the full tab model onto the durable backing (debounced by the // owner). Fire-and-forget. persist: (snapshot: PreviewTabsSnapshot) => void - // Point the session's live editor at `target`. Called atomically with the tab - // open/navigate that shows the item, so tab and target can never drift apart. - setTarget: (target: SessionTarget) => void + // Fired synchronously on every tab-set change, so the runtime can drop editor + // cells no open tab references anymore (a closed / navigated-away item). + onTabsChanged?: () => void +} + +// True when a tab's URL is the live editor for a specific editable item. Every +// editable route resolves to an editor, so this doubles as the "same item" dedupe +// test in open()/navigate(). +function isEditorTabFor(url: string, target: SessionTarget): boolean { + const slot = resolvePreviewTab(url) + return slot.kind === 'editor' && slot.editorKind === target.kind && slot.path === target.path } // URL a tab should load for a destination: a page's href, or an item's edit route. @@ -56,8 +64,8 @@ export function canonicalizeObservedLoc(loc: string): string { } // The editor target a destination maps to, or undefined when it isn't an item we -// host live (static pages, legacy drag-and-drop apps). Drives the "set the -// session target iff the destination is an editable item" rule. +// host live (static pages, legacy drag-and-drop apps). Drives the open()/navigate() +// dedupe — one editor tab per (kind, path). function editorTargetFor(target: PreviewTarget): SessionTarget | undefined { if (target.type !== 'item') return undefined const item = target.item @@ -68,13 +76,16 @@ function editorTargetFor(target: PreviewTarget): SessionTarget | undefined { } // Adapt a session editor target (`open_preview` tool arg) to a preview -// destination. Pipeline targets have no full-page route, so they can't be -// previewed as a tab (returns undefined). +// destination. A pipeline target's `path` is a folder name, not a workspace +// item — it maps to the `/pipeline/` route, which resolvePreviewTab +// mounts as the in-process graph editor. export function previewTargetForSessionTarget( kind: SessionTarget['kind'], path: string ): PreviewTarget | undefined { - if (kind === 'pipeline') return undefined + if (kind === 'pipeline') { + return { type: 'page', href: `${base}/pipeline/${encodeURIComponent(path)}`, label: path } + } const item: WorkspaceItem = kind === 'raw_app' ? { kind: 'app', raw_app: true, path, summary: '' } @@ -82,14 +93,12 @@ export function previewTargetForSessionTarget( return { type: 'item', item } } -// Build the initial tab model for a session: its saved tabs, else a single tab -// on its editor target, else empty. Default collapse: collapsed only for a -// session with nothing to preview. +// Build the initial tab model for a session: its saved tabs, else empty. Default +// collapse: collapsed only for a session with nothing to preview. export function hydratePreviewTabs(session: { previewTabs?: SessionPreviewTab[] activePreviewTabId?: string previewCollapsed?: boolean - target?: SessionTarget }): PreviewTabsSnapshot { // Saved tabs come straight from IndexedDB — drop malformed records (missing // id/url) and duplicate ids, which would break the page's keyed {#each}. @@ -107,14 +116,6 @@ export function hydratePreviewTabs(session: { const activeId = wantActive && tabs.some((t) => t.id === wantActive) ? wantActive : tabs[0].id return { tabs, activeId, collapsed: session.previewCollapsed ?? false } } - const seedUrl = sessionTargetHref(session.target) - if (seedUrl) { - return { - tabs: [{ id: 'session', url: seedUrl, loc: seedUrl }], - activeId: 'session', - collapsed: session.previewCollapsed ?? false - } - } return { tabs: [], activeId: '', collapsed: session.previewCollapsed ?? true } } @@ -158,21 +159,16 @@ export class SessionPreviewTabs { } // Open — or focus, if already shown — a tab for a destination, and reveal the - // panel. An editable item is made the session's live editor (setTarget) and - // deduped against the tab already hosting it; anything else dedupes on the - // tab's observed location. + // panel. An editable item dedupes against the tab already hosting that same + // (kind, path); anything else dedupes on the tab's observed location. open(target: PreviewTarget): { status: 'opened' | 'focused' } { const editorTarget = editorTargetFor(target) - if (editorTarget) this.#adapter.setTarget(editorTarget) // A fresh session starts collapsed, so without this the tab opens behind a // collapsed panel and the user sees nothing change. this.#collapsed = false if (editorTarget) { - // resolvePreviewTab(url, target) is 'editor' exactly for the tab showing - // `target`'s item, so it doubles as the dedupe test. - const existing = this.#tabs.find( - (t) => resolvePreviewTab(t.url, editorTarget).kind === 'editor' - ) + // One editor tab per item: focus the tab already hosting this exact item. + const existing = this.#tabs.find((t) => isEditorTabFor(t.url, editorTarget)) if (existing) { this.#activeId = existing.id this.#flush() @@ -180,6 +176,23 @@ export class SessionPreviewTabs { } } const url = targetUrl(target) + // Pipeline previews all share one runtime.pipelineEditorState, so keep at + // most one pipeline tab: re-point the existing one to the requested folder + // rather than opening a second pipeline editor that would fight over the + // shared state (`focused` when it already showed this folder, else `opened` + // since the view now shows a different pipeline). + const pipelineFolder = parsePipelineRoute(url) + if (pipelineFolder) { + const existing = this.#tabs.find((t) => parsePipelineRoute(t.url) !== null) + if (existing) { + const same = existing.url === url + existing.url = url + existing.loc = url + this.#activeId = existing.id + this.#flush() + return { status: same ? 'focused' : 'opened' } + } + } // Focus the tab currently *showing* this destination instead of opening a // duplicate. Matched on the observed `loc`, not `url`: a tab that was // opened here but navigated away no longer counts as showing it. @@ -197,21 +210,16 @@ export class SessionPreviewTabs { } // Re-point the active tab at a destination (breadcrumb pick / in-editor link / - // iframe-posted editor navigation). Same target rule as open: an editable item - // becomes the session's live editor. + // iframe-posted editor navigation). navigate(target: PreviewTarget): void { const t = this.#tabs.find((x) => x.id === this.#activeId) if (!t) return const editorTarget = editorTargetFor(target) if (editorTarget) { - this.#adapter.setTarget(editorTarget) - // Same dedupe as open(): if another tab already hosts `target` as the - // live editor, focus it instead of re-pointing this one — two tabs - // resolving 'editor' for one target would mount two editor instances - // on the same runtime slot. - const existing = this.#tabs.find( - (x) => resolvePreviewTab(x.url, editorTarget).kind === 'editor' - ) + // Same dedupe as open(): if another tab already hosts this exact item, + // focus it instead of re-pointing this one — two tabs for one item would + // mount two editors racing the same (kind, path) cell. + const existing = this.#tabs.find((x) => isEditorTabFor(x.url, editorTarget)) if (existing && existing.id !== t.id) { this.#activeId = existing.id this.#flush() @@ -219,6 +227,21 @@ export class SessionPreviewTabs { } } const url = targetUrl(target) + // Keep at most one pipeline tab (all share runtime.pipelineEditorState): if a + // *different* tab already hosts a pipeline, retarget and focus it rather than + // turning the active tab into a second pipeline editor racing the shared + // state. Same invariant as open(); a no-op when the active tab is that tab. + const pipelineFolder = parsePipelineRoute(url) + if (pipelineFolder) { + const existing = this.#tabs.find((x) => parsePipelineRoute(x.url) !== null) + if (existing && existing.id !== t.id) { + existing.url = url + existing.loc = url + this.#activeId = existing.id + this.#flush() + return + } + } t.url = url t.loc = url this.#flush() @@ -298,6 +321,9 @@ export class SessionPreviewTabs { } #flush(): void { + // Prune cells promptly (cheap, synchronous) even though the durable persist + // stays debounced — a closed tab's editor cell should be reclaimable now. + this.#adapter.onTabsChanged?.() clearTimeout(this.#flushHandle) this.#flushHandle = setTimeout(() => { this.#flushHandle = undefined @@ -335,25 +361,23 @@ export function selectPreviewTabsToClose( } // Human-readable summary of a session's open preview tabs, for the -// `get_preview_status` AI tool. Pure over the owner's model + the session target -// so the owner needs no target-read dependency. The "no session" case is the -// caller's (the tool handler has the session context). -export function describePreview( - tabs: SessionPreviewTab[], - activeId: string, - target: SessionTarget | undefined -): string { +// `get_preview_status` AI tool. Pure over the owner's model. The "no session" +// case is the caller's (the tool handler has the session context). +export function describePreview(tabs: SessionPreviewTab[], activeId: string): string { if (tabs.length === 0) return 'No preview tabs are open in the side panel.' const lines = tabs.map((t) => { const where = t.loc || t.url const page = matchPreviewPage(where) + const pipelineFolder = parsePipelineRoute(where) const route = parsePreviewItemRoute(where) const label = page ? `page "${page.label}"` - : route - ? `${route.raw_app ? 'raw_app' : route.kind} "${route.itemPath}"` - : stripBase(where) - const live = resolvePreviewTab(t.url, target).kind === 'editor' ? ', live editor' : '' + : pipelineFolder + ? `pipeline "${pipelineFolder}"` + : route + ? `${route.raw_app ? 'raw_app' : route.kind} "${route.itemPath}"` + : stripBase(where) + const live = resolvePreviewTab(t.url).kind === 'editor' ? ', live editor' : '' const active = t.id === activeId ? ', active' : '' return `- ${label}${live}${active}` }) diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts index 08f139499e..00e3f6cc90 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts @@ -9,17 +9,16 @@ import { type PreviewTabsSnapshot } from './sessionPreviewTabs.svelte' import type { PreviewTarget } from './previewRouter' -import type { SessionPreviewTab, SessionTarget } from './sessionState.svelte' +import type { SessionPreviewTab } from './sessionState.svelte' +import { base } from '$lib/base' -// In-memory adapter spy: records persisted snapshots + target writes, no IDB. +// In-memory adapter spy: records persisted snapshots, no IDB. function makeAdapter() { const persisted: PreviewTabsSnapshot[] = [] - const targets: SessionTarget[] = [] const adapter: PreviewTabsAdapter = { - persist: (snap) => persisted.push(snap), - setTarget: (t) => targets.push(t) + persist: (snap) => persisted.push(snap) } - return { adapter, persisted, targets } + return { adapter, persisted } } function owner(initial: Partial = {}, adapter?: PreviewTabsAdapter) { @@ -48,6 +47,12 @@ const dndAppTarget: PreviewTarget = { item: { kind: 'app', path: 'u/me/legacy', summary: '' } } const pageTarget: PreviewTarget = { type: 'page', href: '/runs', label: 'Runs' } +const pipelineTarget: PreviewTarget = { type: 'page', href: `${base}/pipeline/crm`, label: 'crm' } +const pipelineTarget2: PreviewTarget = { + type: 'page', + href: `${base}/pipeline/sales`, + label: 'sales' +} beforeEach(() => { vi.useFakeTimers() @@ -76,15 +81,6 @@ describe('hydratePreviewTabs', () => { expect(snap.activeId).toBe('a') }) - it('seeds a single tab on the editor target when there are no saved tabs', () => { - const snap = hydratePreviewTabs({ target: { kind: 'script', path: 'u/me/foo' } }) - expect(snap.tabs).toEqual([ - { id: 'session', url: '/scripts/edit/u/me/foo', loc: '/scripts/edit/u/me/foo' } - ]) - expect(snap.activeId).toBe('session') - expect(snap.collapsed).toBe(false) - }) - it('is empty and collapsed for a session with nothing to preview', () => { const snap = hydratePreviewTabs({}) expect(snap.tabs).toEqual([]) @@ -93,10 +89,7 @@ describe('hydratePreviewTabs', () => { }) it('honours an explicit previewCollapsed override', () => { - expect( - hydratePreviewTabs({ previewCollapsed: true, target: { kind: 'script', path: 'p' } }) - .collapsed - ).toBe(true) + expect(hydratePreviewTabs({ previewCollapsed: true }).collapsed).toBe(true) expect(hydratePreviewTabs({ previewCollapsed: false }).collapsed).toBe(false) }) @@ -115,13 +108,12 @@ describe('hydratePreviewTabs', () => { expect(snap.activeId).toBe('a') }) - it('falls back to the target seed when every saved tab is malformed', () => { + it('is empty when every saved tab is malformed', () => { const snap = hydratePreviewTabs({ - previewTabs: [{ id: '', url: '', loc: '' }], - target: { kind: 'script', path: 'u/me/foo' } + previewTabs: [{ id: '', url: '', loc: '' }] }) - expect(snap.tabs).toHaveLength(1) - expect(snap.activeId).toBe('session') + expect(snap.tabs).toEqual([]) + expect(snap.activeId).toBe('') }) }) @@ -142,22 +134,24 @@ describe('previewTargetForSessionTarget', () => { item: { kind: 'flow', path: 'p', summary: '' } }) }) - it('returns undefined for pipeline (no full-page route)', () => { - expect(previewTargetForSessionTarget('pipeline', 'p')).toBeUndefined() + it('maps pipeline to its folder route page target', () => { + expect(previewTargetForSessionTarget('pipeline', 'my_folder')).toEqual({ + type: 'page', + href: `${base}/pipeline/my_folder`, + label: 'my_folder' + }) }) }) describe('SessionPreviewTabs.open', () => { - it('opens an editor item, points the target at it, activates it, and reveals the panel', () => { - const { adapter, targets } = makeAdapter() - const o = owner({ collapsed: true }, adapter) + it('opens an editor item, activates it, and reveals the panel', () => { + const o = owner({ collapsed: true }) const res = o.open(scriptTarget) expect(res.status).toBe('opened') expect(o.tabs).toHaveLength(1) expect(o.tabs[0].url).toBe('/scripts/edit/u/me/foo') expect(o.activeId).toBe(o.tabs[0].id) expect(o.collapsed).toBe(false) - expect(targets).toEqual([{ kind: 'script', path: 'u/me/foo' }]) }) it('focuses the existing tab instead of duplicating when the item is already shown', () => { @@ -171,27 +165,23 @@ describe('SessionPreviewTabs.open', () => { expect(o.activeId).toBe(firstId) }) - it('opens a second tab and repoints the live editor for a different item', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + it('opens a second tab for a different editor item', () => { + const o = owner() o.open(scriptTarget) const res = o.open(flowTarget) expect(res.status).toBe('opened') expect(o.tabs).toHaveLength(2) - expect(targets.at(-1)).toEqual({ kind: 'flow', path: 'u/me/bar' }) + expect(o.tabs.at(-1)!.url).toBe('/flows/edit/u/me/bar') }) it('opens a raw app via its apps_raw route', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + const o = owner() o.open(rawAppTarget) expect(o.tabs[0].url).toBe('/apps_raw/edit/u/me/app') - expect(targets).toEqual([{ kind: 'raw_app', path: 'u/me/app' }]) }) - it('focuses the tab already showing a page instead of duplicating, and never sets a target', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + it('focuses the tab already showing a page instead of duplicating', () => { + const o = owner() o.open(pageTarget) const firstId = o.activeId o.open(scriptTarget) @@ -199,7 +189,6 @@ describe('SessionPreviewTabs.open', () => { expect(res.status).toBe('focused') expect(o.tabs).toHaveLength(2) expect(o.activeId).toBe(firstId) - expect(targets).toEqual([{ kind: 'script', path: 'u/me/foo' }]) }) it('opens a fresh page tab when the original navigated away', () => { @@ -221,19 +210,16 @@ describe('SessionPreviewTabs.open', () => { expect(o.tabs).toHaveLength(1) }) - it('does not set a target for a legacy drag-and-drop app', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + it('opens a legacy drag-and-drop app as an iframe route', () => { + const o = owner() o.open(dndAppTarget) - expect(targets).toEqual([]) expect(o.tabs[0].url).toBe('/apps/edit/u/me/legacy') }) }) describe('SessionPreviewTabs.navigate', () => { - it('retargets the active tab and sets the target for an editor item', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + it('retargets the active tab to an editor item', () => { + const o = owner() o.open(pageTarget) const tabId = o.activeId o.navigate(flowTarget) @@ -241,25 +227,19 @@ describe('SessionPreviewTabs.navigate', () => { expect(o.activeId).toBe(tabId) expect(o.tabs[0].url).toBe('/flows/edit/u/me/bar') expect(o.tabs[0].loc).toBe('/flows/edit/u/me/bar') - expect(targets).toEqual([{ kind: 'flow', path: 'u/me/bar' }]) }) it('no-ops with no active tab', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + const o = owner() o.navigate(flowTarget) expect(o.tabs).toHaveLength(0) - expect(targets).toEqual([]) }) - it('retargets to a page without touching the target', () => { - const { adapter, targets } = makeAdapter() - const o = owner({}, adapter) + it('retargets to a page', () => { + const o = owner() o.open(scriptTarget) - targets.length = 0 o.navigate(pageTarget) expect(o.tabs[0].url).toBe('/runs') - expect(targets).toEqual([]) }) it('focuses the tab already hosting the item instead of duplicating the editor', () => { @@ -275,6 +255,31 @@ describe('SessionPreviewTabs.navigate', () => { // The page tab must keep its own url — only focus moved. expect(o.tabs.find((t) => t.id === pageTabId)?.url).toBe('/runs') }) + + it('retargets the one pipeline tab instead of turning the active tab into a second', () => { + const o = owner() + o.open(pipelineTarget) + const pipelineTabId = o.activeId + o.open(scriptTarget) // a second, non-pipeline tab is now active + const scriptTabId = o.activeId + o.navigate(pipelineTarget2) + // No second pipeline editor: the existing one is retargeted and focused. + expect(o.tabs).toHaveLength(2) + expect(o.activeId).toBe(pipelineTabId) + expect(o.tabs.find((t) => t.id === pipelineTabId)?.url).toBe(`${base}/pipeline/sales`) + // The script tab is untouched. + expect(o.tabs.find((t) => t.id === scriptTabId)?.url).toBe('/scripts/edit/u/me/foo') + }) + + it('retargets the active pipeline tab in place to a new folder', () => { + const o = owner() + o.open(pipelineTarget) + const tabId = o.activeId + o.navigate(pipelineTarget2) + expect(o.tabs).toHaveLength(1) + expect(o.activeId).toBe(tabId) + expect(o.tabs[0].url).toBe(`${base}/pipeline/sales`) + }) }) describe('SessionPreviewTabs.select / close / setCollapsed', () => { @@ -442,23 +447,23 @@ describe('SessionPreviewTabs persistence', () => { describe('describePreview', () => { it('reports no tabs when there are none', () => { - expect(describePreview([], '', undefined)).toContain('No preview tabs') + expect(describePreview([], '')).toContain('No preview tabs') }) it('lists tabs, marks the active one, and flags the live editor', () => { const tabs: SessionPreviewTab[] = [ { id: 'a', url: '/scripts/edit/u/me/foo', loc: '/scripts/edit/u/me/foo' } ] - const out = describePreview(tabs, 'a', { kind: 'script', path: 'u/me/foo' }) + const out = describePreview(tabs, 'a') expect(out).toContain('1 preview tab') expect(out).toContain('script "u/me/foo"') expect(out).toContain('live editor') expect(out).toContain('active') }) - it('labels a known page and omits the live-editor flag when the target differs', () => { + it('labels a known page and omits the live-editor flag for a non-item page', () => { const tabs: SessionPreviewTab[] = [{ id: 'a', url: '/runs', loc: '/runs' }] - const out = describePreview(tabs, 'a', { kind: 'script', path: 'u/me/foo' }) + const out = describePreview(tabs, 'a') expect(out).toContain('page "Runs"') expect(out).not.toContain('live editor') }) diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index 261cc6aedc..f55fbe68ec 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -1,4 +1,4 @@ -import { SvelteMap, SvelteSet } from 'svelte/reactivity' +import { SvelteMap } from 'svelte/reactivity' import { get } from 'svelte/store' import { base } from '$lib/base' import { AIChatManager, AIMode } from '$lib/components/copilot/chat/AIChatManager.svelte' @@ -39,7 +39,6 @@ import { setSessionChatId, setSessionPreviewCollapsed, setSessionTabs, - setSessionTarget, type Session } from './sessionState.svelte' import { @@ -49,7 +48,7 @@ import { previewTargetForSessionTarget, selectPreviewTabsToClose } from './sessionPreviewTabs.svelte' -import { matchPreviewPage, previewLocationLabel } from './previewRouter' +import { matchPreviewPage, parsePreviewItemRoute, previewLocationLabel } from './previewRouter' import { UserDraft } from '$lib/userDraft.svelte' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { armRestartOnFirstInteraction } from '$lib/userDraftToast' @@ -93,6 +92,55 @@ export interface LoadSlot { export type SessionTargetKind = 'flow' | 'script' | 'raw_app' +// The live runtime value a raw-app editor cell binds. Legacy drag-and-drop apps +// are intentionally NOT hosted in the session preview (only code-based raw apps). +export interface RawAppRuntimeValue { + files: Record + runnables: Record + data: RawAppData + policy: any + summary: string + path: string + custom_path?: string + draft_path?: string +} +// The deployed baseline a raw-app cell diffs against (topbar Diff drawer). +export interface RawAppSavedValue { + value: { + files: Record + runnables: Record + } + draft?: any + path: string + summary: string + policy: any + draft_only?: boolean + /** No deployed counterpart (draft-only); disables the topbar Diff. */ + no_deployed?: boolean + custom_path?: string +} + +// One editor cell per (kind, path) the session loads: the load slot plus the +// content/baseline stores for that item. Keying by path lets several items of the +// same kind stay loaded — and mounted as separate live editors — at once. A tab's +// editor resolves its own cell via the accessors below. +export interface FlowCell { + slot: LoadSlot + store: StateStore + stateStore: { val: Record } + saved: { val: SavedFlow | undefined } +} +export interface ScriptCell { + slot: LoadSlot + store: { val: NewScript | undefined } + saved: { val: SavedScript | undefined } +} +export interface RawAppCell { + slot: LoadSlot + store: { val: RawAppRuntimeValue | undefined } + saved: { val: RawAppSavedValue | undefined } +} + export interface SessionRuntime { readonly sessionId: string readonly manager: AIChatManager @@ -103,54 +151,18 @@ export interface SessionRuntime { // Pipeline target state — persists across editor hide/show (the pane unmounts // on hide, so this can't be component-local) and across session switches. readonly pipelineEditorState: PipelineEditorState - // Kind-agnostic accessor over the per-kind load slots, for consumers (the - // editor-target gate) that only need load state and not the typed store. - slot(kind: SessionTargetKind): LoadSlot - // Flow target state - readonly flowStore: StateStore - readonly flowStateStore: { val: Record } - readonly savedFlow: { val: SavedFlow | undefined } + // Per-(kind, path) editor cells (content/baseline stores + load slot), created + // on demand. Each editable preview tab resolves its own cell, so several items + // stay live at once. + flowCell(path: string): FlowCell loadFlow(workspace: string, path: string, force?: boolean): Promise - // Script target state (parallel to flow, populated only for script-targeted sessions) - readonly scriptStore: { val: NewScript | undefined } - readonly savedScript: { val: SavedScript | undefined } + scriptCell(path: string): ScriptCell loadScript(workspace: string, path: string, force?: boolean): Promise - // Note: legacy drag-and-drop apps are intentionally NOT hosted in the - // session preview pane (only code-based raw apps are), so there's no - // app target state here. - // Raw App (HTML-based) target state - readonly rawApp: { - val: - | { - files: Record - runnables: Record - data: RawAppData - policy: any - summary: string - path: string - custom_path?: string - draft_path?: string - } - | undefined - } - readonly savedRawApp: { - val: - | { - value: { - files: Record - runnables: Record - } - draft?: any - path: string - summary: string - policy: any - draft_only?: boolean - /** No deployed counterpart (draft-only); disables the topbar Diff. */ - no_deployed?: boolean - custom_path?: string - } - | undefined - } + rawAppCell(path: string): RawAppCell + // Non-creating peek at an editor cell's settled path (undefined when no cell + // exists yet for this (kind, path)), so callers can check load state without + // the cell accessors' create-on-miss side effect. + loadedEditorPath(kind: SessionTargetKind, path: string): string | undefined loadRawApp( workspace: string, path: string, @@ -185,6 +197,32 @@ function emptyFlow(): Flow { } } +function emptyLoadSlot(): LoadSlot { + return { loadedPath: undefined, loadedWorkspace: undefined, loading: false, notFound: false } +} + +// Cell factories — a cell starts in the empty-editor state (empty flow / no +// script / no app) until its first load populates it. +function makeFlowCell(): FlowCell { + const slot: LoadSlot = $state(emptyLoadSlot()) + const store: StateStore = $state({ val: emptyFlow() }) + const stateStore: { val: Record } = $state({ val: {} }) + const saved: { val: SavedFlow | undefined } = $state({ val: undefined }) + return { slot, store, stateStore, saved } +} +function makeScriptCell(): ScriptCell { + const slot: LoadSlot = $state(emptyLoadSlot()) + const store: { val: NewScript | undefined } = $state({ val: undefined }) + const saved: { val: SavedScript | undefined } = $state({ val: undefined }) + return { slot, store, saved } +} +function makeRawAppCell(): RawAppCell { + const slot: LoadSlot = $state(emptyLoadSlot()) + const store: { val: RawAppRuntimeValue | undefined } = $state({ val: undefined }) + const saved: { val: RawAppSavedValue | undefined } = $state({ val: undefined }) + return { slot, store, saved } +} + const GENERATED_SUMMARY_TIMEOUT_MS = 15000 const GENERATED_SUMMARY_MAX_TRANSCRIPT_CHARS = 4000 const GENERATED_SUMMARY_MAX_LENGTH = 60 @@ -309,48 +347,65 @@ function createRuntime(session: Session): SessionRuntime { } manager.afterFirstTurnSaved = () => generateAndApplySessionSummary(session.id, manager) - const flowStore: StateStore = $state({ val: emptyFlow() }) - const flowStateStore: { val: Record } = $state({ val: {} }) - const savedFlow: { val: SavedFlow | undefined } = $state({ - val: undefined - }) - - const flowSlot: LoadSlot = $state({ - loadedPath: undefined, - loadedWorkspace: undefined, - loading: false, - notFound: false - }) - - const scriptStore: { val: NewScript | undefined } = $state({ val: undefined }) - const savedScript: { val: SavedScript | undefined } = $state({ val: undefined }) - const scriptSlot: LoadSlot = $state({ - loadedPath: undefined, - loadedWorkspace: undefined, - loading: false, - notFound: false - }) - - const rawApp: { val: SessionRuntime['rawApp']['val'] } = $state({ val: undefined }) - const savedRawApp: { val: SessionRuntime['savedRawApp']['val'] } = $state({ val: undefined }) - const rawAppSlot: LoadSlot = $state({ - loadedPath: undefined, - loadedWorkspace: undefined, - loading: false, - notFound: false - }) + // One cell per (kind, path). Created on demand by the load methods; each holds + // cached content (KB–MB), not a mounted editor. Bounded to the items open + // preview tabs reference: pruneEditorCells (below) drops the rest when the tab + // set changes, so re-pointing one tab through many items can't leak cells. + const flowCells = new Map() + const scriptCells = new Map() + const rawAppCells = new Map() + function flowCell(path: string): FlowCell { + let c = flowCells.get(path) + if (!c) flowCells.set(path, (c = makeFlowCell())) + return c + } + function scriptCell(path: string): ScriptCell { + let c = scriptCells.get(path) + if (!c) scriptCells.set(path, (c = makeScriptCell())) + return c + } + function rawAppCell(path: string): RawAppCell { + let c = rawAppCells.get(path) + if (!c) rawAppCells.set(path, (c = makeRawAppCell())) + return c + } + function loadedEditorPath(kind: SessionTargetKind, path: string): string | undefined { + const cell = + kind === 'flow' + ? flowCells.get(path) + : kind === 'script' + ? scriptCells.get(path) + : rawAppCells.get(path) + return cell?.slot.loadedPath + } + // Drop every editor cell no open preview tab still points at. Called on each + // tab-set change: a closed or navigated-away item's cell (and its cached + // content) is reclaimed. Deduping keeps at most one editor tab per item, so an + // item absent from the open tabs has no live editor to strand. + function pruneEditorCells(): void { + const keep = { flow: new Set(), script: new Set(), raw_app: new Set() } + for (const t of previewTabs.tabs) { + const route = parsePreviewItemRoute(t.url) + if (!route) continue + const kind = route.raw_app ? 'raw_app' : route.kind + if (kind === 'flow' || kind === 'script' || kind === 'raw_app') keep[kind].add(route.itemPath) + } + for (const p of [...flowCells.keys()]) if (!keep.flow.has(p)) flowCells.delete(p) + for (const p of [...scriptCells.keys()]) if (!keep.script.has(p)) scriptCells.delete(p) + for (const p of [...rawAppCells.keys()]) if (!keep.raw_app.has(p)) rawAppCells.delete(p) + } // Hydrate the preview-tab owner from the session record (the durable backing); // from here on the owner is the single live copy and writes back through the - // adapter. setSessionTabs / setSessionPreviewCollapsed / setSessionTarget stay - // the low-level record writers (a transient session's writes land in the - // localStorage draft slot until it materialises). + // adapter. setSessionTabs / setSessionPreviewCollapsed stay the low-level record + // writers (a transient session's writes land in the localStorage draft slot + // until it materialises). const previewTabs = new SessionPreviewTabs(hydratePreviewTabs(session), { persist: (snap) => { setSessionTabs(session.id, snap.tabs, snap.activeId) setSessionPreviewCollapsed(session.id, snap.collapsed) }, - setTarget: (target) => setSessionTarget(session.id, target) + onTabsChanged: pruneEditorCells }) // Let the jobs tray open a run in this session's preview panel (as an iframe @@ -376,23 +431,20 @@ function createRuntime(session: Session): SessionRuntime { sessionId: session.id, manager, previewTabs, - slot(kind: SessionTargetKind): LoadSlot { - return kind === 'flow' ? flowSlot : kind === 'script' ? scriptSlot : rawAppSlot - }, pipelineEditorState, - flowStore, - flowStateStore, - savedFlow, + flowCell, + loadedEditorPath, async loadFlow(workspace: string, path: string, force = false) { - if (flowSlot.loadedPath === path && flowSlot.loadedWorkspace === workspace && !force) return + const { slot, store, stateStore, saved } = flowCell(path) + if (slot.loadedPath === path && slot.loadedWorkspace === workspace && !force) return // See loadScript: forced reload remounts via the render gate. A workspace // retarget (same path, new fork) drops the stale content the same way so // the editor gate shows loading and outbound sync can't write the old // workspace's content into the new one before the fetch lands. - if (force || flowSlot.loadedWorkspace !== workspace) flowSlot.loadedPath = undefined - flowSlot.loading = true - flowSlot.notFound = false + if (force || slot.loadedWorkspace !== workspace) slot.loadedPath = undefined + slot.loading = true + slot.notFound = false try { // Draft first. UserDraft is the shared authoritative content // source — the chat (write_flow / patch_flow_json / @@ -416,21 +468,20 @@ function createRuntime(session: Session): SessionRuntime { // yet on the backend — draft-only flows are a valid state. try { const result = await FlowService.getFlowByPath({ workspace, path, getDraft: true }) - savedFlow.val = result as SavedFlow + saved.val = result as SavedFlow } catch { - savedFlow.val = undefined + saved.val = undefined } - await initFlow(aiDraft, flowStore, flowStateStore) - if (deployedVersionId != null && flowStore.val) - flowStore.val.version_id = deployedVersionId - flowSlot.loadedPath = path - flowSlot.loadedWorkspace = workspace + await initFlow(aiDraft, store, stateStore) + if (deployedVersionId != null && store.val) store.val.version_id = deployedVersionId + slot.loadedPath = path + slot.loadedWorkspace = workspace return } // No local draft yet — seed from `result.draft ?? result`. const result = await FlowService.getFlowByPath({ workspace, path, getDraft: true }) - savedFlow.val = result as SavedFlow + saved.val = result as SavedFlow const flow: Flow = ((result as SavedFlow).draft ?? (result as Flow)) as Flow // Seed the per-tab last_sync from the server draft's timestamp so the // seeding save below attaches a matching last_sync and the server can @@ -443,31 +494,30 @@ function createRuntime(session: Session): SessionRuntime { (result as SavedFlow).draft_saved_at ) UserDraft.save('flow', path, flow, { workspace }) - await initFlow(flow, flowStore, flowStateStore) - if (deployedVersionId != null && flowStore.val) flowStore.val.version_id = deployedVersionId - flowSlot.loadedPath = path - flowSlot.loadedWorkspace = workspace + await initFlow(flow, store, stateStore) + if (deployedVersionId != null && store.val) store.val.version_id = deployedVersionId + slot.loadedPath = path + slot.loadedWorkspace = workspace } catch (err) { console.error('Failed to load flow', err) - flowSlot.notFound = true + slot.notFound = true } finally { - flowSlot.loading = false + slot.loading = false } }, - scriptStore, - savedScript, + scriptCell, async loadScript(workspace: string, path: string, force = false) { - if (scriptSlot.loadedPath === path && scriptSlot.loadedWorkspace === workspace && !force) - return + const { slot, store, saved } = scriptCell(path) + if (slot.loadedPath === path && slot.loadedWorkspace === workspace && !force) return // Forced reload: clearing the slot's loadedPath drops us into // SessionEditorTarget's `{:else if slot.loadedPath === undefined}` gate, // which unmounts then remounts the editor — avoids the Monaco init race a // synchronous {#key} would hit. - if (force || scriptSlot.loadedWorkspace !== workspace) scriptSlot.loadedPath = undefined - scriptSlot.loading = true - scriptSlot.notFound = false + if (force || slot.loadedWorkspace !== workspace) slot.loadedPath = undefined + slot.loading = true + slot.notFound = false try { // Draft first. UserDraft is the shared authoritative content // source — the chat (write_script / edit_script) and the @@ -482,16 +532,16 @@ function createRuntime(session: Session): SessionRuntime { // savedScript undefined and skip parent_hash. try { const result = await ScriptService.getScriptByPath({ workspace, path, getDraft: true }) - savedScript.val = result as SavedScript + saved.val = result as SavedScript } catch { - savedScript.val = undefined + saved.val = undefined } // Clone before layering the AI draft on top, else we'd mutate - // `savedScript.val` in place and lose the pristine diff baseline. - const baseline: NewScript = savedScript.val + // `saved.val` in place and lose the pristine diff baseline. + const baseline: NewScript = saved.val ? (structuredClone( $state.snapshot( - (savedScript.val.draft as NewScript | undefined) ?? (savedScript.val as NewScript) + (saved.val.draft as NewScript | undefined) ?? (saved.val as NewScript) ) ) as NewScript) : { @@ -510,21 +560,21 @@ function createRuntime(session: Session): SessionRuntime { schema: emptySchema(), language: (aiDraft.language ?? 'bun') as any } - if (savedScript.val?.hash) { - baseline.parent_hash = savedScript.val.hash + if (saved.val?.hash) { + baseline.parent_hash = saved.val.hash } baseline.content = aiDraft.content if (aiDraft.language) baseline.language = aiDraft.language if (aiDraft.summary !== undefined) baseline.summary = aiDraft.summary - scriptStore.val = baseline - scriptSlot.loadedPath = path - scriptSlot.loadedWorkspace = workspace + store.val = baseline + slot.loadedPath = path + slot.loadedWorkspace = workspace return } // No local draft yet — seed from `result.draft ?? result`. const result = await ScriptService.getScriptByPath({ workspace, path, getDraft: true }) - savedScript.val = result as SavedScript + saved.val = result as SavedScript // Clone before mutating, else `baseline` aliases `result` and // `baseline.parent_hash` corrupts the diff baseline. const baseline = structuredClone( @@ -542,27 +592,26 @@ function createRuntime(session: Session): SessionRuntime { (result as SavedScript).draft_saved_at ) UserDraft.save('script', path, baseline, { workspace }) - scriptStore.val = baseline - scriptSlot.loadedPath = path - scriptSlot.loadedWorkspace = workspace + store.val = baseline + slot.loadedPath = path + slot.loadedWorkspace = workspace } catch (err) { console.error('Failed to load script', err) - scriptSlot.notFound = true + slot.notFound = true } finally { - scriptSlot.loading = false + slot.loading = false } }, - rawApp, - savedRawApp, + rawAppCell, async loadRawApp(workspace: string, path: string, force = false, deployedOnly = false) { - if (rawAppSlot.loadedPath === path && rawAppSlot.loadedWorkspace === workspace && !force) - return + const { slot, store, saved } = rawAppCell(path) + if (slot.loadedPath === path && slot.loadedWorkspace === workspace && !force) return // See loadScript: forced reload remounts via the render gate. - if (force || rawAppSlot.loadedWorkspace !== workspace) rawAppSlot.loadedPath = undefined - rawAppSlot.loading = true - rawAppSlot.notFound = false + if (force || slot.loadedWorkspace !== workspace) slot.loadedPath = undefined + slot.loading = true + slot.notFound = false try { // Draft first. UserDraft is the shared authoritative content // source — the chat (init_app / write_app_file / ...) and the @@ -586,7 +635,7 @@ function createRuntime(session: Session): SessionRuntime { }) // Top-level fields are the deployed payload — the diff // baseline, since the session has its own `aiDraft`. - savedRawApp.val = { + saved.val = { summary: result.summary, value: result.value as any, path: result.path, @@ -595,9 +644,9 @@ function createRuntime(session: Session): SessionRuntime { no_deployed: result.no_deployed } } catch { - savedRawApp.val = undefined + saved.val = undefined } - rawApp.val = applyDraftToRuntimeRawApp( + store.val = applyDraftToRuntimeRawApp( { files: {}, runnables: {}, @@ -608,8 +657,8 @@ function createRuntime(session: Session): SessionRuntime { }, aiDraft ) - rawAppSlot.loadedPath = path - rawAppSlot.loadedWorkspace = workspace + slot.loadedPath = path + slot.loadedWorkspace = workspace return } @@ -622,7 +671,7 @@ function createRuntime(session: Session): SessionRuntime { rawApp: true }) // Deployed baseline for the diff drawer (top-level fields). - savedRawApp.val = { + saved.val = { summary: result.summary, value: result.value as any, path: result.path, @@ -669,14 +718,14 @@ function createRuntime(session: Session): SessionRuntime { (result as any).draft_saved_at as string | undefined ) UserDraft.save('raw_app', path, runtimeRawAppToDraft(runtimeValue), { workspace }) - rawApp.val = runtimeValue - rawAppSlot.loadedPath = path - rawAppSlot.loadedWorkspace = workspace + store.val = runtimeValue + slot.loadedPath = path + slot.loadedWorkspace = workspace } catch (err) { console.error('Failed to load raw app', err) - rawAppSlot.notFound = true + slot.notFound = true } finally { - rawAppSlot.loading = false + slot.loading = false } }, @@ -794,33 +843,14 @@ export type SessionChatStatus = | 'draft' | 'error' -// MRU set of session ids whose FlowEditorView is currently mounted. Capped at -// MAX_WARM_EDITORS — sessions outside the set show chat-only. Module-scoped so -// both the page (which mutates) and the sidebar (which reads for the dev clue) -// see the same state. -const MAX_WARM_EDITORS = 3 -export const editorWarmIds = new SvelteSet() - -// Full session teardown: dispose the runtime, drop the LRU entry, and remove -// from sessionState in one call. Callers (sidebar / header dropdowns) just -// invoke this; navigation away from a deleted active session is the caller's -// responsibility. +// Full session teardown: dispose the runtime and remove from sessionState in one +// call. Callers (sidebar / header dropdowns) just invoke this; navigation away +// from a deleted active session is the caller's responsibility. export function removeSession(sessionId: string): void { disposeRuntime(sessionId) - editorWarmIds.delete(sessionId) deleteSessionState(sessionId) } -export function promoteEditorWarm(sessionId: string): void { - editorWarmIds.delete(sessionId) - editorWarmIds.add(sessionId) - while (editorWarmIds.size > MAX_WARM_EDITORS) { - const oldest = editorWarmIds.values().next().value - if (oldest === undefined) break - editorWarmIds.delete(oldest) - } -} - // Register the global open_preview tool handler once at module load. It // dispatches to the *calling* session (sessionId threaded from the tool ctx), // falling back to the UI-active session only when none was passed — so a @@ -841,7 +871,6 @@ setOpenPreviewHandler(({ sessionId: callerSessionId, kind, path }) => { return `Error: ${kind} targets cannot be shown in the preview panel.` } const result = getOrCreateRuntime(session).previewTabs.open(target) - promoteEditorWarm(sessionId) return result.status === 'focused' ? `A preview tab is already showing ${kind} "${path}" — focused it.` : `Opened ${kind} preview for ${path} in a new tab in the side panel.` @@ -869,12 +898,10 @@ setOpenPagePreviewHandler(({ sessionId: callerSessionId, href, label, newTab }) owner.select(existing.id) owner.navigate({ type: 'page', href, label }) owner.setCollapsed(false) - promoteEditorWarm(sessionId) return `Updated the ${label} preview tab with the new filters.` } } const result = owner.open({ type: 'page', href, label }) - promoteEditorWarm(sessionId) return result.status === 'focused' ? `A preview tab is already showing ${label} — focused it and applied the filters.` : `Opened ${label} in a new preview tab in the side panel.` @@ -889,7 +916,7 @@ setGetPreviewStatusHandler((callerSessionId) => { const session = sessionState.sessions.find((s) => s.id === sessionId) if (!session) return 'No active session; the preview panel is unavailable.' const owner = getOrCreateRuntime(session).previewTabs - return describePreview(owner.tabs, owner.activeId, session.target) + return describePreview(owner.tabs, owner.activeId) }) // close_page dispatches here to close preview tabs in the calling session's @@ -923,7 +950,9 @@ setDeployedInSessionHandler(({ sessionId: callerSessionId, kind, path }) => { const session = sessionState.sessions.find((s) => s.id === sessionId) const runtime = runtimes.get(sessionId) if (!session?.workspace_id || !runtime) return - if (runtime.slot(kind).loadedPath !== path) return + // Peek without creating a cell: a deploy for an item with no open editor tab + // must not allocate an empty cell that lingers until the next prune. + if (runtime.loadedEditorPath(kind, path) !== path) return runtime.syncPreviewWithDeployed(session.workspace_id, kind, path) }) diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index 7896f6915d..8382c201ce 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -37,21 +37,12 @@ import { sendUserToast } from '$lib/toast' import type HistoryManager from '$lib/components/copilot/chat/HistoryManager.svelte' import { onUserChange, scopedKey } from '$lib/userScopedStorage' -// Kinds the in-session editor pane can host. Legacy drag-and-drop apps are -// intentionally not previewable — only code-based 'raw_app' apps are. A -// 'pipeline' target's `path` is the folder name (not a workspace item path): -// it hosts the data-pipeline graph editor for that folder, which uses its own -// fetch/draft model rather than the single-item load slots the other kinds share. +// A destination the session preview can open as an editor: a workspace item +// (`path`) for flow/script/raw_app, or — for 'pipeline' — a folder name (not an +// item path), which resolves to the data-pipeline graph editor for that folder. +// Legacy drag-and-drop apps aren't previewable; only code-based 'raw_app' apps. export type SessionTarget = { kind: 'flow' | 'script' | 'raw_app' | 'pipeline'; path: string } -// Useful for filtering dropdowns / pickers to "items the side panel can open". -export const EDITOR_TARGET_KINDS: ReadonlySet = new Set([ - 'flow', - 'script', - 'raw_app', - 'pipeline' -]) - // Whether the session points at a workspace that is itself a fork (i.e. // has a parent). Used by the sidebar to pick between a root (Building) // icon and a fork icon. @@ -97,7 +88,6 @@ export type Session = { // workspace_id, not this field. Root sessions store the same id in both fields. workspace_root_id?: string chatId?: string - target?: SessionTarget summary?: string summarySource?: SessionSummarySource createdAt: number @@ -151,12 +141,12 @@ interface SessionSchema extends DBSchema { } // Normalise legacy localStorage records in place: drop empty-string -// workspace_id (older drafts used '' as a missing marker), migrate the -// deprecated 'rawapp' target.kind, and coerce unknown summarySource values. -// Operates on raw parsed JSON, so the record is loosely typed. +// workspace_id (older drafts used '' as a missing marker), drop the retired +// `target` field (the preview is tab-driven now), and coerce unknown +// summarySource values. Operates on raw parsed JSON, so the record is loosely typed. function normalizeLegacySession(s: Record): void { if (s.workspace_id === '') delete s.workspace_id - if (s.target?.kind === 'rawapp') s.target.kind = 'raw_app' + delete s.target if ( s.summarySource !== undefined && s.summarySource !== 'placeholder' && @@ -804,20 +794,6 @@ export function getEffectiveWorkspaceId(session: Session): string | undefined { return session.workspace_id ?? session.pending_workspace_id } -// Canonical mutation for session.target. Persists, optionally seeds the -// session summary, and centralises the path so callers don't reach into -// session.target directly. -export function setSessionTarget(id: string, target: SessionTarget, summary?: string): void { - const s = sessionState.sessions.find((x) => x.id === id) - if (!s) return - s.target = target - if (!s.summary && summary) { - s.summary = summary - s.summarySource = 'generated' - } - void putSession(s) -} - // Persist the session's preview tabs. Fire-and-forget write-behind (transient // sessions land in the localStorage draft slot). export function setSessionTabs(id: string, tabs: SessionPreviewTab[], activeTabId: string): void { diff --git a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts index 2ebe031597..131a299082 100644 --- a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts +++ b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts @@ -123,7 +123,6 @@ describe('sessionState IndexedDB persistence', () => { session({ id: 't1b', transient: true, - target: { kind: 'script', path: 'u/me/foo' }, previewTabs: [{ id: 'session', url: '/x', loc: '/x' }], activePreviewTabId: 'session', previewCollapsed: false @@ -132,7 +131,6 @@ describe('sessionState IndexedDB persistence', () => { await rehydrate(user) await flush() const restored = sessionState.sessions.find((s) => s.id === 't1b') - expect(restored?.target).toEqual({ kind: 'script', path: 'u/me/foo' }) expect(restored?.previewTabs).toEqual([{ id: 'session', url: '/x', loc: '/x' }]) expect(restored?.activePreviewTabId).toBe('session') expect(restored?.previewCollapsed).toBe(false) diff --git a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts index e045632e3c..45d72ae3ae 100644 --- a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts @@ -7,7 +7,6 @@ import { sessionInCurrentFamily, sessionState, setSessionPendingWorkspace, - setSessionTarget, type SessionTarget } from './sessionState.svelte' import { sessionTargetHref } from './sessionMode.svelte' @@ -66,8 +65,8 @@ export async function exitSessionMode(): Promise { await goto(target) } -// Open a fresh AI session pre-targeted at an editor (flow/script/raw_app), then -// route into session mode. The session preview loads that editor via its target, +// Open a fresh AI session showing an editor (flow/script/raw_app) in its preview, +// then route into session mode. The preview loads the item from its live draft, // so the caller MUST persist any unsaved edits first (e.g. save a draft) for the // preview to reflect the live state. `workspaceId` scopes the session to the // editor's workspace (instead of createSession's root default) so it opens the @@ -76,16 +75,14 @@ export async function openEditorInSession( target: SessionTarget, workspaceId?: string ): Promise { - // createSession() reuses an existing transient draft, which may still be - // pointed at a *different* item — and its preview tabs (persisted with the - // draft and/or held by a live runtime) keep showing that old target unless - // the tab model is reset along with the target field. + // createSession() reuses an existing transient draft, whose preview tabs + // (persisted with the draft and/or held by a live runtime) may still show a + // different item — so seed the preview with a single tab on `target`, resetting + // whatever it was showing. const session = createSession() - const retargeted = session.target?.kind !== target.kind || session.target?.path !== target.path if (workspaceId) setSessionPendingWorkspace(session.id, workspaceId) - setSessionTarget(session.id, target) const url = sessionTargetHref(target) - if (url && retargeted) { + if (url) { // Dynamic import: a static one would drag the runtime's heavy graph // (chat manager → monaco) into this thin navigation seam, breaking its // node-run unit tests. diff --git a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts index e5af840242..7e89d3b945 100644 --- a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts +++ b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts @@ -37,7 +37,13 @@ export interface UserDraftSyncOptions { * `loadedX !== path` guards. */ ready: () => boolean - codec: DraftSyncCodec + /** + * Reactive codec. Retargeting a mounted editor (breadcrumb / in-editor link) + * changes `path` without remounting, and the codec closes over one + * `(kind, path)` cell's store — so it must be re-read per path, not captured + * once, or the sync would read/write the previously-targeted cell. + */ + codec: () => DraftSyncCodec } /** @@ -61,18 +67,19 @@ export interface UserDraftSyncOptions { * Must be called once during component init (registers `useMany` + two `$effect`s). */ export function useUserDraftSync(opts: UserDraftSyncOptions): void { - const { codec } = opts const handles = UserDraft.useMany(() => { const p = opts.path() const ws = opts.workspace() - return p && ws ? [{ itemKind: codec.itemKind, path: p, workspace: ws }] : [] + return p && ws ? [{ itemKind: opts.codec().itemKind, path: p, workspace: ws }] : [] }) let lastInboundSig: string | undefined = $state(undefined) // inbound: handle.draft → store. Re-runs when the handle's draft changes - // (chat write / another session's edit). The store read happens inside - // applyDraftToStore under untrack so the editor's own mutations don't refire. + // (chat write / another session's edit) or the target retargets (new codec). + // The store read happens inside applyDraftToStore under untrack so the + // editor's own mutations don't refire. $effect(() => { + const codec = opts.codec() const incoming = handles[0]?.draft if (incoming == null) return const sig = codec.sig(incoming) @@ -101,6 +108,7 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void let pendingFlush: (() => void) | undefined $effect(() => { if (!opts.ready()) return + const codec = opts.codec() const draft = codec.storeToDraft(undefined) if (draft == null) return const sig = codec.sig(draft) diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 232f2fb003..6571e6a1b3 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -38,8 +38,7 @@ import { getOrCreateRuntime, getRuntime, - listRuntimes, - promoteEditorWarm + listRuntimes } from '$lib/components/sessions/sessionRuntime.svelte' import { markSessionSeen } from '$lib/components/sessions/sessionUnread.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' @@ -52,6 +51,7 @@ previewLocationLabel, type PreviewTarget } from '$lib/components/sessions/previewRouter' + import { toolReloadEffect, tabsToReload } from '$lib/components/sessions/previewReload' import { leafKeyFor, type WorkspaceItem } from '$lib/components/workspacePicker' import { splitterPointerCapture } from '$lib/utils/splitterPointerCapture' @@ -132,14 +132,6 @@ .filter((s): s is NonNullable => s != null) ) - // Promote the active session in the LRU. Mutations untracked so the effect - // only re-runs when activeSession changes, not on its own writes. - $effect(() => { - const id = activeSession?.id - if (!id) return - untrack(() => promoteEditorWarm(id)) - }) - // Mark the active session "seen" up to its current message count: arrive → // clear unread; AI streams a new message while we're here → clear again. The // effect depends only on the length, not the array contents, so token-by-token @@ -317,61 +309,49 @@ } } - // Reload mounted preview tabs affected by a mutating chat tool (write_/patch_/ - // delete_/deploy_/…; read/test/navigate tools don't match). Scoped to the changed - // item so editing one item never blank-reboots an unrelated item's preview iframe - // (a full-page /apps_raw/edit reload is jarring). + // Reload mounted preview tabs affected by a mutating chat tool. Item and pipeline + // tabs are live editors that self-sync from the store the chat mutates, so nothing + // reloads them. Only list-page tabs (schedules, resources, …) are iframes, and each + // reloads only when a tool actually changed *its* page (toolReloadEffect) — so a + // schedule write leaves the Resources tab alone, and a purely local tool (saving + // user instructions) reloads nothing. const tabHosts: Record = {} - const MUTATING_TOOL_RE = /^(write_|patch_|delete_|deploy_|discard_|set_|create_|update_|remove_)/ - let reloadHandle: ReturnType | undefined - // Drained each flush: item paths touched since the last flush, and a flag for an - // unresolved mutation that forces a full reload (safe fallback). - let pendingReloadPaths = new Set() - let pendingReloadAll = false - // Reload the batched-mutation tabs across all warm sessions' mounted tabs (a - // hidden preview would otherwise show pre-mutation content on return). `null` - // reloads all; otherwise an item-route iframe reloads only when its item was - // touched. Non-item pages always reload; a live-editor slot no-ops in reload(). - function reloadTabs(paths: Set | null) { + let reloadHandle: ReturnType | undefined + // Base-stripped list-page paths (e.g. `/schedules`) a chat round touched since + // the last flush — see toolReloadEffect for how tools map to pages. + let pendingPages = new Set() + + // Reload the mounted list-page tabs a chat round changed, across all warm + // sessions (a hidden preview would otherwise show pre-mutation content on + // return). tabsToReload picks only the tabs whose page is in `pages`. + function reloadTabs(pages: Set) { for (const s of warmSessions) { - const tabs = getRuntime(s.id)?.previewTabs?.tabs ?? [] - for (const tab of tabs) { + const owner = getRuntime(s.id)?.previewTabs + if (!owner) continue + for (const tab of tabsToReload(owner.tabs, pages)) { const key = tabKey(s.id, tab.id) - if (!mountedTabKeys.has(key)) continue - if (paths) { - const route = parsePreviewItemRoute(tab.url) - if (route && !paths.has(route.itemPath)) continue - } - tabHosts[key]?.reload() + if (mountedTabKeys.has(key)) tabHosts[key]?.reload() } } } function flushReload() { - const paths = pendingReloadAll ? null : pendingReloadPaths - pendingReloadPaths = new Set() - pendingReloadAll = false - reloadTabs(paths) + const pages = pendingPages + pendingPages = new Set() + reloadTabs(pages) } $effect(() => { // Debounced so a burst of writes (the AI editing several files) reloads once. setToolCompletionListener((name, args) => { - if (!MUTATING_TOOL_RE.test(name)) return - // A workspace item path scopes the reload to that item. The raw-app file - // tools (write_app_file, …) pass a leading-'/' frontend file path and edit - // the active session's target app, so scope to the target. Anything else is - // unresolved → reload everything (safe fallback). - const p = typeof args?.path === 'string' ? args.path : undefined - if (p && !p.startsWith('/')) pendingReloadPaths.add(p) - else if (p && activeSession?.target?.path) pendingReloadPaths.add(activeSession.target.path) - else pendingReloadAll = true + const { pages } = toolReloadEffect(name, args) + if (pages.length === 0) return + for (const p of pages) pendingPages.add(p) clearTimeout(reloadHandle) reloadHandle = setTimeout(flushReload, 500) }) return () => { clearTimeout(reloadHandle) - pendingReloadPaths = new Set() - pendingReloadAll = false + pendingPages = new Set() setToolCompletionListener(undefined) } }) @@ -565,7 +545,7 @@ : 'z-0 opacity-0 pointer-events-none'}" aria-hidden={s.id !== activeSession?.id} > - +
{/each}
From c4cb2f373b6361f0f3ce6b1c8e32a4c010207760 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 8 Jul 2026 16:35:33 +0200 Subject: [PATCH 264/273] fix: preserve worker group tag override on 'Run again' (#10004) * fix: preserve worker group tag override on 'Run again' Co-Authored-By: Claude Fable 5 * fix: keep tag override in sharable hash on args change Co-Authored-By: Claude Fable 5 * fix: disambiguate reserved __tag hash key from args named __tag Co-Authored-By: Claude Fable 5 * fix: prefix carried tag in sharable hash and react to tag changes Co-Authored-By: Claude Fable 5 * fix: re-resolve dynamic tags on 'Run again' with an explanatory note Co-Authored-By: Claude Fable 5 * fix: treat only $args-templated tags as dynamic on 'Run again' Co-Authored-By: Claude Fable 5 * fix: let a carried tag coexist with an arg named __tag via duplicate keys Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- frontend/src/lib/components/RunForm.svelte | 9 ++- frontend/src/lib/utils.test.ts | 61 +++++++++++++++++++ frontend/src/lib/utils.ts | 45 ++++++++++++-- .../(logged)/flows/get/[...path]/+page.svelte | 24 +++++++- .../(root)/(logged)/run/[...run]/+page.svelte | 5 +- .../scripts/get/[...hash]/+page.svelte | 19 +++++- 6 files changed, 154 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index b1b562b04d..89eaf8f7ab 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -114,6 +114,7 @@ scheduledForStr: string | undefined invisible_to_owner: boolean | undefined overrideTag: string | undefined + overrideTagNote?: string args?: Record jsonView?: boolean isValid?: boolean @@ -132,6 +133,7 @@ scheduledForStr = $bindable(), invisible_to_owner = $bindable(), overrideTag = $bindable(), + overrideTagNote = undefined, args = $bindable(), jsonView = false, isValid = $bindable(true) @@ -160,7 +162,7 @@ debounced && clearTimeout(debounced) debounced = setTimeout(() => { const nurl = new URL(window.location.href) - nurl.hash = computeSharableHash(args) + nurl.hash = computeSharableHash(args, overrideTag) try { replaceState(nurl.toString(), page.state) @@ -201,6 +203,7 @@ jsonEditor?.setCode(code) } $effect(() => { + overrideTag Object.keys(args ?? {}).forEach((key) => { args?.[key] }) @@ -387,6 +390,10 @@
tag override: {overrideTag}
+ {:else if overrideTagNote} +
+ {overrideTagNote} +
{/if} {#if invisible_to_owner}
diff --git a/frontend/src/lib/utils.test.ts b/frontend/src/lib/utils.test.ts index dc7e52e9f4..accdf73b15 100644 --- a/frontend/src/lib/utils.test.ts +++ b/frontend/src/lib/utils.test.ts @@ -1,6 +1,9 @@ import { describe, it, expect } from 'vitest' import { cleanValueProperties, + computeSharableHash, + extractTagFromSharableHash, + isDynamicTag, getQueryStmtCountHeuristic, parseDbInputFromAssetSyntax } from './utils' @@ -390,3 +393,61 @@ describe('cleanValueProperties', () => { expect(input).toHaveProperty('created_at') }) }) + +describe('computeSharableHash / extractTagFromSharableHash', () => { + function roundTrip(hash: string) { + const params = new URLSearchParams(hash) + const tag = extractTagFromSharableHash(params) + const args = Object.fromEntries([...params.entries()].map(([k, v]) => [k, JSON.parse(v)])) + return { tag, args } + } + + it('carries the tag under the reserved __tag key alongside JSON-encoded args', () => { + const hash = computeSharableHash({ name: 'world' }, 'my-custom-tag') + expect(roundTrip(hash)).toEqual({ tag: 'my-custom-tag', args: { name: 'world' } }) + }) + + it('omits __tag when no tag is given', () => { + const hash = computeSharableHash({ name: 'world' }) + expect(roundTrip(hash)).toEqual({ tag: undefined, args: { name: 'world' } }) + }) + + it('preserves an arg named __tag instead of misreading it as a tag', () => { + const hash = computeSharableHash({ __tag: 'value', name: 'world' }) + expect(roundTrip(hash)).toEqual({ tag: undefined, args: { __tag: 'value', name: 'world' } }) + }) + + it('carries a tag alongside an arg named __tag, preserving both', () => { + const hash = computeSharableHash({ __tag: 'value', name: 'world' }, 'my-custom-tag') + expect(roundTrip(hash)).toEqual({ + tag: 'my-custom-tag', + args: { __tag: 'value', name: 'world' } + }) + }) + + it('carries JSON-parseable tags like 123 or true without corrupting args', () => { + expect(roundTrip(computeSharableHash({ name: 'world' }, '123'))).toEqual({ + tag: '123', + args: { name: 'world' } + }) + expect(roundTrip(computeSharableHash({}, 'true'))).toEqual({ tag: 'true', args: {} }) + }) + + it('carries a tag that itself starts with the value prefix', () => { + expect(roundTrip(computeSharableHash({}, 't:odd'))).toEqual({ tag: 't:odd', args: {} }) + }) +}) + +describe('isDynamicTag', () => { + it('detects args interpolation placeholders', () => { + expect(isDynamicTag('worker-$args[env]')).toBe(true) + }) + + it('is false for plain tags, $workspace-only tags, and undefined', () => { + expect(isDynamicTag('gpu-heavy')).toBe(false) + // $workspace resolves identically on a re-run, so pinning the resolved value is fine + expect(isDynamicTag('$workspace-gpu')).toBe(false) + expect(isDynamicTag('')).toBe(false) + expect(isDynamicTag(undefined)).toBe(false) + }) +}) diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index fdca33fa22..b2f8e0d346 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1137,8 +1137,19 @@ export function extractCustomProperties(styleStr: string): string { return customStyleStr } -export function computeSharableHash(args: any) { - let nargs = {} +// Value prefix marking the reserved `__tag` key as a carried tag: no JSON-encoded +// arg value can start with `t:` (JSON strings start with `"`, numbers with a digit +// or `-`, etc.), so it cannot be confused with a genuine arg named `__tag` +const SHARABLE_HASH_TAG_PREFIX = 't:' + +// `tag` is carried as the reserved key `__tag` with a SHARABLE_HASH_TAG_PREFIX value; +// entry pairs allow a duplicate `__tag` key so an arg with that name can coexist with +// the carried tag (they are told apart by the value prefix) +export function computeSharableHash(args: any, tag?: string) { + let entries: [string, string][] = [] + if (tag) { + entries.push(['__tag', SHARABLE_HASH_TAG_PREFIX + tag]) + } for (let k in args) { let v = args[k] if (v !== undefined) { @@ -1148,11 +1159,11 @@ export function computeSharableHash(args: any) { console.error(`Value at key ${k} too big (${size}) to be shared`) return '' } - nargs[k] = JSON.stringify(v) + entries.push([k, JSON.stringify(v)]) } } try { - let r = new URLSearchParams(nargs).toString() + let r = new URLSearchParams(entries).toString() return r.length > 1000000 ? '' : r } catch (e) { console.error('Error computing sharable hash', e) @@ -1160,6 +1171,32 @@ export function computeSharableHash(args: any) { } } +// `$args[...]` tags are resolved by the backend at push time from the run's args; a +// job's stored tag is the resolved value, so re-running with it would pin a value +// that no longer matches edited args. `$workspace` is also interpolated but resolves +// identically on a re-run (same workspace), so it does not make a tag dynamic here. +export function isDynamicTag(tag: string | undefined): boolean { + return !!tag && tag.includes('$args[') +} + +// Counterpart of computeSharableHash's `tag`: extracts and removes the carried tag. +// Only SHARABLE_HASH_TAG_PREFIX-prefixed `__tag` values are carried tags; any other +// `__tag` value is a genuine arg with that name and is left in `params` for arg parsing. +export function extractTagFromSharableHash(params: URLSearchParams): string | undefined { + const values = params.getAll('__tag') + const carried = values.find((v) => v.startsWith(SHARABLE_HASH_TAG_PREFIX)) + if (carried == undefined) { + return undefined + } + params.delete('__tag') + for (const v of values) { + if (!v.startsWith(SHARABLE_HASH_TAG_PREFIX)) { + params.append('__tag', v) + } + } + return carried.slice(SHARABLE_HASH_TAG_PREFIX.length) +} + export function toCamel(s: string) { return s.replace(/([-_][a-z])/gi, ($1) => { return $1.toUpperCase().replace('-', '').replace('_', '') diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index f24c96e77f..b50bd4e1da 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -8,7 +8,14 @@ type TriggersCount, type WorkspaceDeployUISettings } from '$lib/gen' - import { canWrite, defaultIfEmptyString, emptyString, urlParamsToObject } from '$lib/utils' + import { + canWrite, + defaultIfEmptyString, + emptyString, + urlParamsToObject, + extractTagFromSharableHash, + isDynamicTag + } from '$lib/utils' import { isDeployable, ALL_DEPLOYABLE } from '$lib/utils_deployable' import DetailPageLayout from '$lib/components/details/DetailPageLayout.svelte' @@ -78,6 +85,9 @@ let scheduledForStr: string | undefined = $state(undefined) let invisible_to_owner: boolean | undefined = $state(undefined) let overrideTag: string | undefined = $state(undefined) + let overrideTagNote: string | undefined = $state(undefined) + // Tag carried over from 'Run again', pending the dynamic-tag check in loadFlow + let carriedTag: string | undefined = undefined let inputSelected: 'saved' | 'history' | undefined = $state(undefined) let jsonView = $state(false) let deploymentInProgress = $state(false) @@ -168,6 +178,15 @@ }) pinnedVersion = undefined } + // A carried tag is the previous run's resolved value; when the flow's tag is + // dynamic, drop it so the backend re-resolves from the (possibly edited) args + if (carriedTag && isDynamicTag(flow.tag)) { + if (overrideTag === carriedTag) { + overrideTag = undefined + overrideTagNote = `tag ${flow.tag} is resolved at run time, so the previous run's tag ${carriedTag} was not applied` + } + carriedTag = undefined + } if (!flow.path.startsWith(`u/${$userStore?.username}`) && flow.path.split('/').length > 2) { invisible_to_owner = flow.visible_to_runner_only } @@ -246,6 +265,8 @@ if (hash.length > 1) { try { let searchParams = new URLSearchParams(hash.slice(1)) + carriedTag = extractTagFromSharableHash(searchParams) + overrideTag = carriedTag let params = [...Object.entries(urlParamsToObject(searchParams))].map(([k, v]) => [ k, JSON.parse(v) @@ -710,6 +731,7 @@ bind:scheduledForStr bind:invisible_to_owner bind:overrideTag + {overrideTagNote} viewKeybinding {loading} autofocus diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 14acd3c89c..31327573fb 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -378,7 +378,8 @@ const commonArgs = { workspace: $workspaceStore!, - requestBody: args + requestBody: args, + tag: job?.tag } if (job?.job_kind == 'script' || job?.job_kind == 'script_hub' || job?.job_kind == 'flow') { let id @@ -723,7 +724,7 @@ {#if job?.job_kind === 'script' || job?.job_kind === 'script_hub' || job?.job_kind === 'flow'}
+
+ {/snippet} + + {/if} +
+ {/if}
diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index f66d2e508b..b6a5c6611f 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -136,9 +136,14 @@ loadedFromDraft = false, othersDraftsCount = 0, onOpenOthersDrafts, - onTestJob + onTestJob, + condensedHeader = false }: FlowBuilderProps = $props() + // Top-bar button size + bar height. Condensed (session preview) uses the + // smallest well-supported unified size (`sm`) so the bar is thinner. + const headerBtnSize = $derived(condensedHeader ? 'sm' : 'md') + // The workspace this editor operates on: deploy, save-draft, trigger loading // and the AutosaveIndicator all target it. Falls back to the global store, so // the full-page editor is unchanged; the sessions preview overrides it to the @@ -1101,7 +1106,9 @@
@@ -1110,6 +1117,7 @@ bind:path={$pathStore} savedPath={initialPath} onBehalfOfEmail={$savedOnBehalfOfEmail} + hidePath={condensedHeader} workspaceId={autosaveWorkspace} onNavigate={(item) => onNavigate?.(item)} /> @@ -1132,7 +1140,7 @@ {/if}
- + {#if $tutorialsToDo.includes(getTutorialIndex('flow-live-tutorial')) || $tutorialsToDo.includes(getTutorialIndex('troubleshoot-flow'))}
@@ -1182,6 +1191,7 @@ {#snippet previewButtons()} { select('Trigger') handleSelectTriggerFromKind(triggersState, triggersCount, initialPath, e.detail.kind) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 7b12c02f51..a334ac4c50 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -140,9 +140,14 @@ onResetToDeployed, loadedFromDraft = false, othersDraftsCount = 0, - onOpenOthersDrafts + onOpenOthersDrafts, + condensedHeader = false }: ScriptBuilderProps = $props() + // Top-bar button size + bar height. Condensed (session preview) uses the + // smallest well-supported unified size (`sm`) so the bar is thinner. + const headerBtnSize = $derived(condensedHeader ? 'sm' : 'md') + export function getInitialAndModifiedValues(): SavedAndModifiedValue { return { savedValue: savedScript, @@ -1898,7 +1903,10 @@
-
+
{#if customUi?.topBar?.path != false}
@@ -1919,6 +1927,7 @@ kind="script" summaryEditable={customUi?.topBar?.editableSummary != false} pathEditable={customUi?.topBar?.editablePath != false} + hidePath={condensedHeader} workspaceId={autosaveWorkspace} onNavigate={(item) => onNavigate?.(item)} /> @@ -1954,7 +1963,7 @@ aiId="script-builder-settings" aiDescription="Script builder settings to configure metadata, runtime, triggers, and generated UI." variant="default" - unifiedSize="md" + unifiedSize={headerBtnSize} on:click={() => (metadataOpen = true)} startIcon={{ icon: Settings }} iconOnly={compactTopbar} @@ -1977,7 +1986,7 @@
@@ -2027,6 +2037,7 @@ handleEditScript(false, detail)} /> diff --git a/frontend/src/lib/components/WorkerTagSelect.svelte b/frontend/src/lib/components/WorkerTagSelect.svelte index 214b378dbd..fe00f07f9d 100644 --- a/frontend/src/lib/components/WorkerTagSelect.svelte +++ b/frontend/src/lib/components/WorkerTagSelect.svelte @@ -17,6 +17,7 @@ disabled = false, placeholder, inputClass, + size = 'md', workspaceId = undefined }: { tag: string | undefined @@ -27,6 +28,9 @@ language?: string class?: string inputClass?: string + /** Forwarded to the underlying Select — controls the input height. The + * condensed session top bar passes `sm` to match its smaller buttons. */ + size?: 'sm' | 'md' | 'lg' // Workspace to read custom tags and worker availability from. Defaults to // $workspaceStore. Session editors act on a workspace that differs from the // navigation one, so they pass their effective workspace to keep the tag @@ -164,6 +168,7 @@ clearable class="w-full" bind:open + {size} {inputClass} {disabled} placeholder={nullTag ? nullTag : (placeholder ?? 'lang default')} diff --git a/frontend/src/lib/components/flow_builder.ts b/frontend/src/lib/components/flow_builder.ts index 48340e937e..ce848c6f83 100644 --- a/frontend/src/lib/components/flow_builder.ts +++ b/frontend/src/lib/components/flow_builder.ts @@ -62,4 +62,8 @@ export type FlowBuilderProps = { // Fired whenever a test run is started from the flow editor, with the // preview job id. Used by whitelabel embedders to track test jobs. onTestJob?: (e: { jobId: string }) => void + // Condensed top bar: smaller (sm) buttons, a shorter bar, and the + // EditorHeader's path/breadcrumb row dropped (summary only). Used by the + // session preview to save vertical room. + condensedHeader?: boolean } diff --git a/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte b/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte index 2f1c016385..dbebc82f70 100644 --- a/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte +++ b/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte @@ -1,5 +1,6 @@ {#if helperScript} - +
{#if inputType === 'dynmultiselect'} diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index f5062bc008..1e6a29388f 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -117,6 +117,10 @@ right?: import('svelte').Snippet openAiChat?: boolean moduleId?: string + // Workspace to scope variable/resource/data-table lookups to. Defaults to + // the nav `$workspaceStore`; an AI-session live editor passes the session's + // acting workspace (a fork) so the helper pickers hit the right workspace. + workspace?: string } let { @@ -141,9 +145,12 @@ showHistoryDrawer = $bindable(false), right, openAiChat = false, - moduleId = undefined + moduleId = undefined, + workspace = undefined }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + let contextualVariablePicker: ItemPicker | undefined = $state() let variablePicker: ItemPicker | undefined = $state() let resourcePicker: ItemPicker | undefined = $state() @@ -350,12 +357,12 @@ }) async function loadVariables() { - return await VariableService.listVariable({ workspace: $workspaceStore ?? '' }) + return await VariableService.listVariable({ workspace: ws ?? '' }) } async function loadContextualVariables() { return await VariableService.listContextualVariables({ - workspace: $workspaceStore ?? 'NO_W' + workspace: ws ?? 'NO_W' }) } @@ -366,7 +373,7 @@ async function onScriptPick(e: { detail: { path: string } }) { codeObj = undefined codeViewer?.openDrawer?.() - codeObj = await getScriptByPath(e.detail.path ?? '') + codeObj = await getScriptByPath(e.detail.path ?? '', ws) } const dispatch = createEventDispatcher() @@ -423,7 +430,7 @@ async function resourceTypePickCallback(name: string) { if (!editor) return const resourceType = await ResourceService.getResourceType({ - workspace: $workspaceStore ?? 'NO_W', + workspace: ws ?? 'NO_W', path: name }) @@ -785,8 +792,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS buttons={{ 'Edit/View': (x) => resourceEditor?.initEdit(x) }} extraField="description" extraField2="resource_type" - loadItems={async () => - await ResourceService.listResource({ workspace: $workspaceStore ?? 'NO_W' })} + loadItems={async () => await ResourceService.listResource({ workspace: ws ?? 'NO_W' })} > {#snippet submission()}
@@ -812,12 +818,15 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS documentationLink="https://www.windmill.dev/docs/core_concepts/resources_and_types" itemName="Resource Type" extraField="name" - loadItems={async () => - await ResourceService.listResourceType({ workspace: $workspaceStore ?? 'NO_W' })} + loadItems={async () => await ResourceService.listResourceType({ workspace: ws ?? 'NO_W' })} /> {/if} - - + + {#if showDucklakePicker} - (await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? 'NO_W' })).map( - (path) => ({ path }) - )} + (await WorkspaceService.listDucklakes({ workspace: ws ?? 'NO_W' })).map((path) => ({ path }))} > {#snippet submission()}
@@ -885,9 +892,9 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables" itemName="data table" loadItems={async () => - (await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map( - (d) => ({ path: d.name }) - )} + (await WorkspaceService.listDataTables({ workspace: ws ?? 'NO_W' })).map((d) => ({ + path: d.name + }))} > {#snippet submission()}
@@ -923,7 +930,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS extraField2="resource_type" loadItems={async () => await ResourceService.listResource({ - workspace: $workspaceStore ?? 'NO_W', + workspace: ws ?? 'NO_W', resourceType: 'postgresql,mysql,bigquery' })} > diff --git a/frontend/src/lib/components/EditorHeader.svelte b/frontend/src/lib/components/EditorHeader.svelte index b6e0bb4c82..fdf18b4b50 100644 --- a/frontend/src/lib/components/EditorHeader.svelte +++ b/frontend/src/lib/components/EditorHeader.svelte @@ -220,6 +220,7 @@ {kind} size="sm" drawerOffset={4000} + workspaceOverride={workspaceId} /> {#if savedPath && path && path !== savedPath} @@ -60,18 +64,20 @@ on:click={async () => { if (asset.kind === 'resource' && isDbType(_resourceMetadata?.resource_type)) { let [resourcePath, specificTable] = asset.path.split('?table=') - dbManagerDrawer?.openDrawer({ - type: 'database', - resourceType: _resourceMetadata.resource_type, - resourcePath, - specificTable - }) + dbManagerDrawer?.openDrawer( + { + type: 'database', + resourceType: _resourceMetadata.resource_type, + resourcePath, + specificTable + }, + ws + ) } else if (asset.kind === 's3object' && isS3Uri(assetUri)) { s3FilePicker?.open(assetUri) } else if (asset.kind === 'volume') { - const storage = - (await VolumeService.getVolumeStorage({ workspace: $workspaceStore! })) ?? undefined - s3FilePicker?.open({ s3: `volumes/${$workspaceStore}/${asset.path}/`, storage }) + const storage = (await VolumeService.getVolumeStorage({ workspace: ws! })) ?? undefined + s3FilePicker?.open({ s3: `volumes/${ws}/${asset.path}/`, storage }) } else if (asset.kind === 'ducklake') { let ducklake = asset.path.split('/')[0] let specificTableSplit = asset.path.split('/')[1]?.split('.') as string[] | undefined @@ -79,7 +85,7 @@ specificTableSplit?.length === 2 ? [specificTableSplit[0], specificTableSplit[1]] : [undefined, specificTableSplit?.[0]] - dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake, specificSchema, specificTable }) + dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake, specificSchema, specificTable }, ws) } else if (asset.kind === 'datatable') { let datatable = asset.path.split('/')[0] let specificTableSplit = asset.path.split('/')[1]?.split('.') as string[] | undefined @@ -87,13 +93,16 @@ specificTableSplit?.length === 2 ? [specificTableSplit[0], specificTableSplit[1]] : [undefined, specificTableSplit?.[0]] - dbManagerDrawer?.openDrawer({ - type: 'database', - resourceType: 'postgresql', - resourcePath: `datatable://${datatable}`, - specificTable, - specificSchema - }) + dbManagerDrawer?.openDrawer( + { + type: 'database', + resourceType: 'postgresql', + resourcePath: `datatable://${datatable}`, + specificTable, + specificSchema + }, + ws + ) } onClick?.() }} diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index f92072b5fd..b4dde43289 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -73,7 +73,8 @@ runPreview(previewArgs, undefined) } - const { flowStateStore, pathStore } = getContext('FlowEditorContext') + const { flowStateStore, pathStore, opWorkspace } = + getContext('FlowEditorContext') const dispatch = createEventDispatcher() export async function runPreview( @@ -82,7 +83,15 @@ ) { progressBar?.reset() const newFlow = { value: { modules }, summary: '' } - jobId = await runFlowPreview(args, newFlow, $pathStore, restartedFrom) + jobId = await runFlowPreview( + args, + newFlow, + $pathStore, + restartedFrom, + undefined, + undefined, + opWorkspace?.() + ) isRunning = true } @@ -130,7 +139,7 @@ try { jobId && (await JobService.cancelQueuedJob({ - workspace: $workspaceStore ?? '', + workspace: opWorkspace?.() ?? $workspaceStore ?? '', id: jobId, requestBody: {} })) @@ -177,6 +186,7 @@ {#if jobId} { job = newJob diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index bf06628cc0..dd359fb006 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -133,8 +133,11 @@ fakeInitialPath, customUi, executionCount, - devTempScriptRefs + devTempScriptRefs, + opWorkspace } = $state(getContext('FlowEditorContext')) + // Acting workspace when previewing inside an AI session; else the nav workspace. + let opWs = $derived(opWorkspace?.() ?? $workspaceStore) const dispatch = createEventDispatcher() let renderCount: number = $state(0) @@ -193,14 +196,15 @@ lastPreviewFlow = JSON.stringify(flowStore.val) flowProgressBar?.reset() const newFlow = extractFlow(previewMode) - args = await processSecretArgs(args, flowStore.val.schema as any) + args = await processSecretArgs(args, flowStore.val.schema as any, opWs) newJobId = await runFlowPreview( args, newFlow, $pathStore, restartedFrom, conversationId, - devTempScriptRefs?.() + devTempScriptRefs?.(), + opWorkspace?.() ) jobId = newJobId isRunning = true @@ -286,7 +290,7 @@ subJobIds.map(async (subId) => { try { const subJob = await JobService.getJob({ - workspace: $workspaceStore!, + workspace: opWs!, id: subId }) flowRecording.addCompletedJob(subId, subJob) @@ -332,11 +336,11 @@ untrack(() => { for (const mod of modules) { if (mod.job) { - flowRecording.watchSubJob(mod.job, $workspaceStore!) + flowRecording.watchSubJob(mod.job, opWs!) } } if (job?.flow_status?.failure_module?.job) { - flowRecording.watchSubJob(job.flow_status.failure_module.job, $workspaceStore!) + flowRecording.watchSubJob(job.flow_status.failure_module.job, opWs!) } }) } @@ -347,7 +351,7 @@ try { jobId && (await JobService.cancelQueuedJob({ - workspace: $workspaceStore ?? '', + workspace: opWs ?? '', id: jobId, requestBody: {} })) @@ -514,6 +518,7 @@ runnableId={$initialPathStore} stablePathForCaptures={$initialPathStore || fakeInitialPath} runnableType={'FlowPath'} + workspace={opWs} previewArgs={previewArgs.val} on:openTriggers on:select={(e) => { @@ -562,6 +567,7 @@ { @@ -632,7 +638,7 @@
{ isRunning = false diff --git a/frontend/src/lib/components/HistoricInputs.svelte b/frontend/src/lib/components/HistoricInputs.svelte index 3c784f354f..85f2603005 100644 --- a/frontend/src/lib/components/HistoricInputs.svelte +++ b/frontend/src/lib/components/HistoricInputs.svelte @@ -17,6 +17,8 @@ placement?: 'bottom-start' | 'top-start' | 'bottom-end' | 'top-end' limitPayloadSize?: boolean searchArgs?: Record | undefined + /** Workspace to read run history from; defaults to the nav workspace. */ + workspace?: string } let { @@ -26,9 +28,12 @@ showAuthor = false, placement = 'top-end', limitPayloadSize = false, - searchArgs = undefined + searchArgs = undefined, + workspace = undefined }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + let historicList: HistoricList | undefined = $state(undefined) const dispatch = createEventDispatcher() @@ -111,7 +116,7 @@ jobKinds: getJobKinds(runnableType), syncQueuedRunsCount: false, refreshRate: 10000, - currentWorkspace: $workspaceStore ?? '', + currentWorkspace: ws ?? '', skip: !runnableId, excludesEntrypointOverride: true }) satisfies UseJobLoaderArgs diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index a32eecf93a..c9eb2e7230 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -119,7 +119,7 @@ if (lastJobId && (job || lastCallbacks?.loadExtraLogs)) { plimit(() => JobService.getCompletedJobLogsTail({ - workspace: $workspaceStore!, + workspace: workspace!, id: lastJobId }) ).then((res) => { @@ -222,7 +222,7 @@ return abstractRun( () => JobService.runScriptByPath({ - workspace: $workspaceStore!, + workspace: workspace!, path: path ?? '', requestBody: args, skipPreprocessor: true @@ -239,7 +239,7 @@ return abstractRun( () => JobService.runScriptByHash({ - workspace: $workspaceStore!, + workspace: workspace!, hash: hash ?? '', requestBody: args, skipPreprocessor: true @@ -256,7 +256,7 @@ return abstractRun( () => JobService.runFlowByPath({ - workspace: $workspaceStore!, + workspace: workspace!, path: path ?? '', requestBody: args, skipPreprocessor: true @@ -274,7 +274,7 @@ return abstractRun( () => JobService.runFlowPreview({ - workspace: $workspaceStore!, + workspace: workspace!, requestBody: { args, value: flow.value, @@ -318,7 +318,7 @@ return abstractRun( () => JobService.runDynamicSelect({ - workspace: $workspaceStore!, + workspace: workspace!, requestBody: { entrypoint_function, args, runnable_ref } }), callbacks @@ -342,7 +342,7 @@ return abstractRun( () => JobService.runScriptPreview({ - workspace: $workspaceStore!, + workspace: workspace!, timeout, requestBody: { path, @@ -371,7 +371,7 @@ currentEventSource = undefined try { await JobService.cancelQueuedJob({ - workspace: $workspaceStore ?? '', + workspace: workspace ?? '', id, requestBody: {} }) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index 785e111527..ed9b9a5694 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -39,9 +39,13 @@ stepsInputArgs, previewArgs, modulesTestStates, - devTempScriptRefs + devTempScriptRefs, + opWorkspace } = getContext('FlowEditorContext') + // Acting workspace when the flow editor runs in an AI session; else the nav workspace. + let opWs = $derived(opWorkspace?.() ?? $workspaceStore) + let jobLoader: JobLoader | undefined = $state(undefined) let jobProgressReset: () => void = () => {} let stepHistoryLoader = getStepHistoryLoaderContext() @@ -102,8 +106,8 @@ ) } else if (val.type == 'script') { const script = val.hash - ? await ScriptService.getScriptByHash({ workspace: $workspaceStore!, hash: val.hash }) - : await getScriptByPath(val.path) + ? await ScriptService.getScriptByHash({ workspace: opWs!, hash: val.hash }) + : await getScriptByPath(val.path, opWs) await jobLoader?.runPreview( val.path, script.content, @@ -202,6 +206,7 @@ void + /** Workspace the resource picker lists from; defaults to the nav workspace. */ + workspace?: string } let { @@ -32,7 +34,8 @@ editor = $bindable(undefined), disabled = false, datatableAsPgResource = false, - onClear = undefined + onClear = undefined, + workspace = undefined }: Props = $props() function isResource() { @@ -55,7 +58,7 @@
{#if format === 'resource-s3_object'} - + {:else if value == undefined || typeof value === 'string'} valueToPath(), (v) => { diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index b8c37011bd..27d6b45e6c 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -78,6 +78,11 @@ disableEditing?: boolean size?: 'sm' | 'md' drawerOffset?: number + /** Workspace the folder list and path-existence checks run against. + * Defaults to the navigation `$workspaceStore`; pass the session's acting + * workspace when the editor operates on a workspace other than the one the + * top nav points at (see the sessions preview / dev-workspace flows). */ + workspaceOverride?: string } let { @@ -94,9 +99,12 @@ hideUser = false, disableEditing = false, size = 'md', - drawerOffset = 0 + drawerOffset = 0, + workspaceOverride = undefined }: Props = $props() + let ws = $derived(workspaceOverride ?? $workspaceStore) + $effect.pre(() => { if (path == undefined) { path = '' @@ -203,7 +211,7 @@ folders = initialFolders.concat( ( await FolderService.listFolderNames({ - workspace: $workspaceStore! + workspace: ws! }) ) .filter((x) => !excludedFolders.includes(x)) @@ -244,74 +252,74 @@ async function pathExists(path: string, kind: PathKind): Promise { if (!path.length) return false if (kind == 'flow') { - return await FlowService.existsFlowByPath({ workspace: $workspaceStore!, path: path }) + return await FlowService.existsFlowByPath({ workspace: ws!, path: path }) } else if (kind == 'script') { return await ScriptService.existsScriptByPath({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'resource') { return await ResourceService.existsResource({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'variable') { return await VariableService.existsVariable({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'schedule') { - return await ScheduleService.existsSchedule({ workspace: $workspaceStore!, path: path }) + return await ScheduleService.existsSchedule({ workspace: ws!, path: path }) } else if (kind == 'app') { - return await AppService.existsApp({ workspace: $workspaceStore!, path: path }) + return await AppService.existsApp({ workspace: ws!, path: path }) } else if (kind == 'http_trigger') { return await HttpTriggerService.existsHttpTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'websocket_trigger') { return await WebsocketTriggerService.existsWebsocketTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'kafka_trigger') { return await KafkaTriggerService.existsKafkaTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'postgres_trigger') { return await PostgresTriggerService.existsPostgresTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'nats_trigger') { return await NatsTriggerService.existsNatsTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind === 'mqtt_trigger') { return await MqttTriggerService.existsMqttTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind == 'sqs_trigger') { return await SqsTriggerService.existsSqsTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind === 'gcp_trigger') { return await GcpTriggerService.existsGcpTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind === 'azure_trigger') { return await AzureTriggerService.existsAzureTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else if (kind === 'email_trigger') { return await EmailTriggerService.existsEmailTrigger({ - workspace: $workspaceStore!, + workspace: ws!, path: path }) } else { @@ -398,7 +406,7 @@ }) }) $effect.pre(() => { - if ($workspaceStore && $userStore) { + if (ws && $userStore) { untrack(() => { loadFolders() initPath() @@ -412,30 +420,30 @@ ) let pathUsageInFlowsPromise = $derived( (kind == 'script' || kind == 'flow') && - $workspaceStore && + ws && initialPath && FlowService.listFlowPathsFromWorkspaceRunnable({ - workspace: $workspaceStore, + workspace: ws, path: initialPath, runnableKind: kind }) ) let pathUsageInAppsPromise = $derived( (kind == 'script' || kind == 'flow') && - $workspaceStore && + ws && initialPath && AppService.listAppPathsFromWorkspaceRunnable({ - workspace: $workspaceStore, + workspace: ws, path: initialPath, runnableKind: kind }) ) let pathUsageInScriptsPromise = $derived( kind == 'script' && - $workspaceStore && + ws && initialPath && ScriptService.listScriptPathsFromWorkspaceRunnable({ - workspace: $workspaceStore, + workspace: ws, path: initialPath }) ) @@ -525,6 +533,7 @@ bind:this={inputP} bind:value={meta.name} prefix={`${meta.ownerKind?.charAt(0) ?? ''}/${meta.owner ?? ''}/`} + workspace={ws} {size} {error} {autofocus} diff --git a/frontend/src/lib/components/PathNameAutocomplete.svelte b/frontend/src/lib/components/PathNameAutocomplete.svelte index 9c31ef980a..10d5c10640 100644 --- a/frontend/src/lib/components/PathNameAutocomplete.svelte +++ b/frontend/src/lib/components/PathNameAutocomplete.svelte @@ -101,6 +101,10 @@ error?: string | boolean textInputClass?: string onkeyup?: (e: KeyboardEvent) => void + /** Workspace whose paths feed the autocomplete. Defaults to the navigation + * `$workspaceStore`; pass the acting workspace when the editor operates on + * a workspace other than the one the top nav points at. */ + workspace?: string } let { @@ -113,9 +117,12 @@ size = 'md', error, textInputClass, - onkeyup + onkeyup, + workspace = undefined }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + let inputEl: TextInput | undefined = $state(undefined) export function focus() { inputEl?.focus() @@ -258,12 +265,12 @@ async function loadPaths(workspace: string) { const paths = await fetchWorkspacePaths(workspace) // Guard against workspace changing during the in-flight fetch. - if ($workspaceStore === workspace) allPaths = paths + if (ws === workspace) allPaths = paths } $effect(() => { - const ws = $workspaceStore - if (ws) void loadPaths(ws) + const w = ws + if (w) void loadPaths(w) }) $effect(() => { @@ -300,7 +307,7 @@ function onInputFocus() { hasFocus = true // Opportunistic refresh if the cache is stale. - if ($workspaceStore) void loadPaths($workspaceStore) + if (ws) void loadPaths(ws) } function onInputBlur() { setTimeout(() => { diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 8804ad8ac5..7acc4e485b 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -402,6 +402,7 @@ {loadingSchema} {resourceToEdit} onLoadResourceType={() => resourceTypeResource.refetch()} + workspace={selected} /> {/key} {/if} diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index ced60bf4dc..ef226501b7 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -42,6 +42,9 @@ loadingSchema: boolean resourceToEdit: Resource | undefined onLoadResourceType?: () => void + /** Workspace the path is validated against and the connection is tested in; + * defaults to the nav workspace. */ + workspace?: string | undefined } let { @@ -62,9 +65,12 @@ resourceSchema, loadingSchema, resourceToEdit, - onLoadResourceType + onLoadResourceType, + workspace = undefined }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + let editDescription = $state(false) let rawCode: string | undefined = $state(undefined) let textFileContent: string = $state('') @@ -131,11 +137,12 @@ {/if}
@@ -218,7 +225,11 @@ {#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'} {:else} - + {/if} {#if resource_type === 'git_repository' && $workspaceStore && ($userStore?.is_admin || $userStore?.is_super_admin)} {/if} {:else if !can_write} diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 55bb25c5ef..8829660ea3 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -18,6 +18,9 @@ selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean regexFilter?: RegExp | undefined + /** Workspace to browse S3 storage in — the acting workspace of the editor that + * opened the picker, else the nav workspace. */ + workspace?: string | undefined onClose?: () => void onSelectAndClose?: (selected: { s3: string; storage: string | undefined }) => void } @@ -30,10 +33,13 @@ selectedFileKey = $bindable(undefined), folderOnly = false, regexFilter = undefined, + workspace = undefined, onClose, onSelectAndClose }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + let drawer: Drawer | undefined = $state() let s3FilePickerInner: S3FilePickerInner | undefined = $state() @@ -55,8 +61,8 @@ > = $state({}) let secondaryStorageNames = resource( - () => $workspaceStore, - () => SettingService.getSecondaryStorageNames({ workspace: $workspaceStore! }), + () => ws, + () => SettingService.getSecondaryStorageNames({ workspace: ws! }), { lazy: true } ) @@ -105,6 +111,7 @@ bind:uploadModalOpen {folderOnly} {regexFilter} + {workspace} /> {#snippet actions()}
diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 333857b7eb..9475c0041a 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -65,6 +65,8 @@ regexFilter?: RegExp | undefined hideS3SpecificDetails?: boolean rootPath?: string + /** Workspace to browse S3 storage in — defaults to the nav workspace. */ + workspace?: string | undefined workspaceSettingsInitialized?: boolean storage?: string | undefined uploadModalOpen?: boolean @@ -103,6 +105,7 @@ regexFilter = undefined, hideS3SpecificDetails = false, rootPath: initialRootPath = '', + workspace = undefined, workspaceSettingsInitialized = $bindable(true), storage = $bindable(undefined), uploadModalOpen = $bindable(false), @@ -117,6 +120,8 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() + let ws = $derived(workspace ?? $workspaceStore) + let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) @@ -183,7 +188,7 @@ async function loadFiles() { fileListLoading = true let availableFiles = await listStoredFilesRequest({ - workspace: $workspaceStore!, + workspace: ws!, maxKeys: maxKeys, // fixed pages of 1000 files for now marker: page == 0 ? undefined : listMarkers[page - 1], prefix: rootPath ?? (filter.trim() != '' ? filter : undefined), @@ -280,7 +285,7 @@ } fileInfoLoading = true let fileMetadataRaw = await loadFileMetadataRequest({ - workspace: $workspaceStore!, + workspace: ws!, fileKey: fileKey, storage: storage }) @@ -300,7 +305,7 @@ async function loadFilePreview(fileKey: string, fileSizeInBytes?: number, fileMimeType?: string) { let filePreviewRaw = await loadFilePreviewRequest({ - workspace: $workspaceStore!, + workspace: ws!, fileKey: fileKey, fileSizeInBytes: fileSizeInBytes, fileMimeType: fileMimeType, @@ -349,7 +354,7 @@ } try { await deleteS3FileRequest({ - workspace: $workspaceStore!, + workspace: ws!, fileKey: fileKey, storage: storage }) @@ -409,7 +414,7 @@ } try { await moveS3FileRequest({ - workspace: $workspaceStore!, + workspace: ws!, srcFileKey: srcFileKey, destFileKey: destFileKey!, storage: storage @@ -457,7 +462,7 @@ fileListLoading = true try { await testConnectionRequest({ - workspace: $workspaceStore!, + workspace: ws!, storage: storage }) workspaceSettingsInitialized = true @@ -716,7 +721,7 @@ {#if filePreview !== undefined && (!hideS3SpecificDetails || !readOnlyMode || allowDelete)}
{#if !hideS3SpecificDetails} - {@const downloadApiPath = `/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${encodeURIComponent(fileMetadata?.fileKey ?? '')}${storage ? `&storage=${storage}` : ''}`} + {@const downloadApiPath = `/w/${ws}/job_helpers/download_s3_file?file_key=${encodeURIComponent(fileMetadata?.fileKey ?? '')}${storage ? `&storage=${storage}` : ''}`} {@const downloadName = fileMetadata?.fileKey.split('/').pop() ?? 'unnamed_download.file'} {#if shouldDownloadViaClient()} diff --git a/frontend/src/lib/components/S3ObjectPicker.svelte b/frontend/src/lib/components/S3ObjectPicker.svelte index 1c461708cd..bb3e19599e 100644 --- a/frontend/src/lib/components/S3ObjectPicker.svelte +++ b/frontend/src/lib/components/S3ObjectPicker.svelte @@ -14,9 +14,15 @@ interface Props { value: any editor?: SimpleEditor | undefined + /** Workspace to browse/upload S3 objects in; defaults to the nav workspace. */ + workspace?: string | undefined } - let { value = $bindable(), editor = $bindable(undefined) }: Props = $props() + let { + value = $bindable(), + editor = $bindable(undefined), + workspace = undefined + }: Props = $props() const dispatch = createEventDispatcher() @@ -48,6 +54,7 @@ editor?.setCode(rawValue) }} readOnlyMode={false} + {workspace} />
@@ -85,6 +92,7 @@ } }} defaultValue={value?.s3} + {workspace} /> {/if} {/each} @@ -182,12 +299,27 @@ variant="subtle" unifiedSize="sm" iconOnly - title="Send" - startIcon={{ icon: ArrowUp }} + title={multiSelect ? 'Add answer' : 'Send'} + startIcon={{ icon: multiSelect ? Plus : ArrowUp }} disabled={!canSubmitCustomAnswer} onClick={submitCustomAnswer} btnClasses="shrink-0" />
+ + {#if multiSelect} + + {/if}
diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 0f30076bda..2438302a00 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -3126,7 +3126,7 @@ describe('global AI tools', () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn(), - requestUserQuestion: vi.fn(async (_toolId, question) => question.choices[1]) + requestUserQuestion: vi.fn(async (_toolId, question) => [question.choices[1]]) } const raw = await callGlobalTool( @@ -3152,7 +3152,45 @@ describe('global AI tools', () => { content: 'User answered question: python3', isLoading: false, result: 'python3', - userQuestion: expect.objectContaining({ selectedChoice: 'python3' }) + userQuestion: expect.objectContaining({ selectedChoices: ['python3'] }) + }) + ) + }) + + it('returns a newline-bulleted list when several answers are selected', async () => { + const callbacks: ToolCallbacks = { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestUserQuestion: vi.fn(async (_toolId, question) => [ + question.choices[0], + question.choices[2] + ]) + } + + const raw = await callGlobalTool( + 'askUserQuestion', + { + question: 'Which languages should be supported?', + choices: ['bun', 'python3', 'go'], + multiSelect: true + }, + callbacks + ) + + // Model-facing return stays newline-bulleted; the header readback is a + // compact comma list. + expect(raw).toBe('- bun\n- go') + expect(callbacks.requestUserQuestion).toHaveBeenCalledWith( + 'test-askUserQuestion', + expect.objectContaining({ multiSelect: true }) + ) + expect(callbacks.setToolStatus).toHaveBeenLastCalledWith( + 'test-askUserQuestion', + expect.objectContaining({ + content: 'User answered question: bun, go', + isLoading: false, + result: '- bun\n- go', + userQuestion: expect.objectContaining({ selectedChoices: ['bun', 'go'] }) }) ) }) @@ -3162,7 +3200,7 @@ describe('global AI tools', () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn(), - requestUserQuestion: vi.fn(async (_toolId, question) => question.choices[9]) + requestUserQuestion: vi.fn(async (_toolId, question) => [question.choices[9]]) } const raw = await callGlobalTool( @@ -3207,7 +3245,7 @@ describe('global AI tools', () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn(), - requestUserQuestion: vi.fn(async () => 'use deno instead') + requestUserQuestion: vi.fn(async () => ['use deno instead']) } const raw = await callGlobalTool( @@ -3225,7 +3263,7 @@ describe('global AI tools', () => { expect.objectContaining({ content: 'User answered question: use deno instead', result: 'use deno instead', - userQuestion: expect.objectContaining({ selectedChoice: 'use deno instead' }) + userQuestion: expect.objectContaining({ selectedChoices: ['use deno instead'] }) }) ) }) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 30fb854ede..f26e3a670d 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -217,7 +217,13 @@ const askUserQuestionSchema = z.object({ .array(z.string().min(1).describe('Proposed answer text shown to the user and returned as-is.')) .min(2) .max(10) - .describe('Two to ten mutually exclusive proposed answer strings.') + .describe('Two to ten proposed answer strings.'), + multiSelect: z + .boolean() + .optional() + .describe( + 'When true, the user may select several answers; use only when the choices can genuinely co-apply (not mutually exclusive). Defaults to single-select.' + ) }) // Matches the per-mode cap enforced by the prompt-settings UI (AIPromptsModal) and @@ -905,7 +911,7 @@ Rules: - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run. - Use open_page to show a workspace page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, or Workspace settings on a specific tab (e.g. "open the failed runs of f/foo/bar", "open the schedule for X", "open the git sync settings"). Only the pages listed for this user in the tool are available; don't offer pages that aren't listed. Don't use it as a substitute for list_runs when you just need the data yourself. -- When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. +- When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ previewTools @@ -2106,7 +2112,8 @@ export const globalTools: Tool<{}>[] = [ const parsed = askUserQuestionSchema.parse(args) const userQuestion = { question: parsed.question, - choices: parsed.choices + choices: parsed.choices, + multiSelect: parsed.multiSelect } toolCallbacks.setToolStatus(toolId, { @@ -2126,8 +2133,8 @@ export const globalTools: Tool<{}>[] = [ return JSON.stringify({ success: false, error: message }) } - const selectedChoice = await toolCallbacks.requestUserQuestion(toolId, userQuestion) - if (!selectedChoice) { + const selected = await toolCallbacks.requestUserQuestion(toolId, userQuestion) + if (!selected?.length) { const message = 'Question cancelled by user' toolCallbacks.setToolStatus(toolId, { content: message, @@ -2138,16 +2145,26 @@ export const globalTools: Tool<{}>[] = [ return JSON.stringify({ success: false, error: message }) } + // Model-facing answer: bare string for one pick (preserves the single-select + // contract, even when multiSelect was set), newline-bulleted list for several. + // Comma-joining is avoided here so a choice that itself contains a comma + // ("Yes, immediately") stays unambiguous to the model reading it back. + const answerText = + selected.length === 1 ? selected[0] : selected.map((c) => `- ${c}`).join('\n') + // The collapsed tool-header is a human glance, not model input, so the picks + // read as a compact comma list there instead of a stacked bullet list. + const answerSummary = selected.join(', ') + toolCallbacks.setToolStatus(toolId, { - content: `User answered question: ${selectedChoice}`, + content: `User answered question: ${answerSummary}`, userQuestion: { ...userQuestion, - selectedChoice + selectedChoices: selected }, - result: selectedChoice, + result: answerText, isLoading: false }) - return selectedChoice + return answerText } }, { diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 1bab69152a..1b77e060c4 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -651,7 +651,18 @@ describe('isActiveUserQuestion', () => { expect(isActiveUserQuestion(toolMessage())).toBe(true) }) - it('is false once a choice has been selected', async () => { + it('is false once choices have been selected', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect( + isActiveUserQuestion( + toolMessage({ + userQuestion: { question: 'Pick one', choices: ['a', 'b'], selectedChoices: ['a'] } + }) + ) + ).toBe(false) + }) + + it('is false once a legacy scalar selectedChoice is present', async () => { const { isActiveUserQuestion } = await import('./shared') expect( isActiveUserQuestion( @@ -662,6 +673,17 @@ describe('isActiveUserQuestion', () => { ).toBe(false) }) + it('stays active when selectedChoices is present but empty', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect( + isActiveUserQuestion( + toolMessage({ + userQuestion: { question: 'Pick one', choices: ['a', 'b'], selectedChoices: [] } + }) + ) + ).toBe(true) + }) + it('is false when the question was canceled', async () => { const { isActiveUserQuestion } = await import('./shared') expect( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index b1a9f667bf..dd69139e60 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -509,10 +509,19 @@ export type ToolDisplayAction = CreatedResourceAction | NavigateAction export type UserQuestionDisplay = { question: string choices: string[] - selectedChoice?: string + multiSelect?: boolean + selectedChoices?: string[] // canonical answer (new code writes only this) + selectedChoice?: string // legacy/read-only: pre-multiselect persisted history canceled?: boolean } +// The single place that understands the legacy answer shape: new code writes +// selectedChoices, but history persisted before multi-select only has the +// scalar selectedChoice. Read answers through this so both shapes resolve. +export function answeredChoices(q: UserQuestionDisplay): string[] | undefined { + return q.selectedChoices ?? (q.selectedChoice ? [q.selectedChoice] : undefined) +} + export type ToolDisplayMessage = { role: 'tool' tool_call_id: string @@ -573,7 +582,7 @@ export function isActiveUserQuestion(message: DisplayMessage | undefined): boole message.userQuestion && message.isLoading && !message.error && - !message.userQuestion.selectedChoice && + !answeredChoices(message.userQuestion)?.length && !message.userQuestion.canceled ) } @@ -878,7 +887,7 @@ export interface ToolCallbacks { requestUserQuestion?: ( toolId: string, question: UserQuestionDisplay - ) => Promise + ) => Promise /** Records a workspace item the tool call created/edited/deleted, by its * canonical (itemKind, storagePath). Session chats wire this to accumulate the * chat's modified-items mask; the global side-panel chat omits it (no-op). */ From c139eed631548113b843b466f5505bf6a01f17d3 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 9 Jul 2026 01:54:58 +0200 Subject: [PATCH 273/273] fix: session preview tab labels, splitter hover, and diff-drawer sizing (#10008) * fix: show pending friendly path in new raw app session tab Co-Authored-By: Claude Opus 4.8 (1M context) * fix: reveal a subtle rounded grabber on the sessions chat/preview splitter on hover Co-Authored-By: Claude Opus 4.8 (1M context) * fix: cap session diff blocks so each item's card fits the drawer viewport Co-Authored-By: Claude Opus 4.8 (1M context) * fix: floor flow diff cap at its min height and tidy diff/splitter comments Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../components/WorkspaceItemDiffViewer.svelte | 43 +++++++++++++---- .../components/raw_apps/RawAppFileDiff.svelte | 20 ++++++-- .../sessions/WorkspaceDiffDrawer.svelte | 14 ++++++ .../components/sessions/previewRouter.test.ts | 34 +++++++++++++- .../lib/components/sessions/previewRouter.ts | 23 +++++++++ .../(root)/(logged)/sessions/+page.svelte | 47 ++++++++++++------- 6 files changed, 153 insertions(+), 28 deletions(-) diff --git a/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte b/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte index e792a81e8a..77aae182fa 100644 --- a/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte +++ b/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte @@ -44,6 +44,9 @@ doesn't reflow the parent. disableAutoInline?: boolean /** For `raw_app_file`: the synthesized per-file diff item to render. */ rawFile?: RawAppFileItem + /** Cap (px) on the rendered diff height; the drawer owns the rationale. + * Ignored while ≤ 0 (unmeasured). */ + maxHeight?: number } let { @@ -52,9 +55,30 @@ doesn't reflow the parent. currentRaw, inlineDiff = false, disableAutoInline = false, - rawFile + rawFile, + maxHeight }: Props = $props() + // Applied alongside each block's content-sized `height` so the block never + // exceeds the drawer's visible height — Monaco then scrolls internally. + function capStyle(reserved: number): string { + if (!maxHeight || maxHeight <= 0) return '' + return `max-height: ${Math.max(0, maxHeight - reserved)}px;` + } + const maxHeightStyle = $derived(capStyle(0)) + // FlowDiffViewer enforces its own `min-h-[500px]`; capping the flow below that + // makes its pane spill out of the card instead of scrolling, so never cap the + // flow diff below that minimum (a very short drawer just scrolls to it). + const FLOW_MIN_HEIGHT = 500 + const flowMaxHeightStyle = $derived( + maxHeight && maxHeight > 0 ? `max-height: ${Math.max(FLOW_MIN_HEIGHT, maxHeight)}px;` : '' + ) + // The Content|Metadata tab bar sits above the script editor and eats into the + // viewer's height; measure it so the editor's cap leaves room for it and the + // whole viewer still fits `maxHeight`. + let tabsChromeH = $state(0) + const contentMaxHeightStyle = $derived(capStyle(tabsChromeH)) + type Prepared = { lang?: string; content?: string; metadata: string } function prepareValue(raw: unknown): Prepared { @@ -111,7 +135,7 @@ doesn't reflow the parent. {#if kind === 'flow'} -
+
{:else if hasContent}
- - - - -
+
+ + + + +
+
{#if contentTab === 'content'} {#await import('$lib/components/DiffEditor.svelte')}
@@ -178,7 +205,7 @@ doesn't reflow the parent. {#await import('$lib/components/DiffEditor.svelte')}
{:then Module} -
+
0). + let chromeH = $state(0) + const maxHeightStyle = $derived( + maxHeight && maxHeight > 0 ? `max-height: ${Math.max(0, maxHeight - chromeH)}px;` : '' + ) // The full-YAML view is opt-in, so never guard it; only guard the default // per-file view. const guarded = $derived(!showFullYaml && !forceLoad && lineCount > lineBudget) @@ -65,7 +76,10 @@ the escape hatch, shown only on demand.
{#if canExpandYaml} -
+
@@ -83,7 +97,7 @@ the escape hatch, shown only on demand. {#await import('$lib/components/DiffEditor.svelte')}
{:then Module} -
+
0 ? mainHeight - cardHeaderH - CARD_GAP : 0) // The trailing spacer only needs to cover the gap the last item can't fill on // its own: viewport − lastItemHeight (0 when the item already fills the view). // Reserving the full viewport would let you scroll a whole item-height into @@ -830,6 +840,7 @@ rawFile={sub as RawAppFileItem} {inlineDiff} disableAutoInline + {maxHeight} /> {:else} {@const runnable = sub as RawAppRunnableItem} @@ -839,6 +850,7 @@ currentRaw={runnable.currentRaw} {inlineDiff} disableAutoInline + {maxHeight} /> {/if}
@@ -851,6 +863,7 @@ currentRaw={loaded.after} {inlineDiff} disableAutoInline + {maxHeight} /> {/if} {/snippet} @@ -983,6 +996,7 @@ : 'ring-0 ring-transparent'}" >
diff --git a/frontend/src/lib/components/sessions/previewRouter.test.ts b/frontend/src/lib/components/sessions/previewRouter.test.ts index 0c8ee64d89..747ecda6b7 100644 --- a/frontend/src/lib/components/sessions/previewRouter.test.ts +++ b/frontend/src/lib/components/sessions/previewRouter.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest' -import { parsePreviewItemRoute, resolvePreviewTab } from './previewRouter' +import { parsePreviewItemRoute, previewTabLabel, resolvePreviewTab } from './previewRouter' describe('parsePreviewItemRoute', () => { it('maps edit/get routes to item kinds', () => { @@ -32,6 +32,38 @@ describe('parsePreviewItemRoute', () => { }) }) +describe('previewTabLabel', () => { + it('labels a new raw app by its pending friendly path, not the draft uuid', () => { + const rawApp = { path: 'u/admin/draft_abc123', draft_path: 'u/admin/my_pretty_app' } + expect(previewTabLabel('/apps_raw/edit/u/admin/draft_abc123', rawApp)).toBe('my_pretty_app') + }) + + it('falls back to the uuid leaf when no friendly draft_path is pending', () => { + const rawApp = { path: 'u/admin/draft_abc123' } + expect(previewTabLabel('/apps_raw/edit/u/admin/draft_abc123', rawApp)).toBe('draft_abc123') + }) + + it('keeps the real leaf for a raw app already at a named (non-draft) path', () => { + const rawApp = { path: 'u/admin/my_app', draft_path: 'u/admin/renamed' } + expect(previewTabLabel('/apps_raw/edit/u/admin/my_app', rawApp)).toBe('my_app') + }) + + it('ignores a draft_path that belongs to a different raw app than the tab shows', () => { + const rawApp = { path: 'u/admin/draft_other', draft_path: 'u/admin/friendly' } + expect(previewTabLabel('/apps_raw/edit/u/admin/draft_abc123', rawApp)).toBe('draft_abc123') + }) + + it('does not touch non-raw-app tabs', () => { + const rawApp = { path: 'u/admin/draft_abc123', draft_path: 'u/admin/friendly' } + expect(previewTabLabel('/scripts/edit/f/foo/bar', rawApp)).toBe('bar') + expect(previewTabLabel('/runs', rawApp)).toBe('Runs') + }) + + it('falls back to the plain location label when no raw app is loaded', () => { + expect(previewTabLabel('/apps_raw/edit/u/admin/draft_abc123', undefined)).toBe('draft_abc123') + }) +}) + describe('resolvePreviewTab', () => { it('routes a static page to the iframe fallback', () => { expect(resolvePreviewTab('/runs')).toEqual({ kind: 'iframe' }) diff --git a/frontend/src/lib/components/sessions/previewRouter.ts b/frontend/src/lib/components/sessions/previewRouter.ts index 8fcd038f79..9bc46fce0f 100644 --- a/frontend/src/lib/components/sessions/previewRouter.ts +++ b/frontend/src/lib/components/sessions/previewRouter.ts @@ -110,6 +110,29 @@ export function previewLocationLabel(url: string): string { return stripBase(url) } +/** Tab label for a preview location, preferring the friendly path a raw-app + * editor was renamed to while still parked at its throwaway `…/draft_` + * storage path. `rawAppDraft` is the session's live raw app (its storage `path` + * plus the pending `draft_path` the user typed in the editor). When the tab + * shows that app at a draft placeholder path, it's labelled by the friendly + * leaf rather than the uuid. Display-only — the tab's URL keeps the storage + * path. Falls back to `previewLocationLabel` for every other tab. */ +export function previewTabLabel( + url: string, + rawAppDraft?: { path: string; draft_path?: string } +): string { + const route = parsePreviewItemRoute(url) + if ( + route?.raw_app && + rawAppDraft?.draft_path && + rawAppDraft.path === route.itemPath && + route.itemPath.split('/').pop()?.startsWith('draft_') + ) { + return rawAppDraft.draft_path.split('/').pop() ?? rawAppDraft.draft_path + } + return previewLocationLabel(url) +} + export type PreviewItemRoute = { kind: WorkspaceItemKind; raw_app: boolean; itemPath: string } // Parse a preview URL/pathname into the workspace item it edits, or null for a diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 6571e6a1b3..70e82b2792 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -38,7 +38,8 @@ import { getOrCreateRuntime, getRuntime, - listRuntimes + listRuntimes, + type SessionRuntime } from '$lib/components/sessions/sessionRuntime.svelte' import { markSessionSeen } from '$lib/components/sessions/sessionUnread.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' @@ -48,7 +49,7 @@ matchPreviewPage, pageKey, parsePreviewItemRoute, - previewLocationLabel, + previewTabLabel, type PreviewTarget } from '$lib/components/sessions/previewRouter' import { toolReloadEffect, tabsToReload } from '$lib/components/sessions/previewReload' @@ -243,7 +244,7 @@ // Adapt the session tab model to DraggableTabs items (labels derived from the // observed location; every tab closable, none pinned). const previewTabItems = $derived( - (owner?.tabs ?? []).map((t) => ({ id: t.id, label: tabLabel(t.loc) })) + (owner?.tabs ?? []).map((t) => ({ id: t.id, label: tabLabelFor(activeRuntime, t.loc) })) ) let newTabOpen = $state(false) // Separate open flag for the empty-state launcher: it can be mounted at the @@ -416,10 +417,13 @@ owner?.navigate(target) } - // Short tab label: a known page's name, else a run detail, else the item's leaf - // name, else path. - function tabLabel(url: string): string { - return previewLocationLabel(url) + // Short tab label. For a raw-app tab, feed its own per-path cell so the tab is + // labelled by that app's pending draft path (a rename parked at `draft_`), + // scoped to the tab's own runtime rather than another session's. + function tabLabelFor(rt: SessionRuntime | undefined, url: string): string { + const route = parsePreviewItemRoute(url) + const rawAppDraft = rt && route?.raw_app ? rt.rawAppCell(route.itemPath).store.val : undefined + return previewTabLabel(url, rawAppDraft) } // A link click inside a live editor (e.g. a subflow reference) re-points the @@ -531,7 +535,7 @@
{#if !fullscreen}