Compare commits

..
Author SHA1 Message Date
Diego ImbertandClaude Opus 4.8 2da3ae8cfc feat(frontend): hide add-secondary-storage button until primary storage is set
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-20 12:34:46 +02:00
Diego Imbert e7234ddc7e Merge branch 'main' into improve-storage-setup-ux 2026-06-20 12:25:36 +02:00
Diego ImbertandClaude Opus 4.8 6d1f394b5f feat(frontend): red error border on empty primary object storage picker
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-20 12:21:34 +02:00
222 changed files with 1880 additions and 15862 deletions
-132
View File
@@ -1,132 +0,0 @@
name: AI Agent Integration Tests
# Exercises the AI agent flow path (preview_flow with `aiagent` modules) against
# real LLM providers. Runs only when AI-agent backend code or the tests change,
# because each run makes real (paid) LLM calls. To avoid spending on every commit,
# the PR side triggers only when a PR is marked ready for review (out of draft) —
# not on `synchronize` — plus push to main and manual dispatch.
on:
workflow_dispatch:
push:
branches: [main]
paths:
- "integration_tests/ai_agent_tests/**"
- "backend/windmill-ai/**"
- "backend/windmill-api/src/ai.rs"
- "backend/windmill-worker/src/ai_executor.rs"
- "backend/windmill-worker/src/ai/**"
- "backend/windmill-worker/src/memory_common.rs"
- "backend/windmill-common/src/flow_conversations.rs"
- ".github/workflows/ai-agent-tests.yml"
pull_request:
types: [opened, reopened, ready_for_review]
paths:
- "integration_tests/ai_agent_tests/**"
- "backend/windmill-ai/**"
- "backend/windmill-api/src/ai.rs"
- "backend/windmill-worker/src/ai_executor.rs"
- "backend/windmill-worker/src/ai/**"
- "backend/windmill-worker/src/memory_common.rs"
- "backend/windmill-common/src/flow_conversations.rs"
- ".github/workflows/ai-agent-tests.yml"
concurrency:
group: ai-agent-tests-${{ github.ref }}
cancel-in-progress: true
jobs:
ai_agent_e2e:
# Skip draft PRs; the `opened`/`reopened` types would otherwise fire while
# still a draft. `ready_for_review` always arrives non-draft.
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
runs-on: ubicloud-standard-16
services:
postgres:
image: postgres:16
ports:
- 5432:5432
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.93.0
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.10
- uses: actions/setup-node@v4
with:
node-version: "20"
- uses: actions/setup-python@v5
with:
python-version: "3.11"
# CE build (no enterprise/license needed for AI agents). `quickjs` powers
# flow input-transform JS eval; `mcp` is required by the deepwiki MCP tool
# test. Bun tool scripts run via the always-on worker (BUN_PATH).
- name: Build Windmill
working-directory: ./backend
env:
SQLX_OFFLINE: true
CARGO_BUILD_JOBS: 12
RUSTFLAGS: ""
run: cargo build --features quickjs,mcp
- name: Start Windmill
working-directory: ./backend
env:
DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
BUN_PATH: bun
NODE_BIN_PATH: node
RUST_LOG: info
run: |
mkdir -p ../integration_tests/logs
./target/debug/windmill > ../integration_tests/logs/windmill.log 2>&1 &
echo "Waiting for Windmill to be ready..."
for i in $(seq 1 60); do
if curl -sf http://localhost:8000/api/version > /dev/null 2>&1; then
echo "Windmill is ready"
break
fi
sleep 2
done
curl -sf http://localhost:8000/api/version > /dev/null || { echo "Windmill failed to start"; tail -50 ../integration_tests/logs/windmill.log; exit 1; }
- name: Run AI agent integration tests
timeout-minutes: 20
working-directory: ./integration_tests/ai_agent_tests
env:
WINDMILL_URL: http://localhost:8000
# Only the providers we have org secrets for. Other providers
# (Azure, Bedrock, OpenRouter) are skipped by conftest when their
# keys are absent — see skip_provider_without_credentials.
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
run: |
python -m venv .venv
.venv/bin/pip install -r requirements.txt
# The S3/vision-attachment tests need MinIO large-file storage and
# image-capable provider setup; out of scope for this cost-controlled
# smoke. Add MinIO secrets + a storage service to enable them.
.venv/bin/python -m pytest -v \
--ignore=test_user_attachments.py \
--ignore=test_user_images.py \
--ignore=test_image_output.py
- name: Archive Windmill logs
uses: actions/upload-artifact@v4
if: always()
with:
name: ai-agent-tests-windmill-logs
path: integration_tests/logs
-166
View File
@@ -1,166 +0,0 @@
name: AI Evals (global mode)
# Smoke-tests the production global AI chat proxy/frontend execution path via
# the ai_evals harness, one case across one cheap model per provider. Runs only
# when the eval harness or the global chat code change, since each run makes real
# (paid) LLM calls. The backend is built from source purely as the AI proxy the
# harness routes model calls through; the global tools/drafts run in-process in
# the Vitest bridge against production frontend code. To avoid spending on every
# commit, the PR side triggers only when a PR is marked ready for review (out of
# draft) — not on `synchronize` — plus push to main and manual dispatch.
on:
workflow_dispatch:
push:
branches: [main]
paths:
- "ai_evals/**"
- "backend/windmill-api/src/ai.rs"
- "backend/windmill-ai/**"
- "frontend/src/lib/components/copilot/**"
# The eval harness runs production frontend code in-process; these are the
# AI/draft-specific deps outside copilot/ that the global smoke exercises.
- "frontend/src/lib/userDraft.svelte.ts"
- "frontend/src/lib/userDraftDbSyncer.svelte.ts"
- "frontend/src/lib/infer.ts"
- ".github/workflows/ai-evals-test.yml"
pull_request:
types: [opened, reopened, ready_for_review]
paths:
- "ai_evals/**"
- "backend/windmill-api/src/ai.rs"
- "backend/windmill-ai/**"
- "frontend/src/lib/components/copilot/**"
# The eval harness runs production frontend code in-process; these are the
# AI/draft-specific deps outside copilot/ that the global smoke exercises.
- "frontend/src/lib/userDraft.svelte.ts"
- "frontend/src/lib/userDraftDbSyncer.svelte.ts"
- "frontend/src/lib/infer.ts"
- ".github/workflows/ai-evals-test.yml"
concurrency:
group: ai-evals-test-${{ github.ref }}
cancel-in-progress: true
jobs:
ai_evals_global:
# Provider secrets are unavailable to forked and Dependabot PRs.
if: >-
github.event_name != 'pull_request' ||
(
github.event.pull_request.draft == false &&
github.event.pull_request.head.repo.full_name == github.repository &&
github.event.pull_request.user.login != 'dependabot[bot]'
)
runs-on: ubicloud-standard-16
services:
postgres:
image: postgres:16
ports:
- 5432:5432
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.93.0
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.10
- uses: actions/setup-node@v4
with:
# Node 22.19+ is required by the frontend's undici 8.x, which the
# Vitest bridge loads; Node 20 fails with markAsUncloneable.
node-version: "22"
# CE build used only as the AI proxy (login, workspace, provider resource,
# /ai/proxy). No worker execution or MCP needed — global tools/drafts run
# in the Vitest bridge. quickjs matches the standard CE feature set.
- name: Build Windmill (AI proxy)
working-directory: ./backend
env:
SQLX_OFFLINE: true
CARGO_BUILD_JOBS: 12
RUSTFLAGS: ""
run: cargo build --features quickjs
- name: Start Windmill
working-directory: ./backend
env:
DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
RUST_LOG: info
run: |
mkdir -p ../ai_evals/logs
./target/debug/windmill > ../ai_evals/logs/windmill.log 2>&1 &
echo "Waiting for Windmill to be ready..."
for i in $(seq 1 60); do
if curl -sf http://localhost:8000/api/version > /dev/null 2>&1; then
echo "Windmill is ready"
break
fi
sleep 2
done
curl -sf http://localhost:8000/api/version > /dev/null || { echo "Windmill failed to start"; tail -50 ../ai_evals/logs/windmill.log; exit 1; }
- name: Install frontend deps + generate client
working-directory: ./frontend
run: |
npm ci
npm run generate-backend-client
- name: Run global AI evals
timeout-minutes: 20
working-directory: ./ai_evals
env:
WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000
WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests
# Anthropic backs the haiku model. Google AI uses GEMINI_API_KEY.
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
run: |
bun install
mkdir -p results
# One cheap model per provider (anthropic/openai/googleai/deepseek).
fail=0
for m in haiku 4o gemini-3-flash-preview deepseek-v4-flash; do
echo "::group::global-test1-script-create ($m)"
if ! bun run cli -- run global global-test1-script-create \
--model "$m" --execution-only --output "$PWD/results/ci-$m.json"; then
echo "$m: harness/proxy errored"
fail=1
echo "::endgroup::"
continue
fi
# The CLI exits 0 when the harness records failed attempts, so gate
# on execution-only pass counts while ignoring model output quality.
if jq -e \
'.attemptCount > 0 and .passedAttempts == .attemptCount' \
"results/ci-$m.json" > /dev/null; then
echo "$m: OK — proxy/frontend execution completed"
else
echo "$m: FAILED proxy/frontend execution"
jq -c '.cases[0].attempts[0].checks' "results/ci-$m.json" || true
fail=1
fi
echo "::endgroup::"
done
[ "$fail" = 0 ] || { echo "ai_evals global smoke failed"; exit 1; }
- name: Archive logs and results
uses: actions/upload-artifact@v4
if: always()
with:
name: ai-evals-global-logs
path: |
ai_evals/logs
ai_evals/results
-69
View File
@@ -1,74 +1,5 @@
# Changelog
## [1.737.0](https://github.com/windmill-labs/windmill/compare/v1.736.0...v1.737.0) (2026-06-23)
### Features
* **apps:** opt-in sandbox isolation for published & raw apps (alpha) ([#9420](https://github.com/windmill-labs/windmill/issues/9420)) ([2879cbb](https://github.com/windmill-labs/windmill/commit/2879cbb65a4122c86b4a472206d74d9009b07904))
### Bug Fixes
* allow SQL args in managed // materialize scripts ([#9733](https://github.com/windmill-labs/windmill/issues/9733)) ([fa35968](https://github.com/windmill-labs/windmill/commit/fa3596885bf2d7ee8859f295e820ee362c756911))
* bound orphan-cleanup drain rate with capped multi-batch loop ([#9730](https://github.com/windmill-labs/windmill/issues/9730)) ([31d9215](https://github.com/windmill-labs/windmill/commit/31d9215e5a61f19662cc87be8147007e1d47ebb6))
* **ext-jwt:** reject external JWT auth for non-existent workspaces ([#9723](https://github.com/windmill-labs/windmill/issues/9723)) ([c644311](https://github.com/windmill-labs/windmill/commit/c644311eca4bcaf4b68058cf1d5d79d4078aee1a))
* optimize cleanup_job_perms_orphaned and job_result_stream cleanup queries ([#9727](https://github.com/windmill-labs/windmill/issues/9727)) ([6d94865](https://github.com/windmill-labs/windmill/commit/6d9486510933af9109f52011d93b13847dbdbb39))
* prevent silent audit-partition outage via monitor watchdog + alert ([#9729](https://github.com/windmill-labs/windmill/issues/9729)) ([8dea383](https://github.com/windmill-labs/windmill/commit/8dea38383f884f59b2956c39f1424005a21265bd))
### Performance Improvements
* **monitor:** hash active-root exclusion in retention delete (WIN-2088) ([#9732](https://github.com/windmill-labs/windmill/issues/9732)) ([75bafab](https://github.com/windmill-labs/windmill/commit/75bafabeeec76cf6da33eef41f588e37071df011))
## [1.736.0](https://github.com/windmill-labs/windmill/compare/v1.735.0...v1.736.0) (2026-06-23)
### Features
* **ai-chat:** workspace AI chat skills (SKILL.md upload + read_skill tool) ([#9648](https://github.com/windmill-labs/windmill/issues/9648)) ([6f4017d](https://github.com/windmill-labs/windmill/commit/6f4017d694494a158ebd0579c93336c378cba0fd))
### Bug Fixes
* **drafts:** stop mis-filing workspace-blind legacy drafts on migration ([#9725](https://github.com/windmill-labs/windmill/issues/9725)) ([3bf5b72](https://github.com/windmill-labs/windmill/commit/3bf5b72afab3241ea41a261a40c2434764bdaf72))
* **frontend:** destroy old WebsocketProvider on workspace switch in MultiplayerMenu ([#9719](https://github.com/windmill-labs/windmill/issues/9719)) ([6e96f90](https://github.com/windmill-labs/windmill/commit/6e96f90065dfe2f6ccc5eb4f85f4facd3515c70c))
* **frontend:** ensure type:object in test_run_flow tool schema for Anthropic ([#9721](https://github.com/windmill-labs/windmill/issues/9721)) ([d5cb944](https://github.com/windmill-labs/windmill/commit/d5cb944cf92f074b2ee42c876595eacdfa2f4d76))
* **health:** detect read-only replica via pg_is_in_recovery() ([#9722](https://github.com/windmill-labs/windmill/issues/9722)) ([e16061d](https://github.com/windmill-labs/windmill/commit/e16061df06babeae935a9396de5bdcd46e8119a9))
* re-enforce scoped API token boundaries across handlers ([#9712](https://github.com/windmill-labs/windmill/issues/9712)) ([e19594d](https://github.com/windmill-labs/windmill/commit/e19594df2ad015a0336ade95e04562f5562ec3f6))
## [1.735.0](https://github.com/windmill-labs/windmill/compare/v1.734.0...v1.735.0) (2026-06-22)
### Features
* clarify session draft bar tracks all workspace draft changes ([#9714](https://github.com/windmill-labs/windmill/issues/9714)) ([ed016a5](https://github.com/windmill-labs/windmill/commit/ed016a5edb4527877bf7e6bf92feafc2710669c2))
* **copilot:** improve global-mode path selection + add path-selection evals ([#9698](https://github.com/windmill-labs/windmill/issues/9698)) ([74a2329](https://github.com/windmill-labs/windmill/commit/74a2329d2e8395141807c43acef07ef132490039))
* link files & folders to the global AI chat ([#9520](https://github.com/windmill-labs/windmill/issues/9520)) ([84cc043](https://github.com/windmill-labs/windmill/commit/84cc043406d63a1e1472165cf24ce8c09905fc5f))
* scope default instance db name to workspace (dt_/dl_) ([#9699](https://github.com/windmill-labs/windmill/issues/9699)) ([4a8a724](https://github.com/windmill-labs/windmill/commit/4a8a724895dcecb835e1eb1e4fd7d1bbc8b3e0fb))
### Bug Fixes
* enforce job_dir containment when writing module files ([#9703](https://github.com/windmill-labs/windmill/issues/9703)) ([e403f92](https://github.com/windmill-labs/windmill/commit/e403f92d7e84cebc78709dce1a0928048ba2506d))
* **frontend:** deploy full script/flow draft from AI chat via shared module ([#9642](https://github.com/windmill-labs/windmill/issues/9642)) ([23bf6bf](https://github.com/windmill-labs/windmill/commit/23bf6bf3da01d552d2dfab6dbcfd30f758ed34d2))
* **frontend:** strip raw-app post-deploy diff noise (raw_app/lock/data) ([#9706](https://github.com/windmill-labs/windmill/issues/9706)) ([e20a277](https://github.com/windmill-labs/windmill/commit/e20a27745a08d552e6d2c5a8bbaf08ccfe89c68f))
* ignore NotFound errors when deleting log files from object store ([#9707](https://github.com/windmill-labs/windmill/issues/9707)) ([8a0b0ab](https://github.com/windmill-labs/windmill/commit/8a0b0abead71320c4f69eb3007739a19f76d4126))
* **oauth:** restore bring-your-own CC token URL override ([#9711](https://github.com/windmill-labs/windmill/issues/9711)) ([ef4962e](https://github.com/windmill-labs/windmill/commit/ef4962e52aba0bc79bf72523de9101853c660654))
* sanitize git credentials from ansible executor errors and logs ([#9697](https://github.com/windmill-labs/windmill/issues/9697)) ([ace7b68](https://github.com/windmill-labs/windmill/commit/ace7b68a28b00d715298ffcb6ae907c1974a74b8))
## [1.734.0](https://github.com/windmill-labs/windmill/compare/v1.733.1...v1.734.0) (2026-06-20)
### Features
* ducklake materialization for data pipelines ([#9689](https://github.com/windmill-labs/windmill/issues/9689)) ([3ebf243](https://github.com/windmill-labs/windmill/commit/3ebf24359d66048d6361ce65cd879cdc04b737ed))
### Bug Fixes
* **frontend:** clear branch step state when switching outer loop iterations ([#9650](https://github.com/windmill-labs/windmill/issues/9650)) ([09a8004](https://github.com/windmill-labs/windmill/commit/09a80040ca268a4379d5302e3401435ba93247e0))
## [1.733.1](https://github.com/windmill-labs/windmill/compare/v1.733.0...v1.733.1) (2026-06-19)
+1 -3
View File
@@ -75,8 +75,6 @@ Public CLI surface:
- `--model <alias>`: choose the model under test
- `--models <a,b,c>`: run the same cases sequentially against several model aliases
- `--verbose`: stream assistant output for frontend runs
- `--skip-judge`: skip LLM judge scoring for the run
- `--execution-only`: only require the model/proxy/frontend loop to complete; skip validators, tool expectations, backend artifact validation, and judge scoring
- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
@@ -101,7 +99,7 @@ Notes:
- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
- the judge model is separate and currently defaults to `claude-sonnet-4-6`; use `--skip-judge` for deterministic-only runs
- the judge model is separate and currently defaults to `claude-sonnet-4-6`
## Case Format
@@ -25,12 +25,6 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
);
const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1";
const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1";
const executionOnly =
process.env.WMILL_FRONTEND_AI_EVAL_EXECUTION_ONLY === "1";
const judgeModel =
process.env.WMILL_FRONTEND_AI_EVAL_SKIP_JUDGE === "1" || executionOnly
? null
: DEFAULT_JUDGE_MODEL;
const model = resolveEvalModel(
mode,
process.env.WMILL_FRONTEND_AI_EVAL_MODEL,
@@ -54,8 +48,7 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
cases: selectedCases,
runs,
runModel,
judgeModel,
executionOnly,
judgeModel: DEFAULT_JUDGE_MODEL,
concurrency: verbose ? 1 : undefined,
verbose,
onProgress: emitProgress
@@ -67,7 +60,7 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
mode,
runs,
runModel,
judgeModel,
judgeModel: DEFAULT_JUDGE_MODEL,
caseResults,
});
}
@@ -50,20 +50,6 @@ export interface GlobalLiveEditorDraftFixture {
value?: unknown;
}
// Identity the global system prompt builds paths from. Production reads
// `userStore` (whoami) to fill `u/{username}/...`; the eval harness never logs
// in, so without this the prompt sees an empty username (`u//...`) and no
// path-selection case is meaningful. Seeded per-case via the initial fixture and
// passed straight to `prepareGlobalSystemMessage` (no global-store mutation).
export interface GlobalUserFixture {
username: string;
is_admin?: boolean;
/** Folders the user can write to (the writable set whoami returns). */
folders?: string[];
/** Folders the user can read; read-only folders = folders_read \ folders. */
folders_read?: string[];
}
export interface GlobalEvalResult {
success: boolean;
state: GlobalDraftState;
@@ -79,7 +65,6 @@ export interface GlobalEvalResult {
export interface GlobalEvalOptions {
workspaceFixtures?: BenchmarkWorkspaceRunnables;
liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
user?: GlobalUserFixture;
model?: string;
maxIterations?: number;
provider?: AIProvider;
@@ -105,11 +90,9 @@ export async function runGlobalEval(
const model = options.model ?? "claude-haiku-4-5-20251001";
const injectActiveEditorContext =
process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1";
// Pass the seeded identity straight to the prompt builder rather than mutating
// the process-global `userStore`, so concurrent cases never race on it.
const rawResult = await runEval({
userPrompt,
systemMessage: prepareGlobalSystemMessage(undefined, { user: options.user }),
systemMessage: prepareGlobalSystemMessage(),
userMessage: prepareGlobalUserMessage(
userPrompt,
[],
-5
View File
@@ -24,8 +24,6 @@ export async function runFrontendBenchmarkAdapter(input: {
runs: number;
model?: string;
verbose?: boolean;
skipJudge?: boolean;
executionOnly?: boolean;
backendValidation?: string;
}): Promise<BenchmarkRunResult> {
const tempDir = await mkdtemp(
@@ -42,9 +40,6 @@ export async function runFrontendBenchmarkAdapter(input: {
WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "",
WMILL_FRONTEND_AI_EVAL_PROGRESS: "1",
WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? "1" : "0",
WMILL_FRONTEND_AI_EVAL_SKIP_JUDGE:
input.skipJudge || input.executionOnly ? "1" : "0",
WMILL_FRONTEND_AI_EVAL_EXECUTION_ONLY: input.executionOnly ? "1" : "0",
WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "",
};
+1 -109
View File
@@ -4,7 +4,7 @@
It should take a string `name` input and return `Hello, ${name}!`.
Leave it as an AI draft only; do not deploy or save it.
runtime:
maxTurns: 10
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
@@ -1113,111 +1113,3 @@
- renames the formatCurrency definition, imports, and all call sites to formatMoney
- leaves the unrelated formatCurrencyPrecise helper unchanged
- leaves the result as an AI draft only
# --- Path selection (u/<user> vs f/<folder>) ---
# These cases assert how the assistant picks a workspace path when the user gives
# none: a bare name defaults to the personal scope `u/<user>/`, an existing folder
# whose purpose matches is used, a non-admin targets a writable folder and never a
# read-only one, and shared intent with no matching folder asks rather than invents.
# Each depends on the seeded `user` fixture (username / is_admin / folders /
# folders_read) so the prompt's folder guidance and `u/{username}` are well-formed.
- id: global-path1-bare-name-defaults-to-personal
prompt: |-
Stage a quick draft helper that takes a string and returns it trimmed of
leading and trailing whitespace. Just keep it as a draft.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
pathStartsWith: u/admin/
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- stages a single script draft for a trim helper
- defaults the path to the current user's personal scope (u/admin/...) since no path or folder was given
- does not invent an f/<folder> path
- leaves the result as a draft only
- id: global-path2-match-existing-folder
prompt: |-
Draft a flow for the marketing team's weekly campaign report.
It should take a week number and return a short summary string.
Keep it as a draft only.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_folders.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
pathStartsWith: f/marketing/
toolExpect:
requiredToolsUsed:
- write_flow
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- drafts a flow for the marketing campaign report
- places it in the existing marketing folder (f/marketing/...) rather than the personal scope or an invented folder
- leaves the result as a draft only
- id: global-path3-shared-intent-unknown-folder-asks
prompt: |-
Put together a draft onboarding checklist flow for the People Ops team to use
when a new hire joins. Keep it as a draft.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_folders.json
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- askUserQuestion
forbiddenToolsUsed:
- write_flow
- write_script
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- recognizes the request implies shared/team work but names no existing folder (none of marketing/data_engineering/shared_utils fit People Ops)
- asks which folder to use instead of guessing or inventing one
- does not create a draft until the folder is known
- id: global-path4-nonadmin-avoids-readonly-folder
prompt: |-
Draft a small flow that returns today's date as an ISO string, and stage it in
one of our shared team folders. Keep it as a draft.
initial: ai_evals/fixtures/frontend/global/initial/user_bob_nonadmin_teams.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
pathStartsWith: f/team_a/
forbiddenDrafts:
- type: flow
pathStartsWith: f/team_b/
toolExpect:
requiredToolsUsed:
- write_flow
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- drafts a flow that returns the current date as an ISO string
- places it in team_a (writable by this non-admin user) and not team_b (read-only)
- leaves the result as a draft only
+3 -25
View File
@@ -25,9 +25,7 @@ import {
import { runSuite } from "../core/runSuite";
import { EVAL_MODES, type EvalMode } from "../core/types";
import { DEFAULT_JUDGE_MODEL } from "../core/judge";
// createCliModeRunner is imported lazily in runCliBenchmark so the non-cli modes
// (global/flow/script/app) don't pull in the wmill CLI toolchain and its JSR deps
// (e.g. @cliffy/*) just to load this entrypoint.
import { createCliModeRunner } from "../modes/cli";
import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime";
import { resolveWindmillBackendSettings } from "../core/windmillBackendSettings";
import { assertWindmillBackendReachable } from "../adapters/frontend/windmillBackend";
@@ -99,11 +97,6 @@ async function main() {
"comma-separated model aliases to run sequentially",
)
.option("--verbose", "stream assistant output during frontend runs")
.option("--skip-judge", "skip LLM judge scoring for this run")
.option(
"--execution-only",
"only require the model/proxy/frontend loop to complete",
)
.option(
"--record",
"append a compact summary line to ai_evals/history/<mode>.jsonl",
@@ -122,8 +115,6 @@ async function main() {
model?: string;
models?: string;
verbose?: boolean;
skipJudge?: boolean;
executionOnly?: boolean;
record?: boolean;
backendValidation?: string;
},
@@ -136,8 +127,6 @@ async function main() {
model: options.model,
models: options.models,
verbose: options.verbose ?? false,
skipJudge: options.skipJudge ?? false,
executionOnly: options.executionOnly ?? false,
record: options.record ?? false,
backendValidation: options.backendValidation,
});
@@ -186,8 +175,6 @@ async function handleRun(input: {
model?: string;
models?: string;
verbose: boolean;
skipJudge: boolean;
executionOnly: boolean;
record: boolean;
backendValidation?: string;
}) {
@@ -243,8 +230,6 @@ async function handleRun(input: {
input.runs,
getCliEvalModel(model),
runModel,
input.skipJudge,
input.executionOnly,
)
: await runFrontendBenchmarkAdapter({
mode: input.mode,
@@ -252,8 +237,6 @@ async function handleRun(input: {
runs: input.runs,
model: model.id,
verbose: input.verbose,
skipJudge: input.skipJudge,
executionOnly: input.executionOnly,
backendValidation,
});
@@ -295,25 +278,20 @@ async function runCliBenchmark(
runs: number,
model: ReturnType<typeof getCliEvalModel>,
runModel: string,
skipJudge: boolean,
executionOnly: boolean,
) {
const { createCliModeRunner } = await import("../modes/cli");
const judgeModel = skipJudge || executionOnly ? null : DEFAULT_JUDGE_MODEL;
const caseResults = await runSuite({
modeRunner: createCliModeRunner(model),
cases,
runs,
runModel,
judgeModel,
executionOnly,
judgeModel: DEFAULT_JUDGE_MODEL,
});
return buildRunResult({
mode: "cli",
runs,
runModel,
judgeModel,
judgeModel: DEFAULT_JUDGE_MODEL,
caseResults,
});
}
-102
View File
@@ -1,102 +0,0 @@
import { describe, expect, it } from "bun:test";
import { runSuite } from "./runSuite";
import type { ModeRunner } from "./types";
const modeRunner: ModeRunner<undefined, undefined, { ok: boolean }> = {
mode: "global",
concurrency: 1,
loadInitial: async () => undefined,
loadExpected: async () => undefined,
run: async () => ({
success: true,
actual: { ok: true },
assistantMessageCount: 1,
toolCallCount: 0,
toolsUsed: [],
skillsInvoked: [],
tokenUsage: null,
}),
validate: () => [],
};
describe("runSuite", () => {
it("skips judge checks when the run disables judge scoring", async () => {
const [caseResult] = await runSuite({
modeRunner,
cases: [
{
id: "case-1",
prompt: "Create a draft script",
judgeChecklist: ["the output satisfies the prompt"],
},
],
runs: 1,
runModel: "model-under-test",
judgeModel: null,
});
const [attempt] = caseResult.attempts;
expect(attempt.passed).toBe(true);
expect(attempt.judgeScore).toBeNull();
expect(attempt.judgeSummary).toBeNull();
expect(attempt.checks.map((check) => check.name)).toEqual([
"run succeeded",
]);
});
it("only requires run success when execution-only is enabled", async () => {
let loadExpectedCalls = 0;
let validateCalls = 0;
let backendValidateCalls = 0;
const executionOnlyRunner: ModeRunner<
undefined,
undefined,
{ ok: boolean }
> = {
...modeRunner,
loadExpected: async () => {
loadExpectedCalls++;
return undefined;
},
validate: () => {
validateCalls++;
return [{ name: "validator failed", passed: false }];
},
backendValidate: async () => {
backendValidateCalls++;
return {
checks: [{ name: "backend validation failed", passed: false }],
};
},
};
const [caseResult] = await runSuite({
modeRunner: executionOnlyRunner,
cases: [
{
id: "case-1",
prompt: "Create a draft script",
expectedPath: "fixtures/expected.json",
toolExpect: { requiredToolsUsed: ["write_script"] },
judgeChecklist: ["the output satisfies the prompt"],
},
],
runs: 1,
runModel: "model-under-test",
judgeModel: "judge-model",
executionOnly: true,
});
const [attempt] = caseResult.attempts;
expect(attempt.passed).toBe(true);
expect(attempt.judgeScore).toBeNull();
expect(attempt.judgeSummary).toBeNull();
expect(attempt.checks.map((check) => check.name)).toEqual([
"run succeeded",
]);
expect(loadExpectedCalls).toBe(0);
expect(validateCalls).toBe(0);
expect(backendValidateCalls).toBe(0);
});
});
+17 -36
View File
@@ -15,13 +15,11 @@ export async function runSuite<TInitial, TExpected, TActual>(input: {
runs: number;
runModel: string | null;
judgeModel?: string | null;
executionOnly?: boolean;
concurrency?: number;
verbose?: boolean;
onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
}): Promise<BenchmarkCaseResult[]> {
const judgeModel =
input.judgeModel === undefined ? DEFAULT_JUDGE_MODEL : input.judgeModel;
const judgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL;
const concurrency = Math.max(1, input.concurrency ?? input.modeRunner.concurrency);
const results = new Array<BenchmarkCaseResult>(input.cases.length);
let cursor = 0;
@@ -54,7 +52,6 @@ export async function runSuite<TInitial, TExpected, TActual>(input: {
runs: input.runs,
judgeModel,
judgeThreshold: input.modeRunner.judgeThreshold ?? 80,
executionOnly: input.executionOnly ?? false,
modeRunner: input.modeRunner,
totalCases: input.cases.length,
verbose: input.verbose ?? false,
@@ -75,9 +72,8 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
caseIndex: number;
evalCase: EvalCase;
runs: number;
judgeModel: string | null;
judgeModel: string;
judgeThreshold: number;
executionOnly: boolean;
modeRunner: ModeRunner<TInitial, TExpected, TActual>;
totalCases: number;
verbose: boolean;
@@ -103,9 +99,7 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
try {
const initial = await input.modeRunner.loadInitial(input.evalCase.initialPath);
const expected = input.executionOnly
? undefined
: await input.modeRunner.loadExpected(input.evalCase.expectedPath);
const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath);
const run = await input.modeRunner.run(input.evalCase.prompt, initial, {
evalCase: input.evalCase,
caseId: input.evalCase.id,
@@ -168,30 +162,22 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
});
const checks: BenchmarkCheck[] = [
buildCheck("run succeeded", run.success, run.error),
...input.modeRunner.validate({
evalCase: input.evalCase,
prompt: input.evalCase.prompt,
initial,
expected,
actual: run.actual,
run,
}),
...validateToolExpectations({
run,
toolExpect: input.evalCase.toolExpect,
}),
];
if (!input.executionOnly) {
checks.push(
...input.modeRunner.validate({
evalCase: input.evalCase,
prompt: input.evalCase.prompt,
initial,
expected,
actual: run.actual,
run,
}),
...validateToolExpectations({
run,
toolExpect: input.evalCase.toolExpect,
})
);
}
const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? [];
if (
run.success &&
!input.executionOnly &&
input.modeRunner.backendValidate
) {
if (run.success && input.modeRunner.backendValidate) {
try {
const backendValidation = await input.modeRunner.backendValidate({
evalCase: input.evalCase,
@@ -232,12 +218,7 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
let judgeScore: number | null = null;
let judgeSummary: string | null = null;
if (
run.success &&
!input.executionOnly &&
input.judgeModel !== null &&
!input.evalCase.skipJudge
) {
if (run.success && !input.evalCase.skipJudge) {
const judge = await judgeOutput({
mode: input.modeRunner.mode,
prompt: input.evalCase.prompt,
@@ -1,6 +0,0 @@
{
"user": {
"username": "admin",
"is_admin": true
}
}
@@ -1,8 +0,0 @@
{
"user": {
"username": "admin",
"is_admin": true,
"folders": ["marketing", "data_engineering", "shared_utils"],
"folders_read": ["marketing", "data_engineering", "shared_utils"]
}
}
@@ -1,8 +0,0 @@
{
"user": {
"username": "bob",
"is_admin": false,
"folders": ["team_a"],
"folders_read": ["team_a", "team_b"]
}
}
-4
View File
@@ -4,7 +4,6 @@ import { loadAppFixtureForEval } from "../adapters/frontend/core/app/appFixtureL
import {
runGlobalEval,
type GlobalLiveEditorDraftFixture,
type GlobalUserFixture,
} from "../adapters/frontend/core/global/globalEvalRunner";
import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend";
import type { FrontendEvalModelConfig } from "../core/models";
@@ -16,7 +15,6 @@ import { getFrontendApiKey } from "./frontendCommon";
export interface GlobalInitialFixture {
workspace?: BenchmarkWorkspaceRunnables;
liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
user?: GlobalUserFixture;
}
export function createGlobalModeRunner(
@@ -40,7 +38,6 @@ export function createGlobalModeRunner(
{
workspaceFixtures: initial?.workspace,
liveEditorDrafts: initial?.liveEditorDrafts,
user: initial?.user,
maxIterations: context.evalCase?.runtime?.maxTurns,
provider: modelConfig.provider,
model: modelConfig.model,
@@ -107,7 +104,6 @@ async function loadGlobalInitialFixture(path: string): Promise<GlobalInitialFixt
return {
workspace: parsed.workspace ?? {},
liveEditorDrafts: parsed.liveEditorDrafts ?? [],
user: parsed.user,
};
}
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COUNT(*)::bigint AS \"total!\",\n COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS \"replacing!\"\n FROM ai_skill\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "total!",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "replacing!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"TextArray"
]
},
"nullable": [
null,
null
]
},
"hash": "002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7"
}
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT kind::text as \"kind!\", parent_job, runnable_path\n FROM v2_job WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "parent_job",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "runnable_path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null,
true,
true
]
},
"hash": "19f1cb1c7a1974920549917a6392ff0d56f31ca5662062f4a71fed0ccf859cc4"
}
@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_perms\n WHERE ctid IN (\n SELECT jp.ctid FROM job_perms jp\n WHERE NOT EXISTS (SELECT 1 FROM v2_job_queue q WHERE q.id = jp.job_id)\n LIMIT 100000\n )",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "25ecae25ebc03d6296b0e72482a9201c2bffb0f0f7419b0b598b2786bdb326ab"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_result_stream_v2\n WHERE job_id NOT IN (SELECT id FROM v2_job_queue)\n AND job_id NOT IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at > NOW() - INTERVAL '60 seconds'\n )\n RETURNING job_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "job_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "454a611a5a162b2ace137c139bd5383bc7fe142c515dac3edd47991839485e51"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT jc.id FROM v2_job_completed jc\n LEFT JOIN v2_job j ON j.id = jc.id\n WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (\n SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL\n )\n ORDER BY jc.completed_at ASC\n LIMIT $2\n FOR UPDATE OF jc SKIP LOCKED\n )\n RETURNING id",
"query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT jc.id FROM v2_job_completed jc\n LEFT JOIN v2_job j ON j.id = jc.id\n WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) != ALL($3)\n ORDER BY jc.completed_at ASC\n LIMIT $2\n FOR UPDATE OF jc SKIP LOCKED\n )\n RETURNING id",
"describe": {
"columns": [
{
@@ -20,5 +20,5 @@
false
]
},
"hash": "fbe3a876efd1253d2ef086b03366b2bd117ceb6bc152d2abcd45850ff6aecff9"
"hash": "45997fcb4d9d62c7f7011966bf59bdb86e12ce1d0c8e925e738d2645121a5c1f"
}
@@ -1,47 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n snapshot_id = EXCLUDED.snapshot_id,\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
{
"Custom": {
"name": "asset_kind",
"kind": {
"Enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
}
}
},
"Varchar",
"Text",
{
"Custom": {
"name": "materialization_status",
"kind": {
"Enum": [
"running",
"materialized",
"failed"
]
}
}
},
"Int8",
"Int8",
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19"
}
@@ -1,18 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by)\n VALUES ($1, $2, $3, $4, now(), $5)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET description = EXCLUDED.description,\n instructions = EXCLUDED.instructions,\n edited_at = now(),\n edited_by = EXCLUDED.edited_by",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar"
]
},
"nullable": []
},
"hash": "734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $2\n AND (j.kind = 'appscript' OR j.kind = 'preview')\n AND j.created_by = $4\n AND c.started_at > now() - interval '3 hours'\n AND j.runnable_path LIKE $3 || '/%'\n AND c.result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )",
"query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $2\n AND (j.kind = 'appscript' OR j.kind = 'preview')\n AND j.created_by = 'anonymous'\n AND c.started_at > now() - interval '3 hours'\n AND j.runnable_path LIKE $3 || '/%'\n AND c.result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )",
"describe": {
"columns": [
{
@@ -11,7 +11,6 @@
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
@@ -21,5 +20,5 @@
null
]
},
"hash": "f2760b688a907e7679106aaa2c2063385785f7c2e97364bc04392df11cf364d7"
"hash": "a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d"
}
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT runnable_path FROM v2_job WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "runnable_path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
true
]
},
"hash": "c167db39eeed526449dc064eaeb26e648aa9455cb81bab86fd106f76537701ba"
}
@@ -1,111 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT asset_kind AS \"asset_kind: AssetKind\", asset_path, partition,\n status AS \"status: MaterializationStatus\", snapshot_id,\n row_count, job_id, materialized_at, error\n FROM materialized_partition\n WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3\n ORDER BY partition DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "asset_kind: AssetKind",
"type_info": {
"Custom": {
"name": "asset_kind",
"kind": {
"Enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
}
}
}
},
{
"ordinal": 1,
"name": "asset_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "partition",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "status: MaterializationStatus",
"type_info": {
"Custom": {
"name": "materialization_status",
"kind": {
"Enum": [
"running",
"materialized",
"failed"
]
}
}
}
},
{
"ordinal": 4,
"name": "snapshot_id",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "row_count",
"type_info": "Int8"
},
{
"ordinal": 6,
"name": "job_id",
"type_info": "Uuid"
},
{
"ordinal": 7,
"name": "materialized_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "error",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "asset_kind",
"kind": {
"Enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
}
}
},
"Text"
]
},
"nullable": [
false,
false,
false,
false,
true,
true,
true,
false,
true
]
},
"hash": "c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_perms\nWHERE job_id NOT IN (SELECT id FROM v2_job_queue)\nRETURNING job_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "job_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "c825fa5c6e287068aeaad994c0b42b8ad59b9129f032c6b918c27426ab304f2b"
}
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "description",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b"
}
@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_result_stream_v2\n WHERE ctid IN (\n SELECT jrs.ctid FROM job_result_stream_v2 jrs\n WHERE NOT EXISTS (SELECT 1 FROM v2_job_queue q WHERE q.id = jrs.job_id)\n AND NOT EXISTS (\n SELECT 1 FROM v2_job_completed c\n WHERE c.id = jrs.job_id\n AND c.completed_at > NOW() - INTERVAL '60 seconds'\n )\n LIMIT 100000\n )",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "d059b8a3771e4ac4cd07990ecca84daaed616dc1ac609a6ca81bcd446e4dc230"
}
@@ -1,12 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "SELECT NOT pg_is_in_recovery()",
"query": "SELECT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
"type_info": "Int4"
}
],
"parameters": {
@@ -16,5 +16,5 @@
null
]
},
"hash": "282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf"
"hash": "e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5"
}
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT a.policy::text as policy, a.versions[array_upper(a.versions, 1)] as version, av.raw_app as raw_app\n FROM app a JOIN app_version av ON av.id = a.versions[array_upper(a.versions, 1)]\n WHERE a.path = $1 AND a.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "policy",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "version",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "raw_app",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null,
null,
false
]
},
"hash": "e0a40d2aba02bd6c502d746471c9c14db8fcaaaf8e3c44fb5ea4ed763a1849dd"
}
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "instructions",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false
]
},
"hash": "e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d"
}
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61"
}
@@ -1,41 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT a.path, a.policy::text as policy, a.versions[array_upper(a.versions, 1)] as version, av.raw_app as raw_app\n FROM app a JOIN app_version av ON av.id = a.versions[array_upper(a.versions, 1)]\n WHERE a.id = $1 AND a.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "policy",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "version",
"type_info": "Int8"
},
{
"ordinal": 3,
"name": "raw_app",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false,
null,
null,
false
]
},
"hash": "ef9caf3da759ee6059922632cdf1fdf5c006554a70a322ca8d3449cdba7840db"
}
+104 -107
View File
@@ -237,9 +237,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
[[package]]
name = "arrayvec"
version = "0.7.7"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
@@ -2056,9 +2056,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.65"
version = "1.2.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -5233,9 +5233,9 @@ dependencies = [
[[package]]
name = "gosyn"
version = "0.2.11"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed1657682b1c3f63ece1fe5b60fc6c5f5923612a20d19f6af38ce79eaf361e3"
checksum = "c99c1502d84229dc7ddb6af755f40ebe80e7e932fa78ecef979cedcf9999ba93"
dependencies = [
"anyhow",
"strum",
@@ -6715,9 +6715,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.33"
version = "0.4.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
[[package]]
name = "loom"
@@ -7046,9 +7046,9 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
[[package]]
name = "memmap2"
version = "0.9.11"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
dependencies = [
"libc",
"stable_deref_trait",
@@ -8914,9 +8914,9 @@ dependencies = [
[[package]]
name = "pulp"
version = "0.22.3"
version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a"
checksum = "2e205bb30d5b916c55e584c22201771bcf2bad9aabd5d4127f38387140c38632"
dependencies = [
"bytemuck",
"cfg-if",
@@ -8931,9 +8931,9 @@ dependencies = [
[[package]]
name = "pulp-wasm-simd-flag"
version = "0.1.1"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740"
checksum = "40e24eee682d89fb193496edf918a7f407d30175b2e785fe057e4392dfd182e0"
[[package]]
name = "pure-rust-locales"
@@ -8975,9 +8975,9 @@ dependencies = [
[[package]]
name = "quinn"
version = "0.11.11"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
dependencies = [
"bytes",
"cfg_aliases",
@@ -8995,9 +8995,9 @@ dependencies = [
[[package]]
name = "quinn-proto"
version = "0.11.15"
version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
dependencies = [
"aws-lc-rs",
"bytes",
@@ -9031,9 +9031,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.46"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
@@ -12189,9 +12189,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.51"
version = "0.3.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327"
checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469"
dependencies = [
"deranged",
"num-conv",
@@ -12209,9 +12209,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]]
name = "time-macros"
version = "0.2.30"
version = "0.2.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935"
checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d"
dependencies = [
"num-conv",
"time-core",
@@ -13735,7 +13735,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-nats",
@@ -13817,7 +13817,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"async-stream",
"async-trait",
@@ -13850,7 +13850,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -13863,7 +13863,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"argon2",
@@ -14001,7 +14001,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14024,7 +14024,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14037,7 +14037,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14063,7 +14063,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -14073,7 +14073,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14090,7 +14090,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -14112,7 +14112,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14135,7 +14135,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14151,7 +14151,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14172,7 +14172,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14193,7 +14193,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14207,7 +14207,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-nats",
@@ -14242,7 +14242,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14267,7 +14267,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"flate2",
@@ -14285,7 +14285,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14307,7 +14307,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14327,7 +14327,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14354,7 +14354,6 @@ dependencies = [
"windmill-parser",
"windmill-parser-py",
"windmill-parser-py-asset",
"windmill-parser-sql",
"windmill-parser-sql-asset",
"windmill-parser-ts",
"windmill-parser-ts-asset",
@@ -14364,7 +14363,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14392,7 +14391,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"lazy_static",
"serde",
@@ -14404,7 +14403,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -14429,7 +14428,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14443,7 +14442,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14476,7 +14475,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"chrono",
"lazy_static",
@@ -14490,7 +14489,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14509,7 +14508,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -14611,7 +14610,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -14630,7 +14629,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"regex",
"serde",
@@ -14645,7 +14644,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -14669,7 +14668,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"futures",
@@ -14686,7 +14685,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -14702,7 +14701,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -14723,7 +14722,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -14754,7 +14753,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"arc-swap",
@@ -14779,7 +14778,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-stream",
@@ -14813,7 +14812,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"futures",
@@ -14831,7 +14830,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -14840,7 +14839,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14852,7 +14851,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde_json",
@@ -14864,7 +14863,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"gosyn",
@@ -14876,7 +14875,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14888,7 +14887,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde_json",
@@ -14900,7 +14899,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -14911,7 +14910,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14922,7 +14921,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14934,7 +14933,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -14945,7 +14944,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -14967,7 +14966,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde_json",
@@ -14979,7 +14978,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -14993,7 +14992,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15010,7 +15009,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15023,7 +15022,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde",
@@ -15035,7 +15034,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15053,7 +15052,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -15069,7 +15068,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -15085,7 +15084,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde",
@@ -15096,7 +15095,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15134,7 +15133,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"const_format",
@@ -15173,7 +15172,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -15184,19 +15183,17 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-recursion",
"axum 0.8.9",
"base64 0.22.1",
"chrono",
"futures",
"hex",
"http 1.4.2",
"hyper 1.10.1",
"lazy_static",
"magic-crypt",
"quick_cache",
"reqwest 0.13.1",
"serde",
@@ -15218,7 +15215,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15242,7 +15239,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15275,7 +15272,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15308,7 +15305,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15328,7 +15325,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15362,7 +15359,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15398,7 +15395,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15421,7 +15418,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15445,7 +15442,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-nats",
@@ -15469,7 +15466,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15504,7 +15501,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15532,7 +15529,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-trait",
@@ -15557,7 +15554,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"bitflags 2.13.0",
@@ -15576,7 +15573,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-once-cell",
@@ -15686,7 +15683,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"bytes",
"futures",
@@ -16504,9 +16501,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.6.4"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "977347db8caa080403f6b6b7c1cda9479a8e869316f7e13a59b19076a40f94e3"
checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
[[package]]
name = "zmij"
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.737.0"
version = "1.733.1"
authors.workspace = true
edition.workspace = true
@@ -87,7 +87,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.737.0"
version = "1.733.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
+1 -1
View File
@@ -1 +1 @@
ac1f6f666f36141cb6ba6f8eaa614821a90464ad
ba677ea142011462ad4dfe77e8375a6dd274cdef
@@ -1 +0,0 @@
DROP TABLE IF EXISTS ai_skill;
@@ -1,15 +0,0 @@
-- Workspace-scoped AI chat skills (Claude/Codex-style SKILL.md instructions).
-- `name` is the skill folder slug; `description` is advertised in the AI chat
-- system prompt, `instructions` is the SKILL.md body fetched on demand.
CREATE TABLE ai_skill (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
name VARCHAR(255) NOT NULL,
description TEXT NOT NULL,
instructions TEXT NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT now(),
edited_by VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY (workspace_id, name)
);
GRANT ALL ON ai_skill TO windmill_user;
GRANT ALL ON ai_skill TO windmill_admin;
@@ -1,3 +0,0 @@
DROP INDEX IF EXISTS idx_materialized_partition_asset_status;
DROP TABLE IF EXISTS materialized_partition;
DROP TYPE IF EXISTS MATERIALIZATION_STATUS;
@@ -1,29 +0,0 @@
-- Per-partition materialization state for managed `// materialize` assets.
-- One row per (asset, partition): the latest materialization of that slice.
-- Drives: the partition-status grid (CE observability), run-stale/gap
-- detection, and the EE backfill worklist (missing/failed partitions). The
-- `partition` column uses '' as the sentinel for an unpartitioned (whole-table)
-- materialization, since partition is part of the primary key and cannot be
-- NULL.
CREATE TYPE MATERIALIZATION_STATUS AS ENUM ('running', 'materialized', 'failed');
CREATE TABLE IF NOT EXISTS materialized_partition (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE,
asset_kind ASSET_KIND NOT NULL,
asset_path VARCHAR(255) NOT NULL,
partition TEXT NOT NULL DEFAULT '',
status MATERIALIZATION_STATUS NOT NULL,
-- DuckLake snapshot id produced by the write; NULL while running / on
-- failure. The pin that makes downstream reads reproducible.
snapshot_id BIGINT,
row_count BIGINT,
job_id UUID,
materialized_at TIMESTAMPTZ NOT NULL DEFAULT now(),
error TEXT,
PRIMARY KEY (workspace_id, asset_kind, asset_path, partition)
);
-- Backfill enumeration / grid "show only gaps": filter an asset's partitions
-- by status without scanning the whole table.
CREATE INDEX IF NOT EXISTS idx_materialized_partition_asset_status
ON materialized_partition (workspace_id, asset_kind, asset_path, status);
+24 -24
View File
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6272,7 +6272,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"proc-macro2",
"quote",
@@ -6284,7 +6284,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"convert_case",
"serde",
@@ -6293,7 +6293,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6305,7 +6305,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde_json",
@@ -6317,7 +6317,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"gosyn",
@@ -6329,7 +6329,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6341,7 +6341,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde_json",
@@ -6353,7 +6353,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -6364,7 +6364,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6375,7 +6375,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6387,7 +6387,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6398,7 +6398,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -6420,7 +6420,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde_json",
@@ -6432,7 +6432,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6446,7 +6446,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"convert_case",
@@ -6463,7 +6463,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde",
@@ -6488,7 +6488,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6522,7 +6522,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6538,7 +6538,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6570,7 +6570,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"serde",
@@ -6581,7 +6581,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.737.0"
version = "1.733.1"
dependencies = [
"anyhow",
"bitflags",
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.737.0"
version = "1.733.1"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
@@ -107,11 +107,6 @@ pub struct ParseAssetsOutput {
// The delay is a raw duration string parsed at deploy (parser-light).
#[serde(skip_serializing_if = "Option::is_none", default)]
pub retry: Option<RetrySpec>,
// `// materialize [manual] <asset> [append] [key=<col>]` —
// managed-materialization target + its strategy. At most one per script.
// Drives the worker's write-strategy + snapshot capture.
#[serde(skip_serializing_if = "Option::is_none", default)]
pub materialize: Option<MaterializeSpec>,
}
#[derive(Serialize, Debug, PartialEq, Clone)]
@@ -214,27 +209,6 @@ pub struct RetrySpec {
pub delay: Option<String>,
}
// `// materialize [manual] <asset> [append] [key=<col>]` — declares that this
// script produces a *managed* materialization of `<asset>` (a `ducklake://`
// table). By default the runtime generates the write DDL around the script's
// single trailing `SELECT` and owns idempotency, partition-state and snapshot
// capture. `manual` is the escape hatch: the script writes its own DDL and the
// runtime only records state (track-only). The reconciliation strategy options
// (`append`, `key=<col>`) apply to managed mode: none → DELETE-by-partition +
// INSERT (replace); `key=<col>` → MERGE (dedup within slice); `append` →
// INSERT-only. `append` wins if both are given (deploy-time warning).
#[derive(Serialize, Debug, PartialEq, Clone)]
pub struct MaterializeSpec {
pub target_kind: AssetKind,
pub target_path: String,
#[serde(skip_serializing_if = "std::ops::Not::not", default)]
pub manual: bool,
#[serde(skip_serializing_if = "std::ops::Not::not", default)]
pub append: bool,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub unique_key: Option<String>,
}
// `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger
// firing runs the script (current behaviour). `All` = AND: the script
// runs only once every partition-bearing input has materialized at the
@@ -265,7 +239,6 @@ pub struct PipelineAnnotations {
pub debounce_default: Option<String>,
pub tag: Option<String>,
pub retry: Option<RetrySpec>,
pub materialize: Option<MaterializeSpec>,
}
impl ParseAssetsOutput {
@@ -289,7 +262,6 @@ impl ParseAssetsOutput {
debounce_default: pipeline.debounce_default,
tag: pipeline.tag,
retry: pipeline.retry,
materialize: pipeline.materialize,
}
}
}
@@ -599,15 +571,6 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations {
continue;
}
if let Some(after_kw) = consume_keyword(rest, "materialize") {
if out.materialize.is_none() {
if let Some(spec) = parse_materialize_spec(after_kw.trim()) {
out.materialize = Some(spec);
}
}
continue;
}
if let Some(after_kw) = consume_keyword(rest, "on") {
let spec_text = after_kw.trim();
if spec_text.is_empty() {
@@ -655,35 +618,6 @@ fn parse_retry_spec(s: &str) -> Option<RetrySpec> {
Some(RetrySpec { count, delay })
}
// Parse a `// materialize [manual] <asset> [append] [key=<col>]` right-hand
// side. An optional leading `manual` token (whitespace-delimited) opts out of
// managed mode (track-only). The next whitespace token is the target asset URI
// (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the
// remainder are strategy options — bare `append` and `key=<col>` (merge key),
// which apply to managed mode only. A missing/empty target yields `None` (the
// annotation is dropped, fail-safe).
fn parse_materialize_spec(s: &str) -> Option<MaterializeSpec> {
let (manual, rest) = match s.strip_prefix("manual") {
Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => {
(true, after.trim_start())
}
_ => (false, s),
};
let mut it = rest.trim().splitn(2, char::is_whitespace);
let asset_tok = it.next()?;
let opts_str = it.next().unwrap_or("");
let (target_kind, path) = parse_asset_syntax(asset_tok.trim(), true)?;
if path.is_empty() {
return None;
}
let append = opts_str.split_whitespace().any(|t| t == "append");
let unique_key = parse_kv_opts(opts_str)
.get("key")
.filter(|k| !k.is_empty())
.cloned();
Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key })
}
// Parse a `// partitioned <kind> [opts]` right-hand side. Recognized kinds:
// `daily`, `hourly`, `weekly`, `monthly` (with optional tz/format/start),
// and `dynamic key="<jsonpath>"` (plus optional format).
@@ -1110,67 +1044,6 @@ mod pipeline_annotation_tests {
assert!(out.retry.is_none());
}
#[test]
fn materialize_managed_default() {
let out = parse_pipeline_annotations("// materialize ducklake://analytics/orders_daily");
let m = out.materialize.expect("materialize");
assert_eq!(m.target_kind, AssetKind::Ducklake);
assert_eq!(m.target_path, "analytics/orders_daily");
// managed by default; replace strategy (no append / key)
assert!(!m.manual);
assert!(!m.append);
assert_eq!(m.unique_key, None);
}
#[test]
fn materialize_manual_escape_hatch() {
let out =
parse_pipeline_annotations("// materialize manual ducklake://analytics/orders_daily");
let m = out.materialize.expect("materialize");
assert!(m.manual);
assert_eq!(m.target_path, "analytics/orders_daily");
}
#[test]
fn materialize_merge_and_append_options() {
let out =
parse_pipeline_annotations("// materialize ducklake://a/orders_daily key=order_id");
let m = out.materialize.expect("materialize");
assert_eq!(m.unique_key.as_deref(), Some("order_id"));
assert!(!m.append);
let out = parse_pipeline_annotations("// materialize ducklake://a/events append");
let m = out.materialize.expect("materialize");
assert!(m.append);
assert_eq!(m.unique_key, None);
}
#[test]
fn materialize_default_syntax_shorthand() {
let out = parse_pipeline_annotations("// materialize ducklake");
let m = out.materialize.expect("materialize");
assert_eq!(m.target_kind, AssetKind::Ducklake);
assert_eq!(m.target_path, "main");
assert!(!m.manual);
}
#[test]
fn materialize_manual_only_is_dropped() {
// `manual` with no target is not a valid materialization.
let out = parse_pipeline_annotations("// materialize manual");
assert!(out.materialize.is_none());
}
#[test]
fn materialize_first_wins() {
let out = parse_pipeline_annotations(
"// materialize ducklake://a/x\n# materialize manual ducklake://b/y",
);
let m = out.materialize.expect("materialize");
assert_eq!(m.target_path, "a/x");
assert!(!m.manual);
}
#[test]
fn combined() {
let code = concat!(
@@ -1180,8 +1053,7 @@ mod pipeline_annotation_tests {
"// partitioned daily tz=\"UTC\"\n",
"// freshness 2h\n",
"// tag heavy\n",
"// retry 3 5s\n",
"// materialize ducklake://analytics/orders_daily key=order_id\n"
"// retry 3 5s\n"
);
let out = parse_pipeline_annotations(code);
assert!(out.in_pipeline);
@@ -1192,10 +1064,6 @@ mod pipeline_annotation_tests {
let r = out.retry.expect("retry");
assert_eq!(r.count, 3);
assert_eq!(r.delay.as_deref(), Some("5s"));
let m = out.materialize.expect("materialize");
assert!(!m.manual);
assert_eq!(m.target_path, "analytics/orders_daily");
assert_eq!(m.unique_key.as_deref(), Some("order_id"));
}
#[test]
@@ -13,7 +13,6 @@ use serde::Serialize;
use serde_json::Value;
pub mod asset_parser;
pub mod sql_materialize;
/// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types)
#[derive(Clone, Copy, Debug)]
@@ -1,817 +0,0 @@
//! Eligibility classifier + materialization SQL codegen for managed `// materialize`.
//!
//! Managed `// materialize` (the default) promises the script is "setup
//! statements, then one trailing SELECT" — Windmill generates the write DDL
//! around that SELECT (the `// materialize manual` escape hatch opts out and
//! writes its own DDL). This module is the single source of truth for *which
//! block is that SELECT* and *what DDL gets generated*, so save-time validation
//! (deploy path) and run-time codegen (DuckDB executor) can never disagree.
//!
//! Everything here is pure and string-level: no SQL is executed, no type
//! inference is done. The classifier is leading-keyword based and deliberately
//! conservative — anything it can't positively recognize as a read-only output
//! or a known-safe setup statement is rejected, so a script is only accepted
//! for managed mode when its shape is unambiguous.
/// One top-level statement's role in a wrap-mode script.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlockClass {
/// Read-only relation the wrap writes from: `SELECT` / `WITH …SELECT` /
/// `FROM` (DuckDB from-first) / `VALUES` / `TABLE x` / `(UN)PIVOT`.
Output,
/// Known-safe preamble: `ATTACH` / `INSTALL` / `LOAD` / `SET` / `PRAGMA` /
/// `USE` / `CREATE TEMP …`. Runs verbatim before the generated write.
Setup,
/// Anything that writes or whose effect we can't vouch for: non-temp
/// `CREATE` / `INSERT` / `UPDATE` / `DELETE` / `MERGE` / `DROP` / `COPY` /
/// `ALTER` / `TRUNCATE`, or an unrecognized leading keyword. Disqualifies
/// managed mode (the user should use `// materialize manual`).
Disallowed,
}
/// A script accepted for wrapping: zero+ setup blocks then one terminal SELECT.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WrapPlan {
/// Setup statements in source order, verbatim, **without** trailing `;`.
pub setup: Vec<String>,
/// The single terminal output statement, verbatim, **without** trailing `;`.
pub output: String,
}
/// Why a script is not eligible for managed `// materialize`. Carries enough to
/// render the targeted save-time messages.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WrapError {
/// No statements at all (empty / comments only).
Empty,
/// No terminal SELECT — nothing to wrap.
NoOutput,
/// More than one top-level SELECT. `count` is how many were found.
MultipleOutputs { count: usize },
/// A SELECT exists but isn't the last statement (something runs after it).
OutputNotLast,
/// A write/unknown statement appears among the setup blocks. `snippet` is a
/// short prefix of the offending statement for the error message.
DisallowedBlock { snippet: String },
}
impl WrapError {
/// Human-facing, actionable message (matches the spec's rejection text).
pub fn message(&self) -> String {
let base =
"managed `// materialize` requires the script to be setup statements then a single trailing SELECT";
let manual = "use `// materialize manual` to write the DDL yourself";
match self {
WrapError::Empty => format!("{base}: the script is empty."),
WrapError::NoOutput => format!("{base}: found no SELECT — {manual}."),
WrapError::MultipleOutputs { count } => format!(
"{base}: found {count} SELECT statements; combine them with a CTE, or {manual}."
),
WrapError::OutputNotLast => format!(
"{base}: found statements after the SELECT — move them above it, or {manual}."
),
WrapError::DisallowedBlock { snippet } => {
format!("{base}: `{snippet}` writes or is unrecognized — {manual}.")
}
}
}
}
/// Split SQL into top-level, `;`-separated statements, skipping line comments
/// (`-- …`), block comments (`/* … */`), single-quoted strings (`'…'` with
/// `''` escape) and double-quoted identifiers (`"…"`). Semicolons inside any of
/// those are not separators. Returns each statement trimmed, comments stripped,
/// empties dropped. Self-contained so the parser crate stays dependency-free;
/// it must stay behaviourally aligned with the executor's block splitter (both
/// route wrap through `classify_wrap`, so the split they see is this one).
pub fn split_statements(sql: &str) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
let bytes = sql.as_bytes();
let mut i = 0;
let n = bytes.len();
while i < n {
let c = bytes[i] as char;
// line comment — `--` (SQL) or `//`. The `//` form is not SQL, but it
// is how Windmill pipeline annotations (`// materialize`, `// pipeline`,
// …) are written, and they sit above the SQL in the same script; strip
// them so they don't pollute the first statement block's classification
// or the generated setup SQL.
if (c == '-' && i + 1 < n && bytes[i + 1] == b'-')
|| (c == '/' && i + 1 < n && bytes[i + 1] == b'/')
{
while i < n && bytes[i] != b'\n' {
i += 1;
}
continue;
}
// block comment
if c == '/' && i + 1 < n && bytes[i + 1] == b'*' {
i += 2;
while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
i += 1;
}
i += 2;
continue;
}
// single-quoted string
if c == '\'' {
cur.push(c);
i += 1;
while i < n {
cur.push(bytes[i] as char);
if bytes[i] == b'\'' {
// doubled '' is an escaped quote, stay in string
if i + 1 < n && bytes[i + 1] == b'\'' {
cur.push('\'');
i += 2;
continue;
}
i += 1;
break;
}
i += 1;
}
continue;
}
// double-quoted identifier
if c == '"' {
cur.push(c);
i += 1;
while i < n {
cur.push(bytes[i] as char);
if bytes[i] == b'"' {
i += 1;
break;
}
i += 1;
}
continue;
}
if c == ';' {
let t = cur.trim();
if !t.is_empty() {
out.push(t.to_string());
}
cur.clear();
i += 1;
continue;
}
cur.push(c);
i += 1;
}
let t = cur.trim();
if !t.is_empty() {
out.push(t.to_string());
}
out
}
/// Lowercased top-level keyword tokens of a single statement (parens collapsed
/// away: tokens *inside* balanced `(...)` are skipped, so a CTE body's verbs
/// don't leak up). Strings/identifiers are already gone from the split, but we
/// re-guard quotes defensively. Used to disambiguate `WITH …` and `CREATE …`.
fn top_level_keywords(stmt: &str) -> Vec<String> {
let mut toks = Vec::new();
let mut cur = String::new();
let mut depth: i32 = 0;
let bytes = stmt.as_bytes();
let mut i = 0;
let n = bytes.len();
let flush = |cur: &mut String, toks: &mut Vec<String>| {
if !cur.is_empty() {
toks.push(cur.to_lowercase());
cur.clear();
}
};
while i < n {
let c = bytes[i] as char;
if c == '\'' || c == '"' {
let q = bytes[i];
i += 1;
while i < n && bytes[i] != q {
i += 1;
}
i += 1;
continue;
}
if c == '(' {
flush(&mut cur, &mut toks);
depth += 1;
i += 1;
continue;
}
if c == ')' {
if depth > 0 {
depth -= 1;
}
i += 1;
continue;
}
if depth > 0 {
i += 1;
continue;
}
if c.is_alphanumeric() || c == '_' {
cur.push(c);
} else {
flush(&mut cur, &mut toks);
}
i += 1;
}
flush(&mut cur, &mut toks);
toks
}
const OUTPUT_KW: &[&str] = &["select", "from", "values", "table", "pivot", "unpivot"];
const SETUP_KW: &[&str] = &["attach", "install", "load", "set", "pragma", "use"];
const WRITE_VERBS: &[&str] = &["insert", "update", "delete", "merge"];
/// Classify a single statement by its leading keyword (with `WITH`/`CREATE`
/// disambiguation). See [`BlockClass`].
pub fn classify_block(stmt: &str) -> BlockClass {
let kws = top_level_keywords(stmt);
let Some(first) = kws.first().map(String::as_str) else {
return BlockClass::Disallowed;
};
// CREATE TEMP … is setup (staging); any other CREATE is a write.
if first == "create" {
let temp = kws
.iter()
.skip(1)
.take(3)
.any(|k| k == "temp" || k == "temporary");
return if temp {
BlockClass::Setup
} else {
BlockClass::Disallowed
};
}
// WITH … : the main statement's verb decides. CTE bodies are parenthesized,
// so their verbs are not in `kws`; the first top-level write verb or SELECT
// after the CTE list is the real one.
if first == "with" {
for k in kws.iter().skip(1) {
if k == "select" {
return BlockClass::Output;
}
if WRITE_VERBS.contains(&k.as_str()) {
return BlockClass::Disallowed;
}
}
// `WITH x AS (...) SELECT` where SELECT got collapsed is impossible
// (SELECT here is top-level), so a WITH with no top-level verb is a
// malformed/unknown statement — reject conservatively.
return BlockClass::Disallowed;
}
if OUTPUT_KW.contains(&first) {
return BlockClass::Output;
}
if SETUP_KW.contains(&first) {
return BlockClass::Setup;
}
BlockClass::Disallowed
}
/// Validate a script for managed `// materialize` and, on success, return the
/// setup/output split. Enforces the four conditions from the spec:
/// 1. exactly one Output block, 2. it is last, 3. all preceding blocks are
/// Setup, 4. nothing after it.
pub fn classify_wrap(sql: &str) -> Result<WrapPlan, WrapError> {
let stmts = split_statements(sql);
if stmts.is_empty() {
return Err(WrapError::Empty);
}
let classes: Vec<BlockClass> = stmts.iter().map(|s| classify_block(s)).collect();
let output_idxs: Vec<usize> = classes
.iter()
.enumerate()
.filter(|(_, c)| **c == BlockClass::Output)
.map(|(i, _)| i)
.collect();
match output_idxs.len() {
0 => return Err(WrapError::NoOutput),
1 => {}
count => return Err(WrapError::MultipleOutputs { count }),
}
let out_idx = output_idxs[0];
if out_idx != stmts.len() - 1 {
return Err(WrapError::OutputNotLast);
}
// Everything before the output must be Setup (no Disallowed preamble).
for (i, c) in classes.iter().enumerate().take(out_idx) {
if *c != BlockClass::Setup {
return Err(WrapError::DisallowedBlock { snippet: snippet(&stmts[i]) });
}
}
Ok(WrapPlan { setup: stmts[..out_idx].to_vec(), output: stmts[out_idx].clone() })
}
fn snippet(stmt: &str) -> String {
let one_line: String = stmt.split_whitespace().collect::<Vec<_>>().join(" ");
if one_line.chars().count() > 40 {
let truncated: String = one_line.chars().take(40).collect();
format!("{truncated}")
} else {
one_line
}
}
// ---------------------------------------------------------------------------
// Codegen
// ---------------------------------------------------------------------------
/// How a (partition of a) materialized table is reconciled on each run.
/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else
/// `unique_key` → `Merge`, else `Replace`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MaterializeStrategy {
/// DELETE the current partition, then INSERT — partition becomes exactly
/// what the SELECT returned. Full-refresh of the slice.
Replace,
/// Upsert within the slice on `unique_key` (delete-by-key + insert); rows
/// absent from the SELECT are left in place.
Merge { unique_key: String },
/// INSERT only — immutable event-log semantics.
Append,
}
/// Inputs to materialization codegen, all resolved at run time by the worker.
/// Pure: produces SQL text; executes nothing.
#[derive(Debug, Clone)]
pub struct MaterializeCodegen<'a> {
/// Fully-qualified target, e.g. `_wm_target.orders_daily`. Always qualified
/// so a user `USE …;` in setup can't redirect the write.
pub target_qualified: &'a str,
/// The user's output SELECT (verbatim, no trailing `;`) — embedded as a
/// subquery so its own shape is irrelevant to the generated wrapper.
pub select_sql: &'a str,
/// Physical partition column added to the managed table.
pub partition_col: &'a str,
/// SQL expression for the current partition value — a literal like
/// `'2026-06-19'` or a bind placeholder. The caller is responsible for
/// safe quoting/binding.
pub partition_value_sql: &'a str,
/// Whether `// partitioned` applies. When false the table is unpartitioned
/// and the partition column / `SET PARTITIONED BY` are omitted.
pub partitioned: bool,
pub strategy: MaterializeStrategy,
}
impl<'a> MaterializeCodegen<'a> {
/// The ordered statements that perform the materialization, to be run after
/// the setup blocks and inside the caller's execution. The first-run
/// bootstrap is idempotent (`IF NOT EXISTS`), so this is safe to run every
/// time. The DELETE/INSERT body is wrapped in one transaction so a partial
/// failure leaves the prior snapshot intact. Every strategy reduces to
/// DELETE+INSERT (no `MERGE INTO`) — see the `Merge` arm for why.
pub fn statements(&self) -> Vec<String> {
let t = self.target_qualified;
let sel = self.select_sql;
let pcol = self.partition_col;
let pval = self.partition_value_sql;
let mut out = Vec::new();
// Whole-table replace: rebuild the table to match the SELECT's *current*
// schema each run with one atomic `CREATE OR REPLACE` (which DuckLake
// still snapshots). This is the only path that survives a changed SELECT
// or a pre-existing table with a different schema — the persist-and-
// mutate paths below fix the schema at first create.
if !self.partitioned && matches!(self.strategy, MaterializeStrategy::Replace) {
out.push(format!(
"CREATE OR REPLACE TABLE {t} AS SELECT * FROM ({sel});"
));
return out;
}
// Persist-and-mutate (partitioned, or merge/append): bootstrap the table
// if absent, then write into it. The schema is fixed at first create —
// a later SELECT-schema change needs a manual rebuild (schema evolution
// is a follow-up).
if self.partitioned {
out.push(format!(
"CREATE TABLE IF NOT EXISTS {t} AS \
SELECT *, CAST(NULL AS VARCHAR) AS {pcol} FROM ({sel}) WHERE false;"
));
out.push(format!("ALTER TABLE {t} SET PARTITIONED BY ({pcol});"));
} else {
out.push(format!(
"CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;"
));
}
out.push("BEGIN TRANSACTION;".to_string());
// The rows to write, with the partition column appended when partitioned.
let source = if self.partitioned {
format!("SELECT *, {pval} AS {pcol} FROM ({sel})")
} else {
format!("SELECT * FROM ({sel})")
};
match &self.strategy {
MaterializeStrategy::Replace => {
// Only reached when partitioned (whole-table replace returned above).
out.push(format!("DELETE FROM {t} WHERE {pcol} = {pval};"));
out.push(format!("INSERT INTO {t} {source};"));
}
MaterializeStrategy::Append => {
out.push(format!("INSERT INTO {t} {source};"));
}
MaterializeStrategy::Merge { unique_key } => {
// Upsert within the slice via delete-by-key + insert (dbt's
// `delete+insert`): rows whose key is in the incoming SELECT are
// replaced, others are left in place. This deliberately avoids
// `MERGE INTO` — DuckLake's MERGE fails writing the first rows of
// a fresh partition (HTTP 404 on the new parquet), and a failed
// write leaves the table needing a DROP. DELETE+INSERT is the
// same write shape as `replace`, which is reliable. The DELETE is
// scoped to the current partition when partitioned so it stays
// slice-local (a key present in another partition is untouched).
let scope = if self.partitioned {
format!("{pcol} = {pval} AND ")
} else {
String::new()
};
out.push(format!(
"DELETE FROM {t} WHERE {scope}{unique_key} IN (SELECT {unique_key} FROM ({sel}));"
));
out.push(format!("INSERT INTO {t} {source};"));
}
}
out.push("COMMIT;".to_string());
out
}
}
/// The read that captures the DuckLake snapshot id produced by the write, for
/// the given attach alias (e.g. `_wm_target`). The worker runs this last and
/// records the result into `materialized_partition`.
pub fn snapshot_capture_sql(alias: &str) -> String {
format!("SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('{alias}');")
}
/// Reserved attach alias for the materialization target, fully-qualified in all
/// generated SQL so a user `USE …;` in the setup blocks can't redirect the
/// write. The worker resolves the real `ATTACH 'ducklake:…' AS _wm_target (…)`
/// from the target ducklake's config and passes it in as `target_attach`.
pub const TARGET_ALIAS: &str = "_wm_target";
/// Assemble the full ordered statement list the DuckDB executor runs for a
/// managed `// materialize` script. This is the single entry point the worker
/// calls; it composes the already-tested pieces (classifier split → target
/// ATTACH → strategy codegen → snapshot capture) so their ordering lives in one
/// tested place rather than inline in the executor.
///
/// `target_attach` is the real `ATTACH 'ducklake:…' AS _wm_target (…);` string
/// the worker built from config (it depends on resolved credentials, so it
/// can't be generated here). `target_table` is the table within that catalog
/// (e.g. `orders_daily`), referenced as `_wm_target.<table>`. `asset_path` is
/// the full `<name>/<table>` for the result summary. The trailing statement is
/// a one-row summary read (asset / rows / snapshot_id) that is both the job's
/// result (a useful preview) and what the worker records.
pub fn build_wrap_blocks(
plan: &WrapPlan,
target_attach: &str,
target_table: &str,
asset_path: &str,
partition_col: &str,
partition_value_sql: &str,
partitioned: bool,
strategy: MaterializeStrategy,
) -> Vec<String> {
let target_qualified = format!("{TARGET_ALIAS}.{target_table}");
let cg = MaterializeCodegen {
target_qualified: &target_qualified,
select_sql: &plan.output,
partition_col,
partition_value_sql,
partitioned,
strategy,
};
let mut blocks: Vec<String> = Vec::new();
// Setup blocks come from the splitter with their `;` stripped — re-terminate
// each so that when the executor re-joins and re-splits the assembled query,
// adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH)
// don't merge into one malformed statement.
blocks.extend(plan.setup.iter().map(|s| terminate(s)));
blocks.push(target_attach.to_string());
blocks.extend(cg.statements());
blocks.push(materialize_result_sql(
&target_qualified,
asset_path,
partition_col,
partition_value_sql,
partitioned,
));
blocks
}
/// The trailing one-row summary the materialize run returns: the asset it
/// produced, the row count of the materialized slice (the partition when
/// partitioned, else the whole table), and the DuckLake snapshot it created.
/// This is both a useful preview result and the row the worker records.
pub fn materialize_result_sql(
target_qualified: &str,
asset_path: &str,
partition_col: &str,
partition_value_sql: &str,
partitioned: bool,
) -> String {
let (count_expr, partition_sel) = if partitioned {
// Row count is the slice this run wrote (the partition); `partition`
// lets the UI label the count and scope the preview to it.
(
format!(
"(SELECT count(*) FROM {target_qualified} WHERE {partition_col} = {partition_value_sql})"
),
format!("{partition_value_sql} AS partition, "),
)
} else {
(
format!("(SELECT count(*) FROM {target_qualified})"),
String::new(),
)
};
format!(
"SELECT 'ducklake://{asset_path}' AS materialized, \
{partition_sel}{count_expr} AS rows, \
(SELECT max(snapshot_id) FROM ducklake_snapshots('{TARGET_ALIAS}')) AS snapshot_id;"
)
}
// Ensure a statement ends with a single `;`.
fn terminate(stmt: &str) -> String {
let t = stmt.trim_end();
if t.ends_with(';') {
t.to_string()
} else {
format!("{t};")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn ok(sql: &str) -> WrapPlan {
classify_wrap(sql).expect("expected wrap-eligible")
}
fn err(sql: &str) -> WrapError {
classify_wrap(sql).expect_err("expected wrap-ineligible")
}
#[test]
fn split_respects_strings_comments_idents() {
let sql = "SET x=1; -- a; comment\nSELECT ';' AS a, \"weird;col\" /* ; */ FROM t;";
let s = split_statements(sql);
assert_eq!(s.len(), 2);
assert_eq!(s[0], "SET x=1");
assert!(s[1].starts_with("SELECT"));
assert!(s[1].contains("\"weird;col\""));
}
#[test]
fn split_handles_escaped_quote() {
let s = split_statements("SELECT 'it''s; fine' AS a;");
assert_eq!(s.len(), 1);
assert!(s[0].contains("it''s; fine"));
}
#[test]
fn pipeline_annotations_are_stripped() {
// The real shape: `//` annotation lines above the SQL must not pollute
// the first block's classification (regression — they were being read
// as a leading `pipeline` keyword and rejected).
let p = ok("// pipeline\n// materialize ducklake://main/t\n// partitioned daily\nATTACH 'ducklake://main' AS dl;\nSELECT 1 AS id");
assert_eq!(p.setup.len(), 1);
// The annotation lines are gone — the setup block starts at the real
// SQL (the `//` inside `ducklake://main` is legitimately retained).
assert!(p.setup[0].starts_with("ATTACH"));
assert!(p.output.starts_with("SELECT"));
}
#[test]
fn bare_select_is_eligible() {
let p = ok("SELECT a, b FROM t WHERE c = '{partition}'");
assert!(p.setup.is_empty());
assert!(p.output.starts_with("SELECT"));
}
#[test]
fn setup_then_select_is_eligible() {
let p = ok(
"ATTACH 'ducklake://main' AS dl;\n SET memory_limit='4GB';\n SELECT * FROM dl.orders",
);
assert_eq!(p.setup.len(), 2);
assert!(p.output.starts_with("SELECT"));
}
#[test]
fn create_temp_staging_is_setup() {
let p = ok("CREATE TEMP TABLE s AS SELECT 1; SELECT * FROM s");
assert_eq!(p.setup.len(), 1);
assert_eq!(
classify_block("CREATE TEMP TABLE s AS SELECT 1"),
BlockClass::Setup
);
assert_eq!(
classify_block("CREATE OR REPLACE TEMPORARY VIEW v AS SELECT 1"),
BlockClass::Setup
);
}
#[test]
fn with_cte_select_is_output_write_is_disallowed() {
assert_eq!(
classify_block("WITH x AS (SELECT 1) SELECT * FROM x"),
BlockClass::Output
);
// CTE whose main statement inserts is a write, even though it starts WITH.
assert_eq!(
classify_block("WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x"),
BlockClass::Disallowed
);
}
#[test]
fn from_first_and_values_are_output() {
assert_eq!(classify_block("FROM t SELECT a"), BlockClass::Output);
assert_eq!(classify_block("VALUES (1),(2)"), BlockClass::Output);
assert_eq!(classify_block("TABLE t"), BlockClass::Output);
}
#[test]
fn trailing_write_rejected() {
assert_eq!(
err("SELECT * FROM t; INSERT INTO u VALUES (1)"),
WrapError::OutputNotLast
);
}
#[test]
fn write_in_preamble_rejected() {
match err("INSERT INTO t VALUES (1); SELECT * FROM t") {
WrapError::DisallowedBlock { snippet } => assert!(snippet.starts_with("INSERT")),
e => panic!("wrong error: {e:?}"),
}
}
#[test]
fn multiple_selects_rejected() {
assert_eq!(
err("SELECT 1; SELECT 2"),
WrapError::MultipleOutputs { count: 2 }
);
}
#[test]
fn no_select_and_empty_rejected() {
assert_eq!(err("CREATE TABLE t (a INT)"), WrapError::NoOutput);
assert_eq!(err(" -- just a comment\n"), WrapError::Empty);
}
#[test]
fn use_cannot_redirect_is_classified_setup() {
// `USE` is allowed setup; generated SQL is fully qualified regardless.
assert_eq!(classify_block("USE dl"), BlockClass::Setup);
}
#[test]
fn codegen_replace_partitioned() {
let cg = MaterializeCodegen {
target_qualified: "_wm_target.orders_daily",
select_sql: "SELECT a FROM dl.orders",
partition_col: "_wm_partition",
partition_value_sql: "'2026-06-19'",
partitioned: true,
strategy: MaterializeStrategy::Replace,
};
let st = cg.statements();
assert!(st[0].contains("CREATE TABLE IF NOT EXISTS _wm_target.orders_daily"));
assert!(st[0].contains("CAST(NULL AS VARCHAR) AS _wm_partition"));
assert!(st.iter().any(
|s| s == "ALTER TABLE _wm_target.orders_daily SET PARTITIONED BY (_wm_partition);"
));
assert!(st.iter().any(|s| s.starts_with(
"DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'"
)));
assert!(st.iter().any(|s| s.contains(
"INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19' AS _wm_partition"
)));
assert_eq!(st.first().map(|_| &st[st.len() - 1]).unwrap(), "COMMIT;");
}
#[test]
fn codegen_merge_is_delete_by_key_plus_insert() {
let cg = MaterializeCodegen {
target_qualified: "_wm_target.orders_daily",
select_sql: "SELECT order_id, amount FROM dl.orders",
partition_col: "_wm_partition",
partition_value_sql: "'2026-06-19'",
partitioned: true,
strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() },
};
let st = cg.statements();
// upsert = delete-by-key (partition-scoped) + insert — NO `MERGE INTO`
// (DuckLake's MERGE fails on fresh partitions).
assert!(!st.iter().any(|s| s.contains("MERGE INTO")));
let del = st
.iter()
.find(|s| s.starts_with("DELETE FROM"))
.expect("delete stmt");
assert!(del.contains(
"WHERE _wm_partition = '2026-06-19' AND order_id IN (SELECT order_id FROM (SELECT order_id, amount FROM dl.orders))"
));
assert!(st
.iter()
.any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'")));
}
#[test]
fn codegen_append_inserts_only() {
let cg = MaterializeCodegen {
target_qualified: "_wm_target.events",
select_sql: "SELECT * FROM dl.raw",
partition_col: "_wm_partition",
partition_value_sql: "'2026-06-19'",
partitioned: true,
strategy: MaterializeStrategy::Append,
};
let st = cg.statements();
assert!(st
.iter()
.any(|s| s.starts_with("INSERT INTO _wm_target.events")));
assert!(!st.iter().any(|s| s.starts_with("DELETE")));
assert!(!st.iter().any(|s| s.starts_with("MERGE")));
}
#[test]
fn codegen_whole_table_replace_is_create_or_replace() {
// Unpartitioned replace must use CREATE OR REPLACE so a changed SELECT
// schema (or a pre-existing table with a different schema) doesn't break
// — and nothing else (no bootstrap / DELETE / INSERT / txn).
let cg = MaterializeCodegen {
target_qualified: "_wm_target.customer_dim",
select_sql: "SELECT a, b, c FROM dl.src",
partition_col: "_wm_partition",
partition_value_sql: "''",
partitioned: false,
strategy: MaterializeStrategy::Replace,
};
let st = cg.statements();
assert_eq!(
st,
vec![
"CREATE OR REPLACE TABLE _wm_target.customer_dim AS SELECT * FROM (SELECT a, b, c FROM dl.src);"
.to_string()
]
);
}
#[test]
fn snapshot_capture_targets_alias() {
assert_eq!(
snapshot_capture_sql("_wm_target"),
"SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('_wm_target');"
);
}
#[test]
fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() {
let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'");
let blocks = build_wrap_blocks(
&plan,
"ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');",
"orders_daily",
"main/orders_daily",
"_wm_partition",
"'2026-06-19'",
true,
MaterializeStrategy::Replace,
);
// setup block first, then the target ATTACH, then codegen, then result.
assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl"));
// every setup block must be `;`-terminated so re-splitting can't merge it
// with the synthetic target ATTACH that follows.
assert!(blocks[0].ends_with(';'));
assert_eq!(
blocks[1],
"ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');"
);
assert!(blocks.iter().any(|b| b.contains("_wm_target.orders_daily")));
assert!(blocks.iter().any(|b| b.starts_with(
"DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'"
)));
// the trailing block is the one-row summary (asset / rows / snapshot_id),
// partition-scoped for the row count
let last = blocks.last().unwrap();
assert!(last.contains("'ducklake://main/orders_daily' AS materialized"));
assert!(last.contains("'2026-06-19' AS partition"));
assert!(last.contains("WHERE _wm_partition = '2026-06-19') AS rows"));
assert!(last.contains("ducklake_snapshots('_wm_target')"));
}
}
@@ -234,72 +234,5 @@
"tag": null,
"retry": null
}
},
{
"name": "materialize managed (default) with merge key",
"code": "// pipeline\n// materialize ducklake://analytics/orders_daily key=order_id\nSELECT 1;",
"expected": {
"in_pipeline": true,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null,
"materialize": {
"target_kind": "ducklake",
"target_path": "analytics/orders_daily",
"unique_key": "order_id"
}
}
},
{
"name": "materialize manual escape hatch, first value wins",
"code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}",
"expected": {
"in_pipeline": false,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null,
"materialize": {
"target_kind": "ducklake",
"target_path": "analytics/orders_daily",
"manual": true
}
}
},
{
"name": "materialize default-syntax shorthand with append",
"code": "// materialize ducklake append\nexport function main() {}",
"expected": {
"in_pipeline": false,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null,
"materialize": {
"target_kind": "ducklake",
"target_path": "main",
"append": true
}
}
},
{
"name": "materialize manual with no target is dropped",
"code": "// materialize manual\nexport function main() {}",
"expected": {
"in_pipeline": false,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null
}
}
]
@@ -34,22 +34,6 @@ struct Expected {
freshness: Option<String>,
tag: Option<String>,
retry: Option<ExpectedRetry>,
// Default-on-absent so the pre-existing fixtures (which omit it) keep
// deserializing; only fixtures exercising materialization set it.
#[serde(default)]
materialize: Option<ExpectedMaterialize>,
}
#[derive(Deserialize)]
struct ExpectedMaterialize {
target_kind: String,
target_path: String,
#[serde(default)]
manual: bool,
#[serde(default)]
append: bool,
#[serde(default)]
unique_key: Option<String>,
}
#[derive(Deserialize)]
@@ -169,25 +153,5 @@ fn pipeline_annotation_fixtures_match() {
want.is_some()
),
}
match (&got.materialize, &f.expected.materialize) {
(None, None) => {}
(Some(m), Some(e)) => {
assert_eq!(
kind_str(m.target_kind),
e.target_kind,
"{ctx}: materialize kind"
);
assert_eq!(m.target_path, e.target_path, "{ctx}: materialize path");
assert_eq!(m.manual, e.manual, "{ctx}: materialize manual");
assert_eq!(m.append, e.append, "{ctx}: materialize append");
assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key");
}
(got, want) => panic!(
"{ctx}: materialize mismatch — got {:?}, want present={}",
got,
want.is_some()
),
}
}
}
+12 -38
View File
@@ -1446,45 +1446,19 @@ Windmill Community Edition {GIT_VERSION}
} else {
None
};
// Hard cap on a single monitor pass. monitor_db runs all its
// periodic tasks under one join!, so a single task stuck on a
// non-DB await (statement_timeout only bounds DB statements)
// would otherwise freeze the whole loop indefinitely — silently
// stopping critical maintenance like audit-partition creation.
// Larger than statement_timeout (5min) so a slow-but-progressing
// statement is never killed prematurely.
const MONITOR_DB_TIMEOUT: Duration = Duration::from_secs(600);
let monitor_timed_out = tokio::time::timeout(
MONITOR_DB_TIMEOUT,
monitor_db(
&conn,
&base_internal_url,
server_mode,
worker_mode,
false,
tx.clone(),
Some(MonitorIteration {
rd_shift,
iter: monitor_iteration,
}),
),
monitor_db(
&conn,
&base_internal_url,
server_mode,
worker_mode,
false,
tx.clone(),
Some(MonitorIteration {
rd_shift,
iter: monitor_iteration,
}),
)
.await
.is_err();
if monitor_timed_out {
windmill_common::utils::report_critical_error(
format!(
"monitor task did not finish within {}s and was aborted; \
a background maintenance task is likely stuck. \
Continuing to the next iteration.",
MONITOR_DB_TIMEOUT.as_secs()
),
db.clone(),
None,
None,
)
.await;
}
.await;
monitor_iteration += 1;
if let Some(handle) = warn_handle {
handle.abort();
+52 -144
View File
@@ -1532,22 +1532,14 @@ async fn delete_expired_jobs_batch(
.await?;
// Use FOR UPDATE SKIP LOCKED to avoid contention between replicas
// ORDER BY completed_at ensures we delete oldest jobs first.
// Active-root exclusion uses `NOT IN (SELECT ... unnest($3))` rather than
// `!= ALL($3)`: the subquery form lets the planner build a one-time hashed
// SubPlan and apply it as a filter on the ordered index scan, giving O(1)
// membership per candidate instead of a per-row linear array scan (which
// degrades sharply when many root jobs are active). The `u IS NOT NULL` guard
// sidesteps NOT IN's null-trap semantics ($3 holds non-null PK ids).
// ORDER BY completed_at ensures we delete oldest jobs first
let deleted_jobs: Vec<Uuid> = sqlx::query_scalar!(
"DELETE FROM v2_job_completed
WHERE id IN (
SELECT jc.id FROM v2_job_completed jc
LEFT JOIN v2_job j ON j.id = jc.id
WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
)
AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) != ALL($3)
ORDER BY jc.completed_at ASC
LIMIT $2
FOR UPDATE OF jc SKIP LOCKED
@@ -1646,30 +1638,11 @@ async fn delete_log_files_from_disk_and_store(
.collect();
let stream = futures::stream::iter(s3_paths).boxed();
let mut result = os.delete_stream(stream);
let mut deleted = 0u64;
let mut not_found = 0u64;
let mut failed = 0u64;
while let Some(r) = result.next().await {
match r {
Ok(_) => deleted += 1,
// Deleting a non-existent object is a successful no-op. S3's
// DeleteObjects ignores missing keys, but GCS returns 404 per
// delete, surfacing as NotFound — count it separately rather
// than logging it as an error.
Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) => {
not_found += 1;
}
Err(e) => {
failed += 1;
tracing::error!("Failed to delete from object store: {e}");
}
if let Err(e) = r {
tracing::error!("Failed to delete from object store: {e}");
}
}
if deleted + not_found + failed > 0 {
tracing::info!(
"object store log cleanup: {deleted} deleted, {not_found} already absent (404), {failed} failed"
);
}
}
}
}
@@ -4376,65 +4349,39 @@ RETURNING key,job_id
Ok(())
}
// Per-statement cap keeps each delete short and lock-light; the per-cycle batch
// cap bounds total work per monitor iteration so monitor_db stays responsive.
// A large backlog drains across several iterations rather than one long delete.
const ORPHAN_CLEANUP_BATCH_SIZE: u64 = 100_000;
const ORPHAN_CLEANUP_MAX_BATCHES: usize = 10;
async fn cleanup_job_perms_orphaned(db: &DB) -> error::Result<()> {
let mut total: u64 = 0;
for _ in 0..ORPHAN_CLEANUP_MAX_BATCHES {
let count = sqlx::query!(
"DELETE FROM job_perms
WHERE ctid IN (
SELECT jp.ctid FROM job_perms jp
WHERE NOT EXISTS (SELECT 1 FROM v2_job_queue q WHERE q.id = jp.job_id)
LIMIT 100000
)"
)
.execute(db)
.await?
.rows_affected();
total += count;
if count < ORPHAN_CLEANUP_BATCH_SIZE {
break;
}
}
let result = sqlx::query_scalar!(
"DELETE FROM job_perms
WHERE job_id NOT IN (SELECT id FROM v2_job_queue)
RETURNING job_id"
)
.fetch_all(db)
.await?;
if total > 0 {
tracing::info!("Cleaned up {total} orphaned job_perms rows");
if !result.is_empty() {
tracing::info!("Cleaned up {} orphaned job_perms rows", result.len());
}
Ok(())
}
async fn cleanup_job_result_stream_orphaned_jobs(db: &DB) -> error::Result<()> {
let mut total: u64 = 0;
for _ in 0..ORPHAN_CLEANUP_MAX_BATCHES {
let count = sqlx::query!(
"DELETE FROM job_result_stream_v2
WHERE ctid IN (
SELECT jrs.ctid FROM job_result_stream_v2 jrs
WHERE NOT EXISTS (SELECT 1 FROM v2_job_queue q WHERE q.id = jrs.job_id)
AND NOT EXISTS (
SELECT 1 FROM v2_job_completed c
WHERE c.id = jrs.job_id
AND c.completed_at > NOW() - INTERVAL '60 seconds'
)
LIMIT 100000
)",
)
.execute(db)
.await?
.rows_affected();
total += count;
if count < ORPHAN_CLEANUP_BATCH_SIZE {
break;
}
}
let result = sqlx::query!(
"DELETE FROM job_result_stream_v2
WHERE job_id NOT IN (SELECT id FROM v2_job_queue)
AND job_id NOT IN (
SELECT id FROM v2_job_completed
WHERE completed_at > NOW() - INTERVAL '60 seconds'
)
RETURNING job_id",
)
.fetch_all(db)
.await?;
if total > 0 {
tracing::info!("Cleaned up {total} orphaned job_result_stream_v2 rows");
if result.len() > 0 {
tracing::info!(
"Cleaned up {} orphaned job_result_stream_v2 rows",
result.len()
);
}
Ok(())
}
@@ -4470,19 +4417,11 @@ async fn audit_log_retention_days() -> i64 {
}
}
/// Number of days ahead (including today) for which an audit partition must
/// always exist. A missing partition in this window means audit inserts fail
/// once that date is reached — and because some callers (notably login) write
/// the audit row in the same transaction as their own work, that failure
/// poisons the whole transaction, so a missing partition is a hard outage, not
/// just a dropped audit row.
const AUDIT_PARTITION_LOOKAHEAD_DAYS: i64 = 3;
async fn manage_audit_partitions(db: &DB, retention_days: i64) {
let today = chrono::Utc::now().date_naive();
// Create partitions for today and the next few days
for days_ahead in 0..=AUDIT_PARTITION_LOOKAHEAD_DAYS {
// Create partitions for today and the next 3 days
for days_ahead in 0..=3i64 {
let date = today + chrono::Duration::days(days_ahead);
let next_date = date + chrono::Duration::days(1);
let partition_name = format!("audit_{}", date.format("%Y%m%d"));
@@ -4498,6 +4437,9 @@ async fn manage_audit_partitions(db: &DB, retention_days: i64) {
}
}
// Drop expired partitions
let cutoff_date = today - chrono::Duration::days(retention_days);
let partitions = sqlx::query_scalar::<_, String>(
"SELECT c.relname::text \
FROM pg_inherits i \
@@ -4507,62 +4449,28 @@ async fn manage_audit_partitions(db: &DB, retention_days: i64) {
.fetch_all(db)
.await;
let partitions = match partitions {
Ok(partitions) => partitions,
Err(e) => {
tracing::error!("Error listing audit partitions: {e:?}");
return;
}
};
// Verify the lookahead window is actually covered. If a create above failed
// (or this loop has not run for several days), alert loudly instead of
// letting it surface days later as failed audit inserts and broken logins.
let existing: std::collections::HashSet<&str> = partitions.iter().map(|s| s.as_str()).collect();
let missing: Vec<String> = (0..=AUDIT_PARTITION_LOOKAHEAD_DAYS)
.map(|days_ahead| {
format!(
"audit_{}",
(today + chrono::Duration::days(days_ahead)).format("%Y%m%d")
)
})
.filter(|name| !existing.contains(name.as_str()))
.collect();
if !missing.is_empty() {
report_critical_error(
format!(
"Audit log partitions missing after maintenance run: {}. \
Audit inserts will fail once these dates are reached, which also \
breaks logins (the login audit row shares the login transaction). \
Check for earlier 'Error creating audit partition' logs and verify \
the audit-partition maintenance loop is still running.",
missing.join(", ")
),
db.clone(),
None,
None,
)
.await;
}
// Drop expired partitions
let cutoff_date = today - chrono::Duration::days(retention_days);
for partition_name in &partitions {
if let Some(date_str) = partition_name.strip_prefix("audit_") {
if let Ok(date) = chrono::NaiveDate::parse_from_str(date_str, "%Y%m%d") {
if date < cutoff_date {
let quoted_name = format!("\"{}\"", partition_name.replace('"', "\"\""));
let sql = format!("DROP TABLE IF EXISTS {quoted_name}");
match sqlx::query(&sql).execute(db).await {
Ok(_) => {
tracing::info!("Dropped expired audit partition {partition_name}")
match partitions {
Ok(partitions) => {
for partition_name in partitions {
if let Some(date_str) = partition_name.strip_prefix("audit_") {
if let Ok(date) = chrono::NaiveDate::parse_from_str(date_str, "%Y%m%d") {
if date < cutoff_date {
let quoted_name =
format!("\"{}\"", partition_name.replace('"', "\"\""));
let sql = format!("DROP TABLE IF EXISTS {quoted_name}");
match sqlx::query(&sql).execute(db).await {
Ok(_) => tracing::info!(
"Dropped expired audit partition {partition_name}"
),
Err(e) => tracing::error!(
"Error dropping audit partition {partition_name}: {e:?}"
),
}
}
Err(e) => tracing::error!(
"Error dropping audit partition {partition_name}: {e:?}"
),
}
}
}
}
Err(e) => tracing::error!("Error listing audit partitions: {e:?}"),
}
}
-39
View File
@@ -19,45 +19,6 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, sc
ARRAY['jobs:read', 'if_jobs:filter_tags:deno']
);
-- App embed token for the admin viewer (test-user). Mirrors a minted sandboxed
-- low-code app token: carries the `app_embed` sentinel plus the embed scope set.
-- Used to assert the token is confined to jobs the viewer LAUNCHED, not every job
-- the (admin) viewer could otherwise read.
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (
encode(sha256('EMBED_APP_TOKEN'::bytea), 'hex'), 'EMBED_APP_', 'EMBED_APP_TOKEN',
'test@windmill.dev', 'app embed token', false,
ARRAY['apps:run', 'jobs:read', 'app_embed', 'resources:run', 'users:read', 'folders:read']
);
-- A completed app-component job LAUNCHED BY the admin viewer (created_by =
-- test-user), running as the app owner. The embed token must keep reading its own
-- launched job (the `created_by == viewer` fast path).
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner, args
) VALUES (
'12121212-1212-1212-1212-121212121212', 'test-workspace', 'test-user',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/app_component', 'deno', false,
'{"own": "arg"}'
);
INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES
('12121212-1212-1212-1212-121212121212', 'test-workspace', 1000, 'success'::job_status,
'{"own": "EMBED_OWN_RESULT"}');
-- A QUEUED job launched by the admin embed viewer (created_by = test-user). The
-- embed token may cancel its own launched job; it must NOT cancel another user's.
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner
) VALUES (
'13131313-1313-1313-1313-131313131313', 'test-workspace', 'test-user',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/app_component', 'deno', false
);
INSERT INTO public.v2_job_queue (id, workspace_id, scheduled_for, running, tag) VALUES
('13131313-1313-1313-1313-131313131313', 'test-workspace', '2023-01-01 00:00:00', false, 'deno');
-- RUNNING job: queued (no completed row) and owned by test-user-2. Used to check
-- that `completed/get_result_maybe?get_started=true` authorizes before disclosing
-- running-state to a non-reader.
-86
View File
@@ -38,10 +38,6 @@ const TOP_SECRET_FLOW: &str = "ffffffff-ffff-ffff-ffff-ffffffffffff";
const DEEP_LEAF_JOB: &str = "88888888-8888-8888-8888-888888888888";
// A queued/running job (no completed row) owned by test-user-2.
const RUNNING_JOB: &str = "77777777-7777-7777-7777-777777777777";
// An app-component job launched BY the admin embed viewer (created_by test-user).
const EMBED_OWN_JOB: &str = "12121212-1212-1212-1212-121212121212";
// A QUEUED job launched by the embed viewer (created_by test-user) — cancelable by it.
const EMBED_OWN_QUEUED: &str = "13131313-1313-1313-1313-131313131313";
// Secrets that must never leak to an unauthorized viewer.
const RESULT_SECRET: &str = "RESULT_SECRET";
@@ -63,19 +59,6 @@ async fn get(base: &str, path: &str, token: Option<&str>) -> (reqwest::StatusCod
(status, body)
}
async fn post(base: &str, path: &str, token: Option<&str>) -> (reqwest::StatusCode, String) {
let mut req = client()
.post(format!("{base}/{path}"))
.json(&serde_json::json!({}));
if let Some(token) = token {
req = req.header("Authorization", format!("Bearer {token}"));
}
let resp = req.send().await.expect("request");
let status = resp.status();
let body = resp.text().await.expect("body");
(status, body)
}
#[sqlx::test(fixtures("base", "jobs_read_auth"))]
async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
@@ -298,75 +281,6 @@ async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Resul
"top flow in an unreadable folder must stay denied (got {status}): {body}"
);
// ---- APP EMBED TOKEN: confined to jobs the viewer LAUNCHED, not everything
// the (admin) viewer can otherwise read. The token carries the `app_embed`
// sentinel; an admin's normal token reads VICTIM (asserted above), but the
// embed token must stop at the `created_by == viewer` grant so user-authored
// app JS can't reuse it to read unrelated jobs by UUID.
// Its own launched component job (created_by == viewer) still reads.
let (status, body) = get(
&base,
&format!("completed/get_result/{EMBED_OWN_JOB}"),
Some("EMBED_APP_TOKEN"),
)
.await;
assert!(
status.is_success(),
"embed token must read a job it launched (got {status}): {body}"
);
assert!(
body.contains("EMBED_OWN_RESULT"),
"embed token should get its own launched job result: {body}"
);
// The VICTIM job — created by another user but readable by this admin viewer's
// normal token (asserted above) — is denied to the embed token across result /
// logs / live update. NotFound (not 403) so the untrusted app can't even probe
// existence, and no secret leaks.
for path in [
format!("completed/get_result/{VICTIM}"),
format!("get_logs/{VICTIM}"),
format!("getupdate/{VICTIM}?only_result=true"),
] {
let (status, body) = get(&base, &path, Some("EMBED_APP_TOKEN")).await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"embed token must not read a job it did not launch ({path}, got {status}): {body}"
);
for secret in [RESULT_SECRET, ARGS_SECRET, LOGS_SECRET] {
assert!(
!body.contains(secret),
"embed token response for {path} leaked `{secret}`: {body}"
);
}
}
// ---- APP EMBED TOKEN: cancellation confined to the app's own jobs. The token
// may cancel a job it launched (created_by == viewer), but `cancel_job_api`
// denies (NotFound) a job created by someone else, even though cancel
// otherwise has no per-job ownership check.
let (status, body) = post(
&base,
&format!("queue/cancel/{EMBED_OWN_QUEUED}"),
Some("EMBED_APP_TOKEN"),
)
.await;
assert!(
status.is_success(),
"embed token must cancel a job it launched (got {status}): {body}"
);
let (status, body) = post(
&base,
&format!("queue/cancel/{RUNNING_JOB}"),
Some("EMBED_APP_TOKEN"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"embed token must not cancel another user's job (got {status}): {body}"
);
// ---- UNAUTHENTICATED, unchanged: an anonymous-created job is readable
// without a token (public trigger / public app result polling).
let (status, body) = get(&base, &format!("completed/get_result/{ANON_JOB}"), None).await;
-57
View File
@@ -22,63 +22,6 @@ pub fn workspaced_service() -> Router {
.route("/list_favorites", get(list_favorites))
.route("/graph", get(asset_graph))
.route("/pipelines", get(list_pipeline_folders))
.route("/partitions", get(list_partitions))
.route("/record_materialization", post(record_materialization))
}
#[derive(Deserialize)]
struct PartitionsQuery {
// The materialized asset path (`<ducklake>/<table>`).
path: String,
}
// Per-partition materialization status for a ducklake asset — drives the
// partition-status grid and the backfill worklist. Materialization targets are
// ducklake-only in v1, so the kind is fixed.
async fn list_partitions(
authed: ApiAuthed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
Query(q): Query<PartitionsQuery>,
) -> JsonResult<Vec<windmill_common::materialization::MaterializedPartition>> {
let mut tx = user_db.begin(&authed).await?;
let rows = windmill_common::materialization::list_materialized_partitions(
&mut *tx,
&w_id,
AssetKind::Ducklake,
&q.path,
)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
// Record a materialization outcome from a polyglot (Python/TS) `wmill.ducklake`
// helper running as a pipeline step. The DuckDB `// materialize` engine records
// this itself; the SDK helpers post here instead so SDK-materialized slices show
// up in the grid identically. RLS-scoped to the caller's workspace.
async fn record_materialization(
authed: ApiAuthed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
Json(req): Json<windmill_common::materialization::RecordMaterializationRequest>,
) -> JsonResult<()> {
let mut tx = user_db.begin(&authed).await?;
windmill_common::materialization::record_materialization(
&mut *tx,
&w_id,
req.asset_kind,
&req.asset_path,
&req.partition,
req.status,
req.snapshot_id,
req.row_count,
req.job_id,
req.error.as_deref(),
)
.await?;
tx.commit().await?;
Ok(Json(()))
}
#[derive(Deserialize)]
+1 -5
View File
@@ -191,11 +191,7 @@ impl AuthCache {
is_operator: claims.is_operator,
groups: claims.groups,
folders: claims.folders,
// Honor the scopes embedded in the JWT (mirrors the EE
// jwt_ext_ branch). The route middleware only enforces
// scopes when Some, so a None-scoped JWT (e.g. the job
// WM_TOKEN) keeps full user privileges as before.
scopes: claims.scopes,
scopes: None,
username_override,
token_prefix: claims.audit_span,
read_only: false,
+1 -218
View File
@@ -274,7 +274,6 @@ pub enum ScopeDomain {
Configs,
OAuth,
AI,
AiSkills,
Indexer,
Teams, // Microsoft Teams integration
@@ -330,7 +329,6 @@ impl ScopeDomain {
Self::Configs => "configs",
Self::OAuth => "oauth",
Self::AI => "ai",
Self::AiSkills => "ai_skills",
Self::Capture => "capture",
Self::Drafts => "drafts",
Self::Favorites => "favorites",
@@ -380,7 +378,6 @@ impl ScopeDomain {
"configs" => Some(Self::Configs),
"oauth" => Some(Self::OAuth),
"ai" => Some(Self::AI),
"ai_skills" => Some(Self::AiSkills),
"indexer" | "srch" => Some(Self::Indexer),
"teams" => Some(Self::Teams),
"native_triggers" => Some(Self::NativeTriggers),
@@ -451,30 +448,6 @@ pub fn check_route_access(
// Find the domain and kind for this route
let (required_domain, required_kind, route_suffix) = extract_domain_from_route(route_path)?;
// App embed tokens (sentinel) carry broad read scopes (`jobs:read`,
// `users:read`, `folders:read`) that exist only for a handful of routes. The
// whole `/users`, `/folders` and `/jobs` routers are CORS-enabled for the
// opaque app iframe, so default-deny everything in those domains except the
// intended routes — otherwise the token could enumerate/export workspace data.
if has_app_embed_sentinel(Some(token_scopes)) {
if let Some(suffix) = route_suffix.as_deref() {
if app_embed_route_denied(required_domain, suffix) {
return Err(Error::PermissionDenied(
"Access denied. App embed token cannot access this route.".to_string(),
));
}
// The by-id job cancel is a POST (write) that the token's `jobs:read`
// wouldn't satisfy, but cancelling the app's own component runs is
// intended (most components supersede an in-flight run on re-run). Permit
// it here; `cancel_job_api` confines it to jobs the app launched
// (created_by == viewer). A read_only token is still rejected by the
// separate read-only check.
if suffix.starts_with("jobs_u/queue/cancel/") {
return Ok(());
}
}
}
// MCP scopes (mcp:all, mcp:favorites, mcp:hub:*, etc.) use a custom format
// that doesn't fit the standard domain:action model. Verify the token has at
// least one mcp: scope; MCP handlers do their own fine-grained checking.
@@ -561,7 +534,7 @@ const FLOW_JOBS: [&'static str; 6] = [
lazy_static::lazy_static! {
static ref RUN_PATH_ACTIONS: Vec<&'static str> = {
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"];
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component"];
v.extend(SCRIPT_JOBS);
v.extend(FLOW_JOBS);
@@ -664,92 +637,6 @@ const RUN_WHITELISTED_GET_PATHS: [&'static str; 20] = [
"jobs/completed/get_result_maybe/",
];
/// Sentinel scope in app embed tokens. Grants nothing itself; `check_route_access`
/// uses it to deny the workspace-wide job enumeration routes `jobs:read` would
/// otherwise reach, so an embedded app reads only jobs it launched (by id).
pub const APP_EMBED_SENTINEL: &str = "app_embed";
/// True if a token's scopes include the app-embed sentinel (a sandboxed app iframe
/// token). Such tokens carry the viewer's identity but represent untrusted app JS,
/// so several handlers confine them to the app's own resources/runs.
pub fn has_app_embed_sentinel(scopes: Option<&[String]>) -> bool {
scopes.is_some_and(|s| s.iter().any(|x| x == APP_EMBED_SENTINEL))
}
/// Routes an app embed token (sentinel) is denied. Its broad scopes (`apps:run`,
/// `jobs:read`, `users:read`, `folders:read`) exist only for a fixed set of routes a
/// running app uses, but the whole `/apps`, `/jobs`, `/users`, `/folders` routers are
/// CORS-enabled for the opaque app iframe. Default-deny those domains via an explicit
/// allowlist so the token can't reach workspace inventory, counts, exports, or
/// capability-minting routes (job signatures / resume URLs).
fn app_embed_route_denied(domain: ScopeDomain, suffix: &str) -> bool {
match domain {
ScopeDomain::Apps => !app_embed_apps_route_allowed(suffix),
ScopeDomain::Jobs => !app_embed_job_route_allowed(suffix),
ScopeDomain::Users => suffix != "users/whoami",
ScopeDomain::Folders => suffix != "folders/listnames",
_ => false,
}
}
/// App routes a running app uses: its own definition (`apps/get/p/<path>`, further
/// path-scoped by `apps:read:<path>`) and the public app-serving endpoints
/// (`apps_u/*`: public_app, public_resource, get_data, and the path-taking
/// `execute_component` / `download_s3_file`, which re-check `apps:run|read:<path>`
/// in their handlers so they stay confined to this app). Everything else in the
/// domain — workspace app inventory (`exists`, `custom_path_exists`, `list`,
/// `list_paths*`, `secret_of`, history, management) — is denied.
fn app_embed_apps_route_allowed(suffix: &str) -> bool {
// The embed-token mint endpoints live under `apps_u/` but they create
// credentials. A running app never calls them — the trusted embedder session/JWT
// mints the token and hands it to the iframe — so deny them here, otherwise an
// app embed token could renew itself indefinitely past the 12h expiry.
if suffix.starts_with("apps_u/embed_token") {
return false;
}
suffix.starts_with("apps/get/p/") || suffix.starts_with("apps_u/")
}
/// Job routes a running app uses (the by-id poll/cancel surface driven by the
/// frontend JobLoader). Everything else in the jobs domain — enumeration, counts,
/// exports, and the `job_signature`/`resume_urls` capability-minting routes — is
/// denied. By-id reads are further confined to the app's own runs by
/// `require_job_read_access` (the `app_embed` cutoff).
fn app_embed_job_route_allowed(suffix: &str) -> bool {
// `get_root_job_id` is intentionally absent: its handler has no access check at
// all (returns any job's root id by id) and the app never calls it, so denying
// it costs nothing and avoids leaking a foreign job's flow lineage.
const ALLOWED: [&str; 15] = [
"jobs_u/get/",
"jobs_u/getupdate/",
"jobs_u/getupdate_sse/",
"jobs_u/get_logs/",
"jobs_u/get_completed_logs_tail/",
"jobs_u/get_args/",
"jobs_u/get_flow/",
"jobs_u/get_flow_all_logs/",
"jobs_u/get_flow_debug_info/",
"jobs_u/get_log_file/",
"jobs_u/completed/get/",
"jobs_u/completed/get_result/",
"jobs_u/completed/get_result_maybe/",
"jobs_u/completed/get_timing/",
"jobs_u/queue/cancel/",
];
ALLOWED.iter().any(|p| suffix.starts_with(p))
}
/// Resource routes a metadata-only `resources:run` scope (app embed tokens) may
/// GET: pickers (`/list`) and type schemas. Excludes every value-returning route
/// (`get`, `get_value`, `get_value_interpolated`, `list_search`) so resource
/// values — which can hold credentials — are never exposed.
fn resource_metadata_route_allowed(suffix: &str) -> bool {
suffix == "resources/list"
|| suffix.starts_with("resources/list_names/")
|| suffix.starts_with("resources/exists/")
|| suffix.starts_with("resources/type/")
}
fn scope_grants_access(
scope: &ScopeDefinition,
required_domain: ScopeDomain,
@@ -769,14 +656,6 @@ fn scope_grants_access(
let scope_action = ScopeAction::from_str(&scope.action)
.ok_or_else(|| Error::BadRequest(format!("Invalid scope action: {}", scope.action)))?;
// App embed tokens carry `resources:run`: metadata-only resource access via
// default-deny + allowlist (so a new value route is never exposed by accident).
// See `resource_metadata_route_allowed`.
if scope_domain == ScopeDomain::Resources && scope_action == ScopeAction::Run {
return Ok(required_action == ScopeAction::Read
&& route_path.is_some_and(resource_metadata_route_allowed));
}
if !scope_action.includes(&required_action)
&& !(scope_domain == ScopeDomain::Jobs
&& required_action == ScopeAction::Read
@@ -820,23 +699,6 @@ pub fn check_read_only_for_route(route_path: &str, http_method: &str) -> Result<
}
}
/// The minimal scope string that grants access to exactly `{method} {path}`, as
/// `check_route_access` would require it. Used to mint a least-privilege JWT for
/// a single proxied request (the MCP endpoint proxy), so the minted token can do
/// only that one operation rather than acting as a blank check.
///
/// `path` is the request path (e.g. `/api/w/{workspace}/variables/get/...`).
/// Returns `None` if the route's domain can't be determined — the caller should
/// then fail closed.
pub fn scope_for_route(method: &str, path: &str) -> Option<String> {
let action = map_http_method_to_action(method, path);
let (domain, kind, _suffix) = extract_domain_from_route(path).ok()?;
Some(match (domain, action, kind) {
(ScopeDomain::Jobs, ScopeAction::Run, Some(kind)) => format!("jobs:run:{}", kind),
(domain, action, _) => format!("{}:{}", domain.as_str(), action.as_str()),
})
}
/// Helper function to check if scopes allow access to a route
pub fn check_scopes_for_route(
token_scopes: Option<&[String]>,
@@ -927,12 +789,6 @@ mod tests {
assert_eq!(domain, ScopeDomain::FlowConversations);
assert_eq!(kind, None);
assert_eq!(route_suffix, Some("flow_conversations/list".to_string()));
let (domain, kind, route_suffix) =
extract_domain_from_route("/api/w/test_workspace/ai_skills/list").unwrap();
assert_eq!(domain, ScopeDomain::AiSkills);
assert_eq!(kind, None);
assert_eq!(route_suffix, Some("ai_skills/list".to_string()));
}
#[test]
@@ -989,10 +845,6 @@ mod tests {
ScopeDomain::from_str("flow_conversations"),
Some(ScopeDomain::FlowConversations)
);
assert_eq!(
ScopeDomain::from_str("ai_skills"),
Some(ScopeDomain::AiSkills)
);
// Test canonical string conversion
assert_eq!(ScopeDomain::Acls.as_str(), "acls");
@@ -1002,41 +854,6 @@ mod tests {
ScopeDomain::FlowConversations.as_str(),
"flow_conversations"
);
assert_eq!(ScopeDomain::AiSkills.as_str(), "ai_skills");
}
#[test]
fn test_ai_skills_scope_access() {
let read_scopes = vec!["ai_skills:read".to_string()];
assert!(
check_route_access(&read_scopes, "/api/w/test_workspace/ai_skills/list", "GET").is_ok()
);
assert!(check_route_access(
&read_scopes,
"/api/w/test_workspace/ai_skills/get/foo",
"GET"
)
.is_ok());
assert!(check_route_access(
&read_scopes,
"/api/w/test_workspace/ai_skills/upload",
"POST"
)
.is_err());
let write_scopes = vec!["ai_skills:write".to_string()];
assert!(check_route_access(
&write_scopes,
"/api/w/test_workspace/ai_skills/upload",
"POST"
)
.is_ok());
assert!(check_route_access(
&write_scopes,
"/api/w/test_workspace/ai_skills/delete/foo",
"DELETE"
)
.is_ok());
}
#[test]
@@ -1266,38 +1083,4 @@ mod tests {
let scopes = vec!["jobs:read".to_string(), "mcp:all".to_string()];
assert!(check_route_access(&scopes, "/api/w/test_workspace/mcp/something", "GET").is_ok());
}
#[test]
fn test_scope_for_route() {
// The minted scope must be exactly what check_route_access requires for
// the same route, so a JWT carrying it passes for that one route only.
assert_eq!(
scope_for_route("GET", "/api/w/ws/variables/get/u/x/y").as_deref(),
Some("variables:read")
);
assert_eq!(
scope_for_route("POST", "/api/w/ws/variables/create").as_deref(),
Some("variables:write")
);
assert_eq!(
scope_for_route("DELETE", "/api/w/ws/resources/delete/u/x/y").as_deref(),
Some("resources:write")
);
// jobs run paths carry the runnable kind.
assert_eq!(
scope_for_route("POST", "/api/w/ws/jobs/run/p/u/x/y").as_deref(),
Some("jobs:run:scripts")
);
assert_eq!(
scope_for_route("POST", "/api/w/ws/jobs/run/f/u/x/y").as_deref(),
Some("jobs:run:flows")
);
// The minted scope actually satisfies the route check it targets.
let s = scope_for_route("POST", "/api/w/ws/variables/create").unwrap();
assert!(check_route_access(&[s], "/api/w/ws/variables/create", "POST").is_ok());
// Unknown route -> None so the caller fails closed.
assert!(scope_for_route("GET", "/healthz").is_none());
}
}
-1
View File
@@ -26,7 +26,6 @@ windmill-parser-ts.workspace = true
windmill-parser.workspace = true
windmill-parser-ts-asset.workspace = true
windmill-parser-sql-asset.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-yaml.workspace = true
axum.workspace = true
@@ -1251,54 +1251,6 @@ async fn create_script_internal<'c>(
windmill_common::pipeline_advanced::freshness_enforcement_todo()
);
}
// `// materialize` materializes a `ducklake://<name>/<table>` target from a
// DuckDB script. These two constraints hold for *both* modes: a non-DuckLake
// target would otherwise deploy, register a producer in the asset graph, then
// silently no-op at run time (`build_materialized_query` returns `Ok(None)`),
// and a non-DuckDB script never reaches the executor that records state. The
// managed-only checks (single trailing SELECT, no SQL args) come after — a
// `manual` script owns its DDL and skips them.
if let Some(m) = pipeline_annotations.materialize.as_ref() {
if ns.language != ScriptLang::DuckDb {
return Err(Error::BadRequest(format!(
"`// materialize` is only supported for DuckDB scripts, not {}. Use the \
wmll.ducklake helpers to materialize from other languages.",
ns.language.as_str()
)));
}
if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake {
return Err(Error::BadRequest(
"`// materialize` only supports a DuckLake target \
(`ducklake://<name>/<table>`); other asset kinds aren't materializable."
.to_string(),
));
}
if !m.target_path.contains('/') {
return Err(Error::BadRequest(format!(
"`// materialize` needs a table in the target: \
`ducklake://{0}/<table>` (got `ducklake://{0}`).",
m.target_path
)));
}
if !m.manual {
if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) {
return Err(Error::BadRequest(e.message()));
}
// SQL args are supported: managed materialize strips line comments
// (including `-- $name (type)` declarations) when it wraps the SELECT,
// but the executor parses the signature from the un-wrapped script, so
// `$name` references in the SELECT stay bound at run time.
}
// `key=` (merge) and `append` are mutually exclusive reconciliation
// strategies; append (INSERT-only) wins. Surface the conflict rather
// than silently dropping the dedup the author may have intended.
if m.unique_key.is_some() && m.append {
tracing::warn!(
"script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)",
ns.path
);
}
}
let in_pipeline = pipeline_annotations.in_pipeline;
// `// trigger all` → AND join barrier (else OR, the default).
let pipeline_join_all = !pipeline_annotations.join_mode.is_any();
@@ -1338,26 +1290,6 @@ async fn create_script_internal<'c>(
&ns.content,
ns.assets.take(),
);
// Register the `// materialize` target as a write asset so the deployed
// asset graph shows this script as the producer of the managed table — the
// body's `SELECT` doesn't express the write (the runtime generates it), so
// server-side inference wouldn't otherwise link it.
let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() {
let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind);
let mut a = effective_assets.unwrap_or_default();
if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) {
a.push(windmill_common::assets::AssetWithAltAccessType {
path: m.target_path.clone(),
kind,
access_type: Some(windmill_common::assets::AssetUsageAccessType::W),
alt_access_type: None,
columns: None,
});
}
Some(a)
} else {
effective_assets
};
let auto_kind = if in_pipeline {
Some("pipeline".to_string())
} else if ci_test_refs.is_some() {
@@ -31,9 +31,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE};
use windmill_common::worker::WINDMILL_DIR;
use windmill_common::{DB, INSTANCE_NAME, JOB_RETENTION_SECS, SERVICE_LOG_RETENTION_SECS};
use windmill_object_store::object_store_reexports::{
ObjectStore, ObjectStoreError, Path as ObjectPath,
};
use windmill_object_store::object_store_reexports::{ObjectStore, Path as ObjectPath};
pub const TASK_NAME: &str = "log_cleanup";
@@ -63,10 +61,6 @@ pub struct LogCleanupProgress {
pub total_jobs: u64,
pub processed_jobs: u64,
pub s3_deleted: u64,
/// Number of delete calls that returned 404 (object already absent — a no-op
/// success). GCS returns 404 per missing key where S3's DeleteObjects stays silent.
#[serde(default)]
pub s3_not_found: u64,
/// Number of S3 objects inspected during the orphan scan phase.
pub orphans_scanned: u64,
/// Number of orphan S3 objects deleted (no corresponding DB row).
@@ -87,7 +81,6 @@ impl LogCleanupProgress {
total_jobs: 0,
processed_jobs: 0,
s3_deleted: 0,
s3_not_found: 0,
orphans_scanned: 0,
orphans_deleted: 0,
errors: 0,
@@ -139,13 +132,6 @@ impl Session {
p.phase = "done".to_string();
p.clone()
};
tracing::info!(
"log cleanup finished: {} object(s) deleted from object store, {} already absent (404), {} orphans deleted, {} error(s)",
snapshot.s3_deleted,
snapshot.s3_not_found,
snapshot.orphans_deleted,
snapshot.errors
);
if let Err(e) = background_task::release(&self.db, TASK_NAME, &self.owner, &snapshot).await
{
tracing::warn!("log cleanup: failed to release lease: {e:#}");
@@ -193,32 +179,21 @@ pub async fn get_status(db: &DB) -> error::Result<Option<LogCleanupProgress>> {
async fn s3_bulk_delete(
store: &Arc<dyn ObjectStore>,
paths: Vec<ObjectPath>,
) -> (
u64, /* deleted */
u64, /* not_found */
u64, /* errors */
) {
) -> (u64 /* deleted */, u64 /* errors */) {
let stream = futures::stream::iter(paths.into_iter().map(Ok)).boxed();
let mut deleted = 0u64;
let mut not_found = 0u64;
let mut errors = 0u64;
let mut res = store.delete_stream(stream);
while let Some(r) = res.next().await {
match r {
Ok(_) => deleted += 1,
// Deleting a non-existent object is a successful no-op. S3's DeleteObjects
// ignores missing keys, but GCS returns 404 per delete, surfacing as
// NotFound — track it separately so it isn't reported as an error.
Err(ObjectStoreError::NotFound { .. }) => {
not_found += 1;
}
Err(e) => {
errors += 1;
tracing::warn!("log cleanup: failed to delete object: {e:#}");
}
}
}
(deleted, not_found, errors)
(deleted, errors)
}
/// Delete the given relative paths from the local filesystem under `base_dir`.
@@ -290,7 +265,7 @@ async fn cleanup_service_logs(
.iter()
.map(|p| ObjectPath::from(format!("{}{}", LOGS_SERVICE, p)))
.collect();
let (deleted, not_found, errors) = s3_bulk_delete(store, s3_paths).await;
let (deleted, errors) = s3_bulk_delete(store, s3_paths).await;
disk_bulk_delete(&*TMP_WINDMILL_LOGS_SERVICE, &rel_paths).await;
session
@@ -300,7 +275,6 @@ async fn cleanup_service_logs(
p.total_service = p.processed_service;
}
p.s3_deleted = p.s3_deleted.saturating_add(deleted);
p.s3_not_found = p.s3_not_found.saturating_add(not_found);
p.errors = p.errors.saturating_add(errors);
})
.await;
@@ -351,7 +325,7 @@ async fn cleanup_job_logs(
.iter()
.map(|p| ObjectPath::from(p.clone()))
.collect();
let (deleted, not_found, errors) = s3_bulk_delete(store, s3_paths).await;
let (deleted, errors) = s3_bulk_delete(store, s3_paths).await;
disk_bulk_delete(&*WINDMILL_DIR, &rel_paths).await;
session
@@ -361,7 +335,6 @@ async fn cleanup_job_logs(
p.total_jobs = p.processed_jobs;
}
p.s3_deleted = p.s3_deleted.saturating_add(deleted);
p.s3_not_found = p.s3_not_found.saturating_add(not_found);
p.errors = p.errors.saturating_add(errors);
})
.await;
@@ -395,17 +368,13 @@ async fn delete_expired_jobs_batch(
.fetch_all(&mut *tx)
.await?;
// Active-root exclusion via NOT IN (hashed SubPlan) instead of `!= ALL($3)`;
// see backend/src/monitor.rs::delete_expired_jobs_batch for the rationale.
let deleted_jobs: Vec<Uuid> = sqlx::query_scalar!(
"DELETE FROM v2_job_completed
WHERE id IN (
SELECT jc.id FROM v2_job_completed jc
LEFT JOIN v2_job j ON j.id = jc.id
WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
)
AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) != ALL($3)
ORDER BY jc.completed_at ASC
LIMIT $2
FOR UPDATE OF jc SKIP LOCKED
@@ -592,12 +561,11 @@ async fn flush_service_orphans(
batch: &mut Vec<ObjectPath>,
) {
let paths = std::mem::take(batch);
let (deleted, not_found, errors) = s3_bulk_delete(store, paths).await;
let (deleted, errors) = s3_bulk_delete(store, paths).await;
session
.update(|p| {
p.orphans_deleted = p.orphans_deleted.saturating_add(deleted);
p.s3_deleted = p.s3_deleted.saturating_add(deleted);
p.s3_not_found = p.s3_not_found.saturating_add(not_found);
p.errors = p.errors.saturating_add(errors);
})
.await;
@@ -648,12 +616,11 @@ async fn flush_job_orphans(
return;
}
let (deleted, not_found, errors) = s3_bulk_delete(store, to_delete).await;
let (deleted, errors) = s3_bulk_delete(store, to_delete).await;
session
.update(|p| {
p.orphans_deleted = p.orphans_deleted.saturating_add(deleted);
p.s3_deleted = p.s3_deleted.saturating_add(deleted);
p.s3_not_found = p.s3_not_found.saturating_add(not_found);
p.errors = p.errors.saturating_add(errors);
})
.await;
+4 -796
View File
@@ -1,7 +1,7 @@
{
"openapi": "3.0.3",
"info": {
"version": "1.734.0",
"version": "1.728.0",
"title": "Windmill API",
"contact": {
"name": "Windmill Team",
@@ -9704,10 +9704,6 @@
"type": "string",
"description": "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side (client_credentials flow only). The token URL is never caller-supplied."
},
"cc_token_url": {
"type": "string",
"description": "Bring-your-own token endpoint override (client_credentials flow only). Only honored together with cc_client_id/cc_client_secret and mutually exclusive with cc_instance; ignored/rejected on the shared-instance path."
},
"mcp_server_url": {
"type": "string",
"description": "MCP server URL for MCP OAuth token refresh"
@@ -9789,10 +9785,6 @@
"cc_instance": {
"type": "string",
"description": "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side. The token URL is never caller-supplied."
},
"cc_token_url": {
"type": "string",
"description": "Bring-your-own token endpoint override. Only honored together with cc_client_id/cc_client_secret and mutually exclusive with cc_instance; rejected on the shared-instance path."
}
}
}
@@ -12196,14 +12188,6 @@
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "all_users",
"in": "query",
"description": "List every draft in the workspace (all users), not just the current user's own + legacy rows. Other users' rows come back with `mine=false` (view-only).",
"schema": {
"type": "boolean"
}
}
],
"responses": {
@@ -12241,27 +12225,6 @@
"created_at": {
"type": "string",
"format": "date-time"
},
"can_write": {
"type": "boolean",
"description": "Whether the current user may deploy/discard this draft (same check the deploy/discard endpoints enforce)."
},
"mine": {
"type": "boolean",
"description": "The row belongs to the current user (own draft or the legacy no-owner row) and is therefore actionable. Always true in the default listing; with `all_users=true`, other users' rows are false (view-only)."
},
"draft_users": {
"description": "Draft authors at this (path, kind) — the legacy NULL-email row surfaced as a null username.\nPopulated only for the shared full-page-editor kinds (script/flow/app/raw_app); omitted for\ndrawer kinds, which keep their drafts private. Feeds the Draft badge's owner-avatar circles.\n",
"type": "array",
"items": {
"type": "object",
"properties": {
"username": {
"type": "string",
"nullable": true
}
}
}
}
},
"required": [
@@ -12269,9 +12232,7 @@
"path",
"draft_only",
"legacy_draft",
"created_at",
"can_write",
"mine"
"created_at"
]
}
}
@@ -12341,55 +12302,6 @@
}
}
},
"/w/{workspace}/drafts/get_own/{kind}/{path}": {
"get": {
"summary": "fetch the current user's own draft content at a path (any kind)",
"operationId": "getOwnDraft",
"tags": [
"draft"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "kind",
"in": "path",
"required": true,
"schema": {
"$ref": "#/components/schemas/UserDraftItemKind"
}
},
{
"$ref": "#/components/parameters/ScriptPath"
}
],
"responses": {
"200": {
"description": "the user's draft content, or null when none exists",
"content": {
"application/json": {
"schema": {
"nullable": true,
"type": "object",
"properties": {
"value": {},
"created_at": {
"type": "string",
"format": "date-time"
}
},
"required": [
"value",
"created_at"
]
}
}
}
}
}
}
},
"/w/{workspace}/drafts/update/{kind}/{path}": {
"post": {
"summary": "upsert (or clear) the current user's draft at a path",
@@ -12436,11 +12348,6 @@
"legacy": {
"type": "boolean",
"description": "Delete-only. Target the legacy workspace-level row (email NULL) instead of the current user's row. Used to discard a legacy draft from the review page."
},
"created_at": {
"type": "string",
"format": "date-time",
"description": "Upsert-only override for the stored creation timestamp. Normal saves omit it (stamped server-side); the localStorage→DB migration passes the draft's original write time so migrated drafts keep their age."
}
}
}
@@ -12478,67 +12385,6 @@
}
}
},
"/w/{workspace}/drafts/migrate_legacy/{kind}/{path}": {
"post": {
"summary": "resolve a legacy (workspace-level) draft (admin only)",
"description": "Delete a legacy draft (email NULL) or assign it to the authed admin as a per-user draft. Workspace admins / superadmins only.",
"operationId": "migrateLegacyDraft",
"tags": [
"draft"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "kind",
"in": "path",
"required": true,
"schema": {
"$ref": "#/components/schemas/UserDraftItemKind"
}
},
{
"$ref": "#/components/parameters/ScriptPath"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"delete",
"assign_to_self"
],
"description": "delete the legacy draft, or take ownership of it."
}
},
"required": [
"action"
]
}
}
}
},
"responses": {
"200": {
"description": "migration result",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/w/{workspace}/scripts/create": {
"post": {
"summary": "create script",
@@ -16253,202 +16099,6 @@
}
}
},
"/w/{workspace}/ai_skills/list": {
"get": {
"summary": "list the workspace AI chat skills (name + description only)",
"operationId": "listAiSkills",
"tags": [
"workspace"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
}
],
"responses": {
"200": {
"description": "skill listing",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"required": [
"name",
"description"
],
"properties": {
"name": {
"type": "string"
},
"description": {
"type": "string"
}
}
}
}
}
}
}
}
}
},
"/w/{workspace}/ai_skills/get/{name}": {
"get": {
"summary": "get a workspace AI chat skill including its instructions",
"operationId": "getAiSkill",
"tags": [
"workspace"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "name",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "skill",
"content": {
"application/json": {
"schema": {
"type": "object",
"required": [
"name",
"description",
"instructions"
],
"properties": {
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"instructions": {
"type": "string"
}
}
}
}
}
}
}
}
},
"/w/{workspace}/ai_skills/upload": {
"post": {
"summary": "upsert workspace AI chat skills (admin only)",
"operationId": "uploadAiSkills",
"tags": [
"workspace"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"required": [
"skills"
],
"properties": {
"skills": {
"type": "array",
"maxItems": 50,
"items": {
"type": "object",
"required": [
"name",
"description",
"instructions"
],
"properties": {
"name": {
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[a-z0-9-]+$"
},
"description": {
"type": "string",
"minLength": 1,
"maxLength": 1024
},
"instructions": {
"type": "string",
"minLength": 1,
"maxLength": 65536
}
}
}
}
}
}
}
}
},
"responses": {
"200": {
"description": "uploaded",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/w/{workspace}/ai_skills/delete/{name}": {
"delete": {
"summary": "delete a workspace AI chat skill (admin only)",
"operationId": "deleteAiSkill",
"tags": [
"workspace"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "name",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "deleted",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/w/{workspace}/apps/get_data/v/{secretWithExtension}": {
"get": {
"summary": "get raw app data by",
@@ -20751,192 +20401,6 @@
}
}
},
"/w/{workspace}/jobs_u/dispatch_events/{id}": {
"get": {
"summary": "list asset-trigger dispatch events for a producer job",
"description": "Returns the chronological log of decisions the asset-trigger dispatcher made after this producer job completed. Each row is one (subscriber, asset write) decision: `dispatched` (with `child_job_id`), `join_pending` (with `received_inputs` / `required_inputs` / `partition`), or `skipped` (with `reason`). Rows are reaped automatically when the producer's `v2_job` row is deleted by the retention sweep.\n",
"operationId": "listDispatchEvents",
"tags": [
"job"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"$ref": "#/components/parameters/JobId"
}
],
"responses": {
"200": {
"description": "dispatch events for this producer job",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"subscriber_path": {
"type": "string"
},
"asset_kind": {
"type": "string",
"enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
},
"asset_path": {
"type": "string"
},
"outcome": {
"type": "string",
"enum": [
"dispatched",
"join_pending",
"skipped"
]
},
"child_job_id": {
"type": "string",
"format": "uuid"
},
"partition": {
"type": "string"
},
"received_inputs": {
"type": "integer"
},
"required_inputs": {
"type": "integer"
},
"debounce_s": {
"type": "integer"
},
"reason": {
"type": "string"
},
"created_at": {
"type": "string",
"format": "date-time"
}
},
"required": [
"subscriber_path",
"asset_kind",
"asset_path",
"outcome",
"created_at"
]
}
}
}
}
}
}
}
},
"/w/{workspace}/jobs/asset_dispatch_edges": {
"get": {
"summary": "list asset-cascade producer→child job edges for a folder",
"description": "Returns the `dispatched` asset-trigger edges (producer job → child job) whose subscriber lives under `path_start`. Lets a pipeline view reconstruct the cascade tree of a folder by job id and group connected runs. Visibility follows the producer job's RLS.\n",
"operationId": "listAssetDispatchEdges",
"tags": [
"job"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "path_start",
"in": "query",
"required": true,
"description": "Folder path prefix the children live under, e.g. `f/orders/`.",
"schema": {
"type": "string"
}
},
{
"name": "created_after",
"in": "query",
"required": false,
"description": "Only edges dispatched at/after this instant.",
"schema": {
"type": "string",
"format": "date-time"
}
}
],
"responses": {
"200": {
"description": "asset-cascade edges for the folder",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"producer_job_id": {
"type": "string",
"format": "uuid"
},
"child_job_id": {
"type": "string",
"format": "uuid",
"description": "Set for `dispatched`; absent for `join_pending` inputs."
},
"subscriber_path": {
"type": "string"
},
"outcome": {
"type": "string",
"enum": [
"dispatched",
"join_pending"
]
},
"asset_kind": {
"type": "string",
"enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
},
"asset_path": {
"type": "string"
},
"created_at": {
"type": "string",
"format": "date-time"
}
},
"required": [
"producer_job_id",
"subscriber_path",
"outcome",
"asset_kind",
"asset_path",
"created_at"
]
}
}
}
}
}
}
}
},
"/w/{workspace}/jobs/completed/delete/{id}": {
"post": {
"summary": "delete completed job (erase content but keep run id)",
@@ -21633,10 +21097,6 @@
}
}
}
},
"view_token": {
"type": "string",
"description": "Share-read-link token for the flow. An authenticated workspace member can append it as a `view_token` query param on the run page to read a flow they don't otherwise have access to."
}
}
}
@@ -22057,10 +21517,6 @@
"approver"
]
}
},
"view_token": {
"type": "string",
"description": "Share-read-link token for the parent flow. An authenticated workspace member can append it as a `view_token` query param on the run page to read a flow they don't otherwise have access to."
}
},
"required": [
@@ -33281,241 +32737,6 @@
}
}
},
"/w/{workspace}/assets/graph": {
"get": {
"summary": "Get the workspace-wide asset <-> runnable graph",
"operationId": "getAssetsGraph",
"tags": [
"asset"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
},
{
"name": "asset_kinds",
"in": "query",
"description": "Filter by asset kinds (comma-separated list)",
"schema": {
"type": "string"
}
},
{
"name": "folder",
"in": "query",
"description": "Scope the graph to runnables in a single folder",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "asset graph nodes, lineage edges and trigger edges",
"content": {
"application/json": {
"schema": {
"type": "object",
"required": [
"assets",
"runnables",
"edges",
"triggers"
],
"properties": {
"assets": {
"type": "array",
"items": {
"type": "object",
"required": [
"kind",
"path"
],
"properties": {
"kind": {
"$ref": "#/components/schemas/AssetKind"
},
"path": {
"type": "string"
}
}
}
},
"runnables": {
"type": "array",
"items": {
"type": "object",
"required": [
"path",
"usage_kind"
],
"properties": {
"path": {
"type": "string"
},
"usage_kind": {
"$ref": "#/components/schemas/AssetUsageKind"
},
"in_pipeline": {
"type": "boolean",
"description": "True iff the script is a pipeline member (deployed with `// pipeline`). Omitted when false."
}
}
}
},
"edges": {
"type": "array",
"items": {
"type": "object",
"required": [
"runnable_path",
"runnable_kind",
"asset_kind",
"asset_path"
],
"properties": {
"runnable_path": {
"type": "string"
},
"runnable_kind": {
"$ref": "#/components/schemas/AssetUsageKind"
},
"asset_kind": {
"$ref": "#/components/schemas/AssetKind"
},
"asset_path": {
"type": "string"
},
"access_type": {
"$ref": "#/components/schemas/AssetUsageAccessType"
}
}
}
},
"triggers": {
"type": "array",
"items": {
"oneOf": [
{
"type": "object",
"description": "Asset trigger edge (`// on <asset>`)",
"required": [
"trigger_kind",
"asset_kind",
"asset_path",
"runnable_kind",
"runnable_path"
],
"properties": {
"trigger_kind": {
"type": "string",
"enum": [
"asset"
]
},
"asset_kind": {
"$ref": "#/components/schemas/AssetKind"
},
"asset_path": {
"type": "string"
},
"runnable_kind": {
"$ref": "#/components/schemas/AssetUsageKind"
},
"runnable_path": {
"type": "string"
}
}
},
{
"type": "object",
"description": "Native trigger edge (schedule, email, kafka, ...). `path` is the trigger row's path.",
"required": [
"trigger_kind",
"path",
"runnable_kind",
"runnable_path"
],
"properties": {
"trigger_kind": {
"type": "string",
"enum": [
"schedule",
"email",
"kafka",
"mqtt",
"nats",
"postgres",
"sqs",
"gcp"
]
},
"path": {
"type": "string"
},
"runnable_kind": {
"$ref": "#/components/schemas/AssetUsageKind"
},
"runnable_path": {
"type": "string"
}
}
}
]
}
}
}
}
}
}
}
}
}
},
"/w/{workspace}/assets/pipelines": {
"get": {
"summary": "List folders that contain at least one pipeline-member script",
"operationId": "listPipelineFolders",
"tags": [
"asset"
],
"parameters": [
{
"$ref": "#/components/parameters/WorkspaceId"
}
],
"responses": {
"200": {
"description": "folders containing pipeline scripts, with their script counts",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"required": [
"folder",
"script_count"
],
"properties": {
"folder": {
"type": "string",
"description": "The folder name (without the `f/` prefix)"
},
"script_count": {
"type": "integer",
"format": "int64",
"description": "Number of pipeline-member scripts in the folder"
}
}
}
}
}
}
}
}
}
},
"/w/{workspace}/volumes/list": {
"get": {
"summary": "List all volumes in the workspace",
@@ -34465,8 +33686,7 @@
"trigger_cli",
"trigger_nextcloud",
"trigger_google",
"trigger_github",
"data_pipeline"
"trigger_github"
]
},
"OpenFlow": {
@@ -37088,10 +36308,6 @@
"parent_hash": {
"type": "string"
},
"auto_parent": {
"type": "boolean",
"description": "When true, the backend resolves the parent to the current deployed head for this path within the transaction (ignoring parent_hash), instead of failing with a \"lineage must be linear\" error when the supplied parent_hash is stale."
},
"summary": {
"type": "string"
},
@@ -38157,12 +37373,6 @@
"type": "string"
}
},
"folders_read": {
"type": "array",
"items": {
"type": "string"
}
},
"folders_owners": {
"type": "array",
"items": {
@@ -38190,7 +37400,6 @@
"operator",
"disabled",
"folders",
"folders_read",
"folders_owners"
]
},
@@ -40094,8 +39303,7 @@
"gcp",
"azure",
"google",
"github",
"asset"
"github"
]
},
"TriggerMode": {
+1 -641
View File
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 1.734.0
version: 1.728.0
title: Windmill API
contact:
name: Windmill Team
@@ -749,10 +749,6 @@ paths:
type: array
items:
type: string
folders_read:
type: array
items:
type: string
folders_owners:
type: array
items:
@@ -792,7 +788,6 @@ paths:
- operator
- disabled
- folders
- folders_read
- folders_owners
/w/{workspace}/users/update/{username}:
post:
@@ -8923,13 +8918,6 @@ paths:
substituted into the fixed-host registry template
server-side (client_credentials flow only). The token URL is
never caller-supplied.
cc_token_url:
type: string
description: >-
Bring-your-own token endpoint override (client_credentials
flow only). Only honored together with
cc_client_id/cc_client_secret and mutually exclusive with
cc_instance; ignored/rejected on the shared-instance path.
mcp_server_url:
type: string
description: MCP server URL for MCP OAuth token refresh
@@ -8997,13 +8985,6 @@ paths:
client-credentials token URL is instance-templated;
substituted into the fixed-host registry template
server-side. The token URL is never caller-supplied.
cc_token_url:
type: string
description: >-
Bring-your-own token endpoint override. Only honored
together with cc_client_id/cc_client_secret and mutually
exclusive with cc_instance; rejected on the shared-instance
path.
responses:
'200':
description: OAuth token response
@@ -12727,14 +12708,6 @@ paths:
in: path
required: true
schema: *ref_4
- name: all_users
in: query
description: >-
List every draft in the workspace (all users), not just the current
user's own + legacy rows. Other users' rows come back with
`mine=false` (view-only).
schema:
type: boolean
responses:
'200':
description: the user's drafts
@@ -12778,7 +12751,6 @@ paths:
- trigger_nextcloud
- trigger_google
- trigger_github
- data_pipeline
path:
type: string
summary:
@@ -12807,43 +12779,12 @@ paths:
created_at:
type: string
format: date-time
can_write:
type: boolean
description: >-
Whether the current user may deploy/discard this draft
(same check the deploy/discard endpoints enforce).
mine:
type: boolean
description: >-
The row belongs to the current user (own draft or the
legacy no-owner row) and is therefore actionable. Always
true in the default listing; with `all_users=true`,
other users' rows are false (view-only).
draft_users:
description: >
Draft authors at this (path, kind) — the legacy
NULL-email row surfaced as a null username.
Populated only for the shared full-page-editor kinds
(script/flow/app/raw_app); omitted for
drawer kinds, which keep their drafts private. Feeds the
Draft badge's owner-avatar circles.
type: array
items:
type: object
properties:
username:
type: string
nullable: true
required:
- kind
- path
- draft_only
- legacy_draft
- created_at
- can_write
- mine
/w/{workspace}/drafts/get/{kind}/{path}:
get:
summary: >-
@@ -12897,48 +12838,6 @@ paths:
- created_at
'404':
description: no draft for that owner at that path
/w/{workspace}/drafts/get_own/{kind}/{path}:
get:
summary: fetch the current user's own draft content at a path (any kind)
operationId: getOwnDraft
tags:
- draft
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: kind
in: path
required: true
schema:
type: string
description: >
Closed set of item kinds a user can autosave as a draft. Mirrors
the
Postgres `DRAFT_KIND` enum and the backend `UserDraftItemKind`.
enum: *ref_100
- name: path
in: path
required: true
schema: *ref_97
responses:
'200':
description: the user's draft content, or null when none exists
content:
application/json:
schema:
nullable: true
type: object
properties:
value: {}
created_at:
type: string
format: date-time
required:
- value
- created_at
/w/{workspace}/drafts/update/{kind}/{path}:
post:
summary: upsert (or clear) the current user's draft at a path
@@ -12992,14 +12891,6 @@ paths:
Delete-only. Target the legacy workspace-level row (email
NULL) instead of the current user's row. Used to discard a
legacy draft from the review page.
created_at:
type: string
format: date-time
description: >-
Upsert-only override for the stored creation timestamp.
Normal saves omit it (stamped server-side); the
localStorage→DB migration passes the draft's original write
time so migrated drafts keep their age.
responses:
'200':
description: save result
@@ -13019,57 +12910,6 @@ paths:
required:
- status
- current_timestamp
/w/{workspace}/drafts/migrate_legacy/{kind}/{path}:
post:
summary: resolve a legacy (workspace-level) draft (admin only)
description: >-
Delete a legacy draft (email NULL) or assign it to the authed admin as a
per-user draft. Workspace admins / superadmins only.
operationId: migrateLegacyDraft
tags:
- draft
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: kind
in: path
required: true
schema:
type: string
description: >
Closed set of item kinds a user can autosave as a draft. Mirrors
the
Postgres `DRAFT_KIND` enum and the backend `UserDraftItemKind`.
enum: *ref_100
- name: path
in: path
required: true
schema: *ref_97
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
action:
type: string
enum:
- delete
- assign_to_self
description: delete the legacy draft, or take ownership of it.
required:
- action
responses:
'200':
description: migration result
content:
text/plain:
schema:
type: string
/w/{workspace}/scripts/create:
post:
summary: create script
@@ -13118,13 +12958,6 @@ paths:
type: string
parent_hash:
type: string
auto_parent:
type: boolean
description: >-
When true, the backend resolves the parent to the current
deployed head for this path within the transaction (ignoring
parent_hash), instead of failing with a "lineage must be
linear" error when the supplied parent_hash is stale.
summary:
type: string
description:
@@ -16767,141 +16600,6 @@ paths:
text/plain:
schema:
type: string
/w/{workspace}/ai_skills/list:
get:
summary: list the workspace AI chat skills (name + description only)
operationId: listAiSkills
tags:
- workspace
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
responses:
'200':
description: skill listing
content:
application/json:
schema:
type: array
items:
type: object
required:
- name
- description
properties:
name:
type: string
description:
type: string
/w/{workspace}/ai_skills/get/{name}:
get:
summary: get a workspace AI chat skill including its instructions
operationId: getAiSkill
tags:
- workspace
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: name
in: path
required: true
schema:
type: string
responses:
'200':
description: skill
content:
application/json:
schema:
type: object
required:
- name
- description
- instructions
properties:
name:
type: string
description:
type: string
instructions:
type: string
/w/{workspace}/ai_skills/upload:
post:
summary: upsert workspace AI chat skills (admin only)
operationId: uploadAiSkills
tags:
- workspace
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- skills
properties:
skills:
type: array
maxItems: 50
items:
type: object
required:
- name
- description
- instructions
properties:
name:
type: string
minLength: 1
maxLength: 64
pattern: ^[a-z0-9-]+$
description:
type: string
minLength: 1
maxLength: 1024
instructions:
type: string
minLength: 1
maxLength: 65536
responses:
'200':
description: uploaded
content:
text/plain:
schema:
type: string
/w/{workspace}/ai_skills/delete/{name}:
delete:
summary: delete a workspace AI chat skill (admin only)
operationId: deleteAiSkill
tags:
- workspace
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: name
in: path
required: true
schema:
type: string
responses:
'200':
description: deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/apps/get_data/v/{secretWithExtension}:
get:
summary: get raw app data by
@@ -20275,7 +19973,6 @@ paths:
- azure
- google
- github
- asset
- name: trigger_path
description: The path of the trigger (can contain forward slashes)
in: path
@@ -21841,154 +21538,6 @@ paths:
type: integer
required:
- created_at
/w/{workspace}/jobs_u/dispatch_events/{id}:
get:
summary: list asset-trigger dispatch events for a producer job
description: >
Returns the chronological log of decisions the asset-trigger dispatcher
made after this producer job completed. Each row is one (subscriber,
asset write) decision: `dispatched` (with `child_job_id`),
`join_pending` (with `received_inputs` / `required_inputs` /
`partition`), or `skipped` (with `reason`). Rows are reaped
automatically when the producer's `v2_job` row is deleted by the
retention sweep.
operationId: listDispatchEvents
tags:
- job
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: id
in: path
required: true
schema: *ref_176
responses:
'200':
description: dispatch events for this producer job
content:
application/json:
schema:
type: array
items:
type: object
properties:
subscriber_path:
type: string
asset_kind:
type: string
enum:
- s3object
- resource
- variable
- ducklake
- datatable
- volume
asset_path:
type: string
outcome:
type: string
enum:
- dispatched
- join_pending
- skipped
child_job_id:
type: string
format: uuid
partition:
type: string
received_inputs:
type: integer
required_inputs:
type: integer
debounce_s:
type: integer
reason:
type: string
created_at:
type: string
format: date-time
required:
- subscriber_path
- asset_kind
- asset_path
- outcome
- created_at
/w/{workspace}/jobs/asset_dispatch_edges:
get:
summary: list asset-cascade producer→child job edges for a folder
description: >
Returns the `dispatched` asset-trigger edges (producer job → child job)
whose subscriber lives under `path_start`. Lets a pipeline view
reconstruct the cascade tree of a folder by job id and group connected
runs. Visibility follows the producer job's RLS.
operationId: listAssetDispatchEdges
tags:
- job
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: path_start
in: query
required: true
description: Folder path prefix the children live under, e.g. `f/orders/`.
schema:
type: string
- name: created_after
in: query
required: false
description: Only edges dispatched at/after this instant.
schema:
type: string
format: date-time
responses:
'200':
description: asset-cascade edges for the folder
content:
application/json:
schema:
type: array
items:
type: object
properties:
producer_job_id:
type: string
format: uuid
child_job_id:
type: string
format: uuid
description: Set for `dispatched`; absent for `join_pending` inputs.
subscriber_path:
type: string
outcome:
type: string
enum:
- dispatched
- join_pending
asset_kind:
type: string
enum:
- s3object
- resource
- variable
- ducklake
- datatable
- volume
asset_path:
type: string
created_at:
type: string
format: date-time
required:
- producer_job_id
- subscriber_path
- outcome
- asset_kind
- asset_path
- created_at
/w/{workspace}/jobs/completed/delete/{id}:
post:
summary: delete completed job (erase content but keep run id)
@@ -22491,13 +22040,6 @@ paths:
type: integer
approver:
type: string
view_token:
type: string
description: >-
Share-read-link token for the flow. An authenticated
workspace member can append it as a `view_token` query
param on the run page to read a flow they don't otherwise
have access to.
/w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}:
get:
summary: resume a job for a suspended flow
@@ -22798,13 +22340,6 @@ paths:
required:
- resume_id
- approver
view_token:
type: string
description: >-
Share-read-link token for the parent flow. An
authenticated workspace member can append it as a
`view_token` query param on the run page to read a flow
they don't otherwise have access to.
required:
- job
- approvers
@@ -35307,181 +34842,6 @@ paths:
path:
type: string
description: The asset path
/w/{workspace}/assets/graph:
get:
summary: Get the workspace-wide asset <-> runnable graph
operationId: getAssetsGraph
tags:
- asset
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
- name: asset_kinds
in: query
description: Filter by asset kinds (comma-separated list)
schema:
type: string
- name: folder
in: query
description: Scope the graph to runnables in a single folder
schema:
type: string
responses:
'200':
description: asset graph nodes, lineage edges and trigger edges
content:
application/json:
schema:
type: object
required:
- assets
- runnables
- edges
- triggers
properties:
assets:
type: array
items:
type: object
required:
- kind
- path
properties:
kind:
type: string
enum: *ref_306
path:
type: string
runnables:
type: array
items:
type: object
required:
- path
- usage_kind
properties:
path:
type: string
usage_kind:
type: string
enum: *ref_308
in_pipeline:
type: boolean
description: >-
True iff the script is a pipeline member (deployed
with `// pipeline`). Omitted when false.
edges:
type: array
items:
type: object
required:
- runnable_path
- runnable_kind
- asset_kind
- asset_path
properties:
runnable_path:
type: string
runnable_kind:
type: string
enum: *ref_308
asset_kind:
type: string
enum: *ref_306
asset_path:
type: string
access_type:
type: string
enum: *ref_307
nullable: true
triggers:
type: array
items:
oneOf:
- type: object
description: Asset trigger edge (`// on <asset>`)
required:
- trigger_kind
- asset_kind
- asset_path
- runnable_kind
- runnable_path
properties:
trigger_kind:
type: string
enum:
- asset
asset_kind:
type: string
enum: *ref_306
asset_path:
type: string
runnable_kind:
type: string
enum: *ref_308
runnable_path:
type: string
- type: object
description: >-
Native trigger edge (schedule, email, kafka, ...).
`path` is the trigger row's path.
required:
- trigger_kind
- path
- runnable_kind
- runnable_path
properties:
trigger_kind:
type: string
enum:
- schedule
- email
- kafka
- mqtt
- nats
- postgres
- sqs
- gcp
path:
type: string
runnable_kind:
type: string
enum: *ref_308
runnable_path:
type: string
/w/{workspace}/assets/pipelines:
get:
summary: List folders that contain at least one pipeline-member script
operationId: listPipelineFolders
tags:
- asset
parameters:
- name: workspace
in: path
required: true
schema: *ref_4
responses:
'200':
description: folders containing pipeline scripts, with their script counts
content:
application/json:
schema:
type: array
items:
type: object
required:
- folder
- script_count
properties:
folder:
type: string
description: The folder name (without the `f/` prefix)
script_count:
type: integer
format: int64
description: Number of pipeline-member scripts in the folder
/w/{workspace}/volumes/list:
get:
summary: List all volumes in the workspace
+1 -250
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.737.0
version: 1.733.1
title: Windmill API
contact:
@@ -1655,9 +1655,6 @@ paths:
s3_deleted:
type: integer
format: int64
s3_not_found:
type: integer
format: int64
orphans_scanned:
type: integer
format: int64
@@ -6294,9 +6291,6 @@ paths:
cc_instance:
type: string
description: "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side (client_credentials flow only). The token URL is never caller-supplied."
cc_token_url:
type: string
description: "Bring-your-own token endpoint override (client_credentials flow only). Only honored together with cc_client_id/cc_client_secret and mutually exclusive with cc_instance; ignored/rejected on the shared-instance path."
mcp_server_url:
type: string
description: "MCP server URL for MCP OAuth token refresh"
@@ -6352,9 +6346,6 @@ paths:
cc_instance:
type: string
description: "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side. The token URL is never caller-supplied."
cc_token_url:
type: string
description: "Bring-your-own token endpoint override. Only honored together with cc_client_id/cc_client_secret and mutually exclusive with cc_instance; rejected on the shared-instance path."
responses:
"200":
description: OAuth token response
@@ -7501,22 +7492,6 @@ paths:
workspace_id:
type: string
/apps_u/embed_token_by_custom_path/{custom_path}:
get:
summary: get app embed token by custom path
operationId: getAppEmbedTokenByCustomPath
tags:
- app
parameters:
- $ref: "#/components/parameters/CustomPath"
responses:
"200":
description: embed token
content:
application/json:
schema:
$ref: "#/components/schemas/EmbedTokenResponse"
/scripts/hub/get/{path}:
get:
summary: get hub script content by path
@@ -10447,133 +10422,6 @@ paths:
schema:
type: string
/w/{workspace}/ai_skills/list:
get:
summary: list the workspace AI chat skills (name + description only)
operationId: listAiSkills
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: skill listing
content:
application/json:
schema:
type: array
items:
type: object
required:
- name
- description
properties:
name:
type: string
description:
type: string
/w/{workspace}/ai_skills/get/{name}:
get:
summary: get a workspace AI chat skill including its instructions
operationId: getAiSkill
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: name
in: path
required: true
schema:
type: string
responses:
"200":
description: skill
content:
application/json:
schema:
type: object
required:
- name
- description
- instructions
properties:
name:
type: string
description:
type: string
instructions:
type: string
/w/{workspace}/ai_skills/upload:
post:
summary: upsert workspace AI chat skills (admin only)
operationId: uploadAiSkills
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- skills
properties:
skills:
type: array
maxItems: 50
items:
type: object
required:
- name
- description
- instructions
properties:
name:
type: string
minLength: 1
maxLength: 64
pattern: "^[a-z0-9-]+$"
description:
type: string
minLength: 1
maxLength: 1024
instructions:
type: string
minLength: 1
maxLength: 65536
responses:
"200":
description: uploaded
content:
text/plain:
schema:
type: string
/w/{workspace}/ai_skills/delete/{name}:
delete:
summary: delete a workspace AI chat skill (admin only)
operationId: deleteAiSkill
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: name
in: path
required: true
schema:
type: string
responses:
"200":
description: deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/apps/get_data/v/{secretWithExtension}:
get:
summary: get raw app data by
@@ -10585,10 +10433,6 @@ paths:
- name: secretWithExtension
in: path
required: true
description: >-
App version secret suffixed with the requested file type extension.
Supported extensions are `.js` (JavaScript bundle), `.css`
(stylesheet), and `.html` (sandboxed wrapper document).
schema:
type: string
responses:
@@ -10598,12 +10442,6 @@ paths:
text/javascript:
schema:
type: string
text/css:
schema:
type: string
text/html:
schema:
type: string
/w/{workspace}/apps/list_search:
get:
@@ -10855,23 +10693,6 @@ paths:
- $ref: "#/components/schemas/AppWithLastVersion"
- $ref: "#/components/schemas/UserDraftOverlay"
/w/{workspace}/apps/embed_token/p/{path}:
get:
summary: get app embed token by path
operationId: getAppEmbedTokenByPath
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: embed token
content:
application/json:
schema:
$ref: "#/components/schemas/EmbedTokenResponse"
/w/{workspace}/apps/get/lite/{path}:
get:
summary: get app lite by path
@@ -10989,27 +10810,6 @@ paths:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps_u/embed_token/{secret}:
get:
summary: get app embed token by secret
operationId: getAppEmbedTokenBySecret
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: secret
in: path
required: true
schema:
type: string
responses:
"200":
description: embed token
content:
application/json:
schema:
$ref: "#/components/schemas/EmbedTokenResponse"
/w/{workspace}/apps_u/public_resource/{path}:
get:
summary: get public resource
@@ -22776,13 +22576,6 @@ components:
type: string
parent_hash:
type: string
auto_parent:
type: boolean
description: >-
When true, the backend resolves the parent to the current deployed
head for this path within the transaction (ignoring parent_hash),
instead of failing with a "lineage must be linear" error when the
supplied parent_hash is stale.
summary:
type: string
description:
@@ -23558,10 +23351,6 @@ components:
type: array
items:
type: string
folders_read:
type: array
items:
type: string
folders_owners:
type: array
items:
@@ -23581,7 +23370,6 @@ components:
- operator
- disabled
- folders
- folders_read
- folders_owners
UserSource:
@@ -27897,13 +27685,6 @@ components:
type: string
on_behalf_of_email:
type: string
sandbox:
type: boolean
description: >
Publisher opt-in to app sandbox isolation (alpha). When true the app
is isolated from each viewer's Windmill session. When false/absent
the app runs same-origin with the viewer's full session (the
default, pre-isolation behavior).
ListableApp:
type: object
@@ -28123,36 +27904,6 @@ components:
required:
- version
EmbedTokenResponse:
type: object
properties:
token:
type: string
nullable: true
description: Narrowly-scoped embed token for the iframe. Absent for fully anonymous or raw apps, which load without a scoped token.
expiration:
type: string
format: date-time
nullable: true
description: Expiration of the embed token.
raw_app:
type: boolean
description: Raw apps render single-iframe and skip the opaque-viewer indirection and the embed token entirely.
sandbox:
type: boolean
description: Publisher opted this app into sandbox isolation. When false the viewer runs the app same-origin with its full session.
app_path:
type: string
nullable: true
description: The resolved app path; the embedder uses it to scope the app's backing localStorage per app.
workspace_id:
type: string
nullable: true
description: The resolved workspace; pairs with app_path so apps at the same path in different workspaces don't share a localStorage store.
required:
- raw_app
- sandbox
FlowVersion:
type: object
properties:
-394
View File
@@ -1,394 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2026
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::db::{ApiAuthed, DB};
use std::collections::HashSet;
use axum::{
extract::{Extension, Json, Path},
routing::{delete, get, post},
Router,
};
use serde::{Deserialize, Serialize};
use windmill_audit::audit_oss::audit_log;
use windmill_audit::ActionKind;
use windmill_common::{
db::UserDB,
error::{Error, JsonResult, Result},
utils::require_admin,
};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_skills))
.route("/get/{name}", get(get_skill))
.route("/upload", post(upload_skills))
.route("/delete/{name}", delete(delete_skill))
}
/// Cheap listing surfaced in the AI chat system prompt — no `instructions` body.
#[derive(Serialize)]
pub struct SkillListItem {
pub name: String,
pub description: String,
}
/// Full skill, including the SKILL.md body, fetched on demand by `read_skill`.
#[derive(Serialize)]
pub struct Skill {
pub name: String,
pub description: String,
pub instructions: String,
}
#[derive(Deserialize)]
pub struct UploadSkills {
pub skills: Vec<SkillUpload>,
}
#[derive(Deserialize)]
pub struct SkillUpload {
pub name: String,
pub description: String,
pub instructions: String,
}
const MAX_SKILLS_PER_UPLOAD: usize = 50;
// Every stored skill's name + description is advertised in the global AI chat
// system prompt, so bound the total a workspace can accumulate across uploads.
const MAX_SKILLS_PER_WORKSPACE: usize = 100;
// `name` and `description` follow the Claude SKILL.md spec
// (https://platform.claude.com/docs/en/agents-and-tools/agent-skills): both are
// loaded into the AI chat system prompt and `name` is the model-facing skill id,
// so matching the upstream limits keeps skills portable with Claude Code.
const MAX_SKILL_NAME_CHARS: usize = 64;
const MAX_SKILL_DESCRIPTION_CHARS: usize = 1_024;
// Not a spec field — a payload bound on the SKILL.md body, so measured in bytes.
const MAX_SKILL_INSTRUCTIONS_BYTES: usize = 64 * 1024;
fn validate_skill(skill: &SkillUpload) -> Result<()> {
let name = skill.name.trim();
if name.is_empty() || name.chars().count() > MAX_SKILL_NAME_CHARS {
return Err(Error::BadRequest(format!(
"skill name must be between 1 and {MAX_SKILL_NAME_CHARS} characters, got {:?}",
skill.name
)));
}
if !name
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
{
return Err(Error::BadRequest(format!(
"skill name {name:?} must only contain lowercase letters, digits or '-'"
)));
}
if skill.description.trim().is_empty() {
return Err(Error::BadRequest(format!(
"skill {name:?} is missing a description (the SKILL.md frontmatter `description`)"
)));
}
if skill.description.chars().count() > MAX_SKILL_DESCRIPTION_CHARS {
return Err(Error::BadRequest(format!(
"skill {name:?} description must be at most {MAX_SKILL_DESCRIPTION_CHARS} characters"
)));
}
if skill.instructions.trim().is_empty() {
return Err(Error::BadRequest(format!(
"skill {name:?} has an empty SKILL.md body"
)));
}
if skill.instructions.len() > MAX_SKILL_INSTRUCTIONS_BYTES {
return Err(Error::BadRequest(format!(
"skill {name:?} instructions must be at most {MAX_SKILL_INSTRUCTIONS_BYTES} bytes"
)));
}
Ok(())
}
/// Collect the trimmed skill names, rejecting duplicates within a single upload.
/// The insert upserts by name, so a duplicate would silently keep only the last
/// and make the reported/audited count wrong.
fn collect_upload_names(skills: &[SkillUpload]) -> Result<Vec<String>> {
let mut names = Vec::with_capacity(skills.len());
let mut seen = HashSet::with_capacity(skills.len());
for skill in skills {
let name = skill.name.trim().to_string();
if !seen.insert(name.clone()) {
return Err(Error::BadRequest(format!(
"duplicate skill name {name:?} in upload"
)));
}
names.push(name);
}
Ok(names)
}
/// Reject an upload that would push the workspace past `MAX_SKILLS_PER_WORKSPACE`.
/// Uploads upsert, so names already present (`replacing`) don't count as new.
fn check_workspace_skill_capacity(
existing_total: i64,
replacing: i64,
upload_count: usize,
) -> Result<()> {
let new_count = upload_count as i64 - replacing;
if existing_total + new_count > MAX_SKILLS_PER_WORKSPACE as i64 {
return Err(Error::BadRequest(format!(
"workspace cannot store more than {MAX_SKILLS_PER_WORKSPACE} skills"
)));
}
Ok(())
}
async fn list_skills(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<SkillListItem>> {
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query!(
"SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name",
&w_id
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(
rows.into_iter()
.map(|r| SkillListItem { name: r.name, description: r.description })
.collect(),
))
}
async fn get_skill(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> JsonResult<Skill> {
let mut tx = user_db.begin(&authed).await?;
let row = sqlx::query!(
"SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2",
&w_id,
&name
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
row.map(|r| {
Json(Skill { name: r.name, description: r.description, instructions: r.instructions })
})
.ok_or_else(|| Error::NotFound(format!("no skill named {name:?} in workspace {w_id}")))
}
/// Bulk upsert the uploaded skills by name. Existing skills not in the payload
/// are left untouched — removal goes through `delete_skill`.
async fn upload_skills(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(payload): Json<UploadSkills>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
if payload.skills.is_empty() {
return Err(Error::BadRequest("no skills to upload".to_string()));
}
if payload.skills.len() > MAX_SKILLS_PER_UPLOAD {
return Err(Error::BadRequest(format!(
"cannot upload more than {MAX_SKILLS_PER_UPLOAD} skills at a time"
)));
}
for skill in &payload.skills {
validate_skill(skill)?;
}
let names = collect_upload_names(&payload.skills)?;
let mut tx = db.begin().await?;
let counts = sqlx::query!(
r#"SELECT
COUNT(*)::bigint AS "total!",
COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS "replacing!"
FROM ai_skill
WHERE workspace_id = $1"#,
&w_id,
&names
)
.fetch_one(&mut *tx)
.await?;
check_workspace_skill_capacity(counts.total, counts.replacing, names.len())?;
for (skill, name) in payload.skills.iter().zip(names.iter()) {
sqlx::query!(
r#"INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by)
VALUES ($1, $2, $3, $4, now(), $5)
ON CONFLICT (workspace_id, name) DO UPDATE
SET description = EXCLUDED.description,
instructions = EXCLUDED.instructions,
edited_at = now(),
edited_by = EXCLUDED.edited_by"#,
&w_id,
name,
skill.description,
skill.instructions,
&authed.username,
)
.execute(&mut *tx)
.await?;
}
let audit_resource = names.join(",");
audit_log(
&mut *tx,
&authed,
"ai_skills.upload",
ActionKind::Update,
&w_id,
Some(&audit_resource),
Some([("skill_count", &names.len().to_string()[..])].into()),
)
.await?;
tx.commit().await?;
Ok(format!(
"Uploaded {} skill(s) to workspace {}",
payload.skills.len(),
&w_id
))
}
async fn delete_skill(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, name)): Path<(String, String)>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = db.begin().await?;
let deleted = sqlx::query_scalar!(
"DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name",
&w_id,
&name
)
.fetch_optional(&mut *tx)
.await?;
if deleted.is_none() {
tx.commit().await?;
return Err(Error::NotFound(format!(
"no skill named {name:?} in workspace {w_id}"
)));
}
audit_log(
&mut *tx,
&authed,
"ai_skills.delete",
ActionKind::Delete,
&w_id,
Some(&name),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Deleted skill {name} from workspace {w_id}"))
}
#[cfg(test)]
mod tests {
use super::*;
fn skill() -> SkillUpload {
SkillUpload {
name: "test-skill".to_string(),
description: "Useful for tests".to_string(),
instructions: "# Test\n\nDo the thing.".to_string(),
}
}
#[test]
fn validate_skill_rejects_oversized_description() {
let mut skill = skill();
skill.description = "x".repeat(MAX_SKILL_DESCRIPTION_CHARS + 1);
assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_))));
}
#[test]
fn validate_skill_rejects_oversized_instructions() {
let mut skill = skill();
skill.instructions = "x".repeat(MAX_SKILL_INSTRUCTIONS_BYTES + 1);
assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_))));
}
#[test]
fn validate_skill_rejects_oversized_name() {
let mut skill = skill();
skill.name = "a".repeat(MAX_SKILL_NAME_CHARS + 1);
assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_))));
}
#[test]
fn validate_skill_rejects_non_slug_name() {
// Uppercase, underscore, space and punctuation are all outside the
// Claude SKILL.md `[a-z0-9-]` name charset.
for bad in ["My-Skill", "my_skill", "my skill", "skill!"] {
let mut skill = skill();
skill.name = bad.to_string();
assert!(
matches!(validate_skill(&skill), Err(Error::BadRequest(_))),
"{bad:?} should be rejected"
);
}
}
#[test]
fn validate_skill_counts_description_in_characters() {
// 1024 two-byte chars exceed the byte limit but sit exactly on the
// character limit, so they must be accepted.
let mut skill = skill();
skill.description = "é".repeat(MAX_SKILL_DESCRIPTION_CHARS);
assert!(validate_skill(&skill).is_ok());
}
#[test]
fn workspace_capacity_allows_replacement_at_cap() {
// Already at the cap, but the upload only replaces an existing skill.
let at_cap = MAX_SKILLS_PER_WORKSPACE as i64;
assert!(check_workspace_skill_capacity(at_cap, 1, 1).is_ok());
}
#[test]
fn workspace_capacity_rejects_new_skill_over_cap() {
let at_cap = MAX_SKILLS_PER_WORKSPACE as i64;
assert!(matches!(
check_workspace_skill_capacity(at_cap, 0, 1),
Err(Error::BadRequest(_))
));
}
#[test]
fn collect_upload_names_trims_and_collects() {
let names = collect_upload_names(&[skill()]).unwrap();
assert_eq!(names, vec!["test-skill".to_string()]);
}
#[test]
fn collect_upload_names_rejects_duplicates() {
// Names are compared after trimming, so whitespace can't smuggle a dup in.
let dup = SkillUpload { name: " test-skill ".to_string(), ..skill() };
assert!(matches!(
collect_upload_names(&[skill(), dup]),
Err(Error::BadRequest(_))
));
}
}
File diff suppressed because it is too large Load Diff
+2 -6
View File
@@ -219,16 +219,12 @@ struct DatabaseCheckResult {
async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult {
let start = std::time::Instant::now();
// `pg_is_in_recovery()` is true on standbys/read-only replicas, so a primary
// returns true here. A read-only replica (e.g. after a failover where the
// primary became a secondary) reports unhealthy, letting liveness probes
// restart the pod instead of silently failing all writes.
let healthy = tokio::time::timeout(
HEALTH_CHECK_TIMEOUT,
sqlx::query_scalar!("SELECT NOT pg_is_in_recovery()").fetch_one(db),
sqlx::query_scalar!("SELECT 1").fetch_one(db),
)
.await
.map(|r| matches!(r, Ok(Some(true))))
.map(|r| r.is_ok())
.unwrap_or(false);
let latency_ms = start.elapsed().as_millis() as i64;
+7 -115
View File
@@ -513,26 +513,6 @@ async fn cancel_job_api(
Path((w_id, id)): Path<(String, Uuid)>,
Json(CancelJob { reason }): Json<CancelJob>,
) -> error::Result<String> {
// App embed tokens (the sandboxed app iframe) may cancel ONLY jobs they launched
// — their app's component runs, stamped created_by == viewer. cancel_job_api has
// no other per-job ownership check, so without this an embed token (which carries
// the viewer's identity) could cancel any job by id. NotFound (not 403) so the
// untrusted app can't probe job existence.
if let Some(authed) = opt_authed.as_ref() {
if windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) {
let created_by = sqlx::query_scalar!(
"SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2",
id,
&w_id
)
.fetch_optional(&db)
.await?;
if created_by.as_deref() != Some(authed.username.as_str()) {
return Err(Error::NotFound(format!("Job {id} not found")));
}
}
}
let tx = db.begin().await?;
let audit_author: AuditAuthor = match opt_authed.as_ref() {
@@ -1027,17 +1007,6 @@ async fn require_job_read_access(
return Ok(());
}
// App embed tokens (the sandboxed app iframe) carry the viewer's identity so the
// app can read its own component runs — which are stamped `created_by == viewer`
// and so already returned above. They must NOT inherit the viewer's *broader*
// job access (share links, folder ACLs, admin RLS): user-authored app JS holds
// this token, and letting it reach any job merely visible to the viewer would
// expose unrelated runs' results/logs. Stop at the launched-by-viewer grant.
// NotFound (not PermissionDenied) so the untrusted app can't probe job existence.
if windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) {
return Err(Error::NotFound(format!("Job {job_id} not found")));
}
// `username_override` is derived from the token *label* (`username_override_from_label`),
// which is fully user-controlled with no uniqueness/ownership check (webhook-/http-/
// email-/ws- trigger tokens, `ephemeral-script-end-user-*`, and the generic `label-*`
@@ -4053,17 +4022,11 @@ fn conditionally_require_authed_user(
}
pub async fn create_job_signature(
authed: ApiAuthed,
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
Query(approver): Query<QueryApprover>,
) -> error::Result<String> {
// The HMAC is treated as full authority by the resume endpoints, so minting
// it requires run scope on the suspended job's flow — not merely any
// jobs:run scope. No-op for unscoped tokens (incl. the in-flow substep token
// used by wmill.get_resume_urls()).
let flow_path = resume_target_flow_path(&db, &w_id, job_id).await?;
check_scopes(&authed, || format!("jobs:run:flows:{}", flow_path))?;
let key = get_workspace_key(&w_id, &db).await?;
create_signature(key, job_id, resume_id, approver.approver)
}
@@ -4146,17 +4109,11 @@ fn build_resume_url(
}
pub async fn get_resume_urls(
authed: ApiAuthed,
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
Query(approver): Query<QueryApprover>,
) -> error::JsonResult<ResumeUrls> {
// These URLs embed a resume signature (full resume capability), so a scoped
// token must hold run scope on the suspended job's flow. No-op for unscoped
// tokens (incl. the in-flow substep token). Trusted internal callers use
// get_resume_urls_internal directly and are unaffected.
let flow_path = resume_target_flow_path(&db, &w_id, job_id).await?;
check_scopes(&authed, || format!("jobs:run:flows:{}", flow_path))?;
get_resume_urls_internal(
Extension(db),
Path((w_id, job_id, resume_id)),
@@ -4223,46 +4180,6 @@ pub async fn get_resume_urls_internal(
Ok(Json(res))
}
/// Resolve the runnable path of the flow a (possibly step) job belongs to, used
/// to scope-check resume-signature minting against `jobs:run:flows:<path>`.
/// Returns an empty string when the path can't be resolved (e.g. previews or an
/// unknown job); an empty path only matters for path-restricted tokens, which
/// would not be running such a flow. Never hard-fails, so it can't break resume
/// for unscoped tokens (the in-flow `get_resume_urls()` path).
async fn resume_target_flow_path(db: &DB, w_id: &str, job_id: Uuid) -> error::Result<String> {
let job = sqlx::query!(
r#"SELECT kind::text as "kind!", parent_job, runnable_path
FROM v2_job WHERE id = $1 AND workspace_id = $2"#,
job_id,
w_id
)
.fetch_optional(db)
.await?;
let Some(job) = job else {
return Ok(String::new());
};
// All flow kinds: the job itself is the flow whose path scopes the resume.
if matches!(
job.kind.as_str(),
"flow" | "flowpreview" | "flownode" | "singlestepflow"
) {
return Ok(job.runnable_path.unwrap_or_default());
}
// Otherwise it's a step; its parent is the flow.
if let Some(parent) = job.parent_job {
return Ok(sqlx::query_scalar!(
"SELECT runnable_path FROM v2_job WHERE id = $1 AND workspace_id = $2",
parent,
w_id
)
.fetch_optional(db)
.await?
.flatten()
.unwrap_or_default());
}
Ok(job.runnable_path.unwrap_or_default())
}
/// Get the flow ID for a job. If the job is a flow, returns the job_id.
/// If the job is a step in a flow, returns the parent flow ID.
async fn get_flow_id_for_job(db: &DB, job_id: Uuid) -> error::Result<Uuid> {
@@ -9571,31 +9488,16 @@ mod approval_view_gate_tests {
fn anonymous_cannot_view_when_auth_required() {
// The regression: an unauthenticated holder of the approval token must see nothing.
let c = Some(conds(true, vec![]));
assert!(!can_view(
&None,
&c,
Some("f/team/flow"),
"trigger@example.com"
));
assert!(!can_view(&None, &c, Some("f/team/flow"), "trigger@example.com"));
}
#[test]
fn anonymous_can_view_when_no_auth_required() {
// Unchanged behaviour: token alone is sufficient when auth isn't required.
let c = Some(conds(false, vec![]));
assert!(can_view(
&None,
&c,
Some("f/team/flow"),
"trigger@example.com"
));
assert!(can_view(&None, &c, Some("f/team/flow"), "trigger@example.com"));
// No approval conditions at all also allows token-only view.
assert!(can_view(
&None,
&None,
Some("f/team/flow"),
"trigger@example.com"
));
assert!(can_view(&None, &None, Some("f/team/flow"), "trigger@example.com"));
}
#[test]
@@ -9620,17 +9522,7 @@ mod approval_view_gate_tests {
let member = Some(authed("carol", false, vec!["approvers".to_string()]));
let outsider = Some(authed("dave", false, vec!["other".to_string()]));
// Use a non-owned folder path so ownership doesn't short-circuit the check.
assert!(can_view(
&member,
&c,
Some("f/team/flow"),
"trigger@example.com"
));
assert!(!can_view(
&outsider,
&c,
Some("f/team/flow"),
"trigger@example.com"
));
assert!(can_view(&member, &c, Some("f/team/flow"), "trigger@example.com"));
assert!(!can_view(&outsider, &c, Some("f/team/flow"), "trigger@example.com"));
}
}
+5 -34
View File
@@ -66,7 +66,6 @@ use crate::scim_oss::has_scim_token;
use windmill_common::error::AppError;
mod ai;
mod ai_skills;
mod apps;
pub mod args;
mod audit;
@@ -546,15 +545,7 @@ pub async fn run_server(
Router::new()
// Reordered alphabetically
.nest("/acls", granular_acls::workspaced_service())
// CORS so the opaque-origin in-workspace app viewer (WIN-2006,
// sandboxed /apps/get) can read the app definition by path
// (apps/get/p, apps/embed_token/p) with a scoped embed token.
// Bearer-token-only (no cookies), consistent with the other
// workspaced services the iframe calls.
.nest(
"/apps",
apps::workspaced_service(request_size_limit * 5).layer(cors.clone()),
)
.nest("/apps", apps::workspaced_service(request_size_limit * 5))
.nest("/assets", windmill_api_assets::workspaced_service())
.nest("/audit", audit::workspaced_service())
.nest("/capture", capture::workspaced_service())
@@ -574,13 +565,7 @@ pub async fn run_server(
"/flow_conversations",
windmill_api_flow_conversations::workspaced_service(),
)
// CORS so an opaque-origin app iframe (WIN-2006 embed,
// no separate domain) can read folders/listnames with a
// scoped embed token. Consistent with apps_u/jobs_u cors.
.nest(
"/folders",
folders::workspaced_service().layer(cors.clone()),
)
.nest("/folders", folders::workspaced_service())
.nest("/folders_history", folder_history::workspaced_service())
.nest("/groups", groups::workspaced_service())
.nest("/groups_history", group_history::workspaced_service())
@@ -623,30 +608,20 @@ pub async fn run_server(
Router::new()
})
.nest("/ai", ai::workspaced_service())
.nest("/ai_skills", ai_skills::workspaced_service())
.nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service())
.nest(
"/path_autocomplete",
path_autocomplete::workspaced_service(),
)
.nest("/raw_apps", raw_apps::workspaced_service())
// CORS so the opaque-origin app iframe can read
// resources/list, resources/type/* with a scoped token.
.nest(
"/resources",
resources::workspaced_service().layer(cors.clone()),
)
.nest("/resources", resources::workspaced_service())
.nest("/shared_ui", workspace_shared_ui::workspaced_service())
.nest("/schedules", windmill_api_schedule::workspaced_service())
.nest("/scripts", scripts::workspaced_service())
.nest("/trash", trash::workspaced_service())
.nest(
"/users",
// CORS so the opaque-origin app iframe can read
// users/whoami with a scoped embed token.
users::workspaced_service()
.layer(Extension(argon2.clone()))
.layer(cors.clone()),
users::workspaced_service().layer(Extension(argon2.clone())),
)
.nest("/variables", variables::workspaced_service())
.nest("/volumes", volumes_oss::workspaced_service())
@@ -752,11 +727,7 @@ pub async fn run_server(
.nest("/apps_u", {
#[cfg(feature = "enterprise")]
{
// CORS so the opaque-origin app viewer (WIN-2006 embed, no
// separate domain) can load a custom-path public app via
// public_app_by_custom_path cross-origin. Consistent with
// the workspaced /w/{workspace_id}/apps_u mount below.
apps_oss::global_unauthed_service().layer(cors.clone())
apps_oss::global_unauthed_service()
}
#[cfg(not(feature = "enterprise"))]
@@ -18,7 +18,6 @@ use windmill_common::{
};
use crate::db::ApiAuthed;
use windmill_mcp::parse_mcp_scopes;
/// Token expiration for MCP OAuth tokens (1 week in seconds)
const MCP_OAUTH_TOKEN_EXPIRATION_SECS: u64 = 7 * 24 * 60 * 60;
@@ -586,8 +585,6 @@ async fn handle_refresh_token_grant(
Some(&new_access_token)
};
let new_refresh_token = rd_string(32);
// Re-issues the already-approved (hence already-contained) scopes verbatim;
// containment is enforced once at approval time, so no re-check here.
let scopes = token_row.scopes;
// Create new access token (rejects archived workspaces inline)
@@ -823,40 +820,6 @@ async fn oauth_approve_inner(
.map(|s| s.to_string())
.collect();
// The approver's own token bounds what it may grant: a scope-restricted MCP
// token must not approve a broader one (e.g. mcp:scripts:f/x -> mcp:all). An
// unrestricted approver (interactive session, scopes None) grants freely,
// which is the normal consent flow. This is the legitimate MCP-narrowing
// path, so it uses MCP-pattern containment rather than the byte-identical
// rule ensure_scopes_within_caller applies on the generic token endpoints.
let caller_restricted = authed
.scopes
.as_deref()
.is_some_and(|s| s.iter().any(|x| !x.starts_with("if_jobs:filter_tags:")));
if caller_restricted {
// An empty grant would mint a token the auth layer treats as unscoped
// (full privileges), so a restricted approver must not produce one.
if scopes.is_empty() {
return Err(Error::NotAuthorized(
"A scope-restricted token cannot approve an empty scope grant".to_string(),
));
}
if scopes.iter().any(|s| !s.starts_with("mcp:")) {
return Err(Error::NotAuthorized(
"A scope-restricted token can only approve MCP (mcp:*) scopes".to_string(),
));
}
let caller_config = parse_mcp_scopes(authed.scopes.as_deref().unwrap_or(&[]))
.map_err(|e| Error::InternalErr(format!("Failed to parse caller MCP scopes: {e}")))?;
let requested_config = parse_mcp_scopes(&scopes)
.map_err(|e| Error::BadRequest(format!("Failed to parse requested MCP scopes: {e}")))?;
if !caller_config.contains(&requested_config) {
return Err(Error::NotAuthorized(
"Requested scopes exceed the approving token's own MCP scopes".to_string(),
));
}
}
sqlx::query!(
"INSERT INTO mcp_oauth_server_code
(code, client_id, user_email, workspace_id, scopes, redirect_uri, code_challenge, code_challenge_method)
+1 -27
View File
@@ -455,35 +455,9 @@ pub async fn create_http_request(
}
};
// Bound the minted JWT to exactly this proxied route so a scope-restricted
// MCP token can't be widened into a full-privilege blank check. The
// endpoint-name gate (in the MCP runner) already authorized *which* endpoint
// may be called; this constrains what the resulting request can do. Unscoped
// callers (cookie / full-privilege tokens) keep an unscoped JWT to preserve
// existing behavior. A scope-restricted caller whose route can't be resolved
// fails closed.
let caller_restricted = api_authed
.scopes
.as_deref()
.is_some_and(|s| s.iter().any(|x| !x.starts_with("if_jobs:filter_tags:")));
let scopes = if caller_restricted {
let parsed = reqwest::Url::parse(url)
.map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?;
let scope =
windmill_api_auth::scopes::scope_for_route(method, parsed.path()).ok_or_else(|| {
ErrorData::internal_error(
"Could not derive route scope for proxied MCP endpoint".to_string(),
None,
)
})?;
Some(vec![scope])
} else {
None
};
// Add authorization header
let authed = Authed::from(api_authed.clone());
let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, scopes)
let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None)
.await
.map_err(|e| ErrorData::internal_error(e.to_string(), None))?;
request_builder = request_builder.header("Authorization", format!("Bearer {}", token));
-1
View File
@@ -98,7 +98,6 @@ fn build_standard_scope_domains() -> Vec<ScopeDomain> {
("configs", "Configs", "Configuration management", false),
("oauth", "OAuth", "OAuth management", false),
("ai", "AI", "AI feature management", false),
("ai_skills", "AI Skills", "AI skill management", false),
(
"agent_workers",
"Agent Workers",
@@ -12,8 +12,6 @@ use crate::db::ApiAuthed;
use crate::{apps::AppWithLastVersion, db::DB, folders::Folder};
use windmill_api_auth::check_scopes;
#[cfg(any(
feature = "http_trigger",
feature = "websocket",
@@ -584,18 +582,6 @@ pub(crate) async fn tarball_workspace(
skip_resources
);
// The route is gated by workspaces:read, but exporting DECRYPTED secrets is a
// variable-read capability beyond workspace metadata. Require variables:read
// only on the plaintext-secret path: ordinary tarball pulls (structure and
// encrypted-only values) keep working with workspaces:read, and the workspace
// key itself stays admin-only (include_key). No-op for unscoped tokens.
if plain_secret.or(plain_secrets).unwrap_or(false)
&& !skip_secrets.unwrap_or(false)
&& !skip_variables.unwrap_or(false)
{
check_scopes(&authed, || "variables:read".to_string())?;
}
// Opt-in behavior for surfacing per-resource ACLs on flow/app rows.
// Folder and group rows have always carried `extra_perms` in source and
// continue to do so unconditionally (`KeepEvenEmpty`) so existing
@@ -608,24 +594,6 @@ pub(crate) async fn tarball_workspace(
let mut tx = user_db.begin(&authed).await?;
// Exporting decrypted secrets in bulk is the same capability as a per-item
// secret read, so record it for parity with variables.decrypt_secret.
if plain_secret.or(plain_secrets).unwrap_or(false)
&& !skip_variables.unwrap_or(false)
&& !skip_secrets.unwrap_or(false)
{
windmill_audit::audit_oss::audit_log(
&mut *tx,
&authed,
"variables.decrypt_secret",
windmill_audit::ActionKind::Execute,
&w_id,
Some("workspace_tarball_export"),
None,
)
.await?;
}
// Source-of-truth for fork-ness: the workspace's parent_workspace_id column.
// The wm-fork-* prefix is a creation-time naming convention that could in
// principle drift (rename, manual SQL); the column is the contract that
@@ -15,29 +15,6 @@ pub async fn get_github_app_token_internal(
));
}
lazy_static::lazy_static! {
/// Matches a `user:password@` (or `user@`) userinfo component right after the URL scheme.
static ref GIT_URL_USERINFO_RE: regex::Regex =
regex::Regex::new(r"://[^/@]+@").unwrap();
}
/// Strip embedded credentials (the `user:password@` userinfo component) from a git URL so it can be
/// safely included in error messages and logs. Falls back to a regex when the URL does not parse.
pub fn sanitize_git_url(url: &str) -> String {
if let Ok(mut parsed) = Url::parse(url) {
if !parsed.username().is_empty() || parsed.password().is_some() {
// These setters only fail for cannot-be-a-base URLs, in which case we keep the parsed
// string as-is and let the regex fallback below handle stripping.
let _ = parsed.set_username("");
let _ = parsed.set_password(None);
}
return GIT_URL_USERINFO_RE
.replace(parsed.as_str(), "://***@")
.into_owned();
}
GIT_URL_USERINFO_RE.replace(url, "://***@").into_owned()
}
pub fn prepend_token_to_github_url(
github_url: &str,
installation_token: &str,
@@ -55,41 +32,3 @@ pub fn prepend_token_to_github_url(
url.path()
))
}
#[cfg(test)]
mod tests {
use super::sanitize_git_url;
#[test]
fn strips_username_and_password() {
assert_eq!(
sanitize_git_url("https://user:p4ssw0rd@github.com/org/repo.git"),
"https://github.com/org/repo.git"
);
}
#[test]
fn strips_token_only_userinfo() {
assert_eq!(
sanitize_git_url("https://ghp_secrettoken@github.com/org/repo.git"),
"https://github.com/org/repo.git"
);
}
#[test]
fn leaves_credential_free_url_untouched() {
assert_eq!(
sanitize_git_url("https://github.com/org/repo.git"),
"https://github.com/org/repo.git"
);
}
#[test]
fn strips_credentials_from_unparseable_url() {
// scp-like syntax that `url::Url` cannot parse
assert_eq!(
sanitize_git_url("not a url://user:secret@host/repo"),
"not a url://***@host/repo"
);
}
}
-1
View File
@@ -61,7 +61,6 @@ pub mod indexer;
pub mod instance_config;
pub mod job_metrics;
pub mod log_context;
pub mod materialization;
pub mod min_version;
pub mod notify_events;
pub mod runtime_assets;
@@ -1,135 +0,0 @@
//! CE materialization state — the per-partition status recorded by the managed
//! `// materialize` write (in windmill-worker), read by the partition-status
//! grid and by the EE backfill worklist.
//!
//! The write engine and this state are CE; only automatic partition
//! *resolution* (`partition_ee`) and *backfill* orchestration
//! (`pipeline_advanced_ee`) are enterprise. This module is the shared seam:
//! the EE backfill enumerates the partitions in a range, diffs them against
//! these rows to find the missing/failed set, and pushes one CE materialization
//! job per gap (with an explicit `partition` arg — which runs idempotently and
//! upserts the row here). Nothing about that orchestration lives in this file;
//! it only needs the rows to exist, which is why recording is CE.
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
use uuid::Uuid;
use crate::assets::AssetKind;
use crate::error::Result;
/// Sentinel `partition` value for an unpartitioned (whole-table)
/// materialization — partition is part of the primary key and cannot be NULL.
pub const UNPARTITIONED: &str = "";
/// Mirrors the `MATERIALIZATION_STATUS` pg enum (see migration
/// `20260619170118_add_materialized_partition`).
#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[sqlx(type_name = "MATERIALIZATION_STATUS", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum MaterializationStatus {
Running,
Materialized,
Failed,
}
/// The materialization outcome an agent worker (`Connection::Http`, no direct
/// DB) sends to the API to be recorded. Mirrors the `record_materialization`
/// args; the API handler unpacks it and calls that function with its own DB.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecordMaterializationRequest {
pub asset_kind: AssetKind,
pub asset_path: String,
pub partition: String,
pub status: MaterializationStatus,
pub snapshot_id: Option<i64>,
pub row_count: Option<i64>,
pub job_id: Option<Uuid>,
pub error: Option<String>,
}
/// Upsert the latest materialization state for one (asset, partition) slice.
/// The worker records the terminal outcome once the write finishes:
/// `Materialized` (with the DuckLake `snapshot_id` + `row_count`) or `Failed`
/// (with `error`). `Running` mirrors the pg enum but has no writer in this flow.
/// Idempotent: re-running the same partition overwrites the row — exactly the
/// backfill / failure-recovery contract.
#[allow(clippy::too_many_arguments)]
pub async fn record_materialization<'e>(
executor: impl PgExecutor<'e>,
workspace_id: &str,
asset_kind: AssetKind,
asset_path: &str,
partition: &str,
status: MaterializationStatus,
snapshot_id: Option<i64>,
row_count: Option<i64>,
job_id: Option<Uuid>,
error: Option<&str>,
) -> Result<()> {
sqlx::query!(
"INSERT INTO materialized_partition
(workspace_id, asset_kind, asset_path, partition, status,
snapshot_id, row_count, job_id, materialized_at, error)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)
ON CONFLICT (workspace_id, asset_kind, asset_path, partition)
DO UPDATE SET status = EXCLUDED.status,
snapshot_id = EXCLUDED.snapshot_id,
row_count = EXCLUDED.row_count,
job_id = EXCLUDED.job_id,
materialized_at = now(),
error = EXCLUDED.error",
workspace_id,
asset_kind as AssetKind,
asset_path,
partition,
status as MaterializationStatus,
snapshot_id,
row_count,
job_id,
error,
)
.execute(executor)
.await?;
Ok(())
}
/// One materialized-partition row, for the status grid / backfill diff.
#[derive(sqlx::FromRow, Debug, Clone, Serialize)]
pub struct MaterializedPartition {
pub asset_kind: AssetKind,
pub asset_path: String,
pub partition: String,
pub status: MaterializationStatus,
pub snapshot_id: Option<i64>,
pub row_count: Option<i64>,
pub job_id: Option<Uuid>,
pub materialized_at: DateTime<Utc>,
pub error: Option<String>,
}
/// All recorded partitions for one asset, newest first — the grid's data and
/// the backfill worklist's "what already exists" set.
pub async fn list_materialized_partitions<'e>(
executor: impl PgExecutor<'e>,
workspace_id: &str,
asset_kind: AssetKind,
asset_path: &str,
) -> Result<Vec<MaterializedPartition>> {
let rows = sqlx::query_as!(
MaterializedPartition,
r#"SELECT asset_kind AS "asset_kind: AssetKind", asset_path, partition,
status AS "status: MaterializationStatus", snapshot_id,
row_count, job_id, materialized_at, error
FROM materialized_partition
WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3
ORDER BY partition DESC"#,
workspace_id,
asset_kind as AssetKind,
asset_path,
)
.fetch_all(executor)
.await?;
Ok(rows)
}
-112
View File
@@ -38,71 +38,6 @@ impl McpScopeConfig {
is_resource_allowed(path, patterns)
}
/// Directional subset check: does this config grant at least everything
/// `requested` grants? Used to enforce monotonic containment when an MCP
/// OAuth approval mints a token (the granted scopes must be within the
/// approving token's own scopes).
///
/// Unlike `is_allowed` (which tests a single concrete path with OR
/// semantics), this requires every requested pattern to be covered by some
/// caller pattern — so `mcp:scripts:f/x` cannot widen into `mcp:scripts:*`.
pub fn contains(&self, requested: &McpScopeConfig) -> bool {
if self.all {
return true;
}
if requested.all {
return false;
}
if requested.favorites && !self.favorites {
return false;
}
if let Some(req_hub) = requested.hub_apps.as_ref() {
match self.hub_apps.as_ref() {
Some(caller_hub) => {
let caller_apps: std::collections::HashSet<&str> =
caller_hub.split(',').map(|s| s.trim()).collect();
if !req_hub
.split(',')
.map(|s| s.trim())
.all(|a| caller_apps.contains(a))
{
return false;
}
}
None => return false,
}
}
resource_list_covers(&self.scripts, &requested.scripts)
&& resource_list_covers(&self.flows, &requested.flows)
&& resource_list_covers(&self.endpoints, &requested.endpoints)
}
}
/// Every requested pattern must be covered by some caller pattern.
fn resource_list_covers(caller: &[String], requested: &[String]) -> bool {
requested
.iter()
.all(|req| caller.iter().any(|c| pattern_covers(c, req)))
}
/// Directional: does the single caller pattern cover `requested`? `caller` may
/// be `*`, an exact path/name, or a `<prefix>/*` subtree; `requested` may itself
/// be a subtree wildcard, in which case the whole requested subtree must fall
/// within the caller's. Mirrors the route-scope containment in windmill-api-auth.
fn pattern_covers(caller: &str, requested: &str) -> bool {
if caller == "*" || caller == requested {
return true;
}
// An exact caller pattern only covers itself (handled above); a wildcard
// requested can never be covered by a non-`*` exact caller.
let Some(prefix) = caller.strip_suffix("/*") else {
return false;
};
let requested_base = requested.strip_suffix("/*").unwrap_or(requested);
requested_base == prefix
|| (requested_base.starts_with(prefix)
&& requested_base.as_bytes().get(prefix.len()) == Some(&b'/'))
}
/// Parse MCP scopes from token scope strings
@@ -319,51 +254,4 @@ mod tests {
assert!(config.is_allowed("flow", "f/automation/test"));
assert!(!config.is_allowed("flow", "f/other/test"));
}
fn cfg(scopes: &[&str]) -> McpScopeConfig {
parse_mcp_scopes(&scopes.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
}
#[test]
fn test_contains_subset_and_widening() {
// mcp:all contains anything.
assert!(cfg(&["mcp:all"]).contains(&cfg(&["mcp:scripts:f/x"])));
assert!(cfg(&["mcp:all"]).contains(&cfg(&["mcp:all"])));
// A wildcard caller covers narrower requests, but not other domains/all.
let star = cfg(&["mcp:scripts:*"]);
assert!(star.contains(&cfg(&["mcp:scripts:f/x"])));
assert!(star.contains(&cfg(&["mcp:scripts:*"])));
assert!(!star.contains(&cfg(&["mcp:all"])));
assert!(!star.contains(&cfg(&["mcp:flows:f/x"])));
// The core regression: a single-path caller must NOT widen into `*` or
// into another path.
let narrow = cfg(&["mcp:scripts:f/x"]);
assert!(narrow.contains(&cfg(&["mcp:scripts:f/x"])));
assert!(!narrow.contains(&cfg(&["mcp:scripts:*"])));
assert!(!narrow.contains(&cfg(&["mcp:scripts:f/y"])));
assert!(!narrow.contains(&cfg(&["mcp:all"])));
// Subtree wildcard covers paths within it but not a sibling subtree.
let subtree = cfg(&["mcp:scripts:f/team/*"]);
assert!(subtree.contains(&cfg(&["mcp:scripts:f/team/sub"])));
assert!(subtree.contains(&cfg(&["mcp:scripts:f/team/sub/*"])));
assert!(!subtree.contains(&cfg(&["mcp:scripts:f/other/x"])));
}
#[test]
fn test_contains_favorites_and_endpoints() {
assert!(cfg(&["mcp:favorites"]).contains(&cfg(&["mcp:favorites"])));
// A caller without favorites cannot grant favorites.
assert!(!cfg(&["mcp:scripts:*"]).contains(&cfg(&["mcp:favorites"])));
// Endpoint names match exactly (or via `*`).
let ep = cfg(&["mcp:endpoints:getVariable"]);
assert!(ep.contains(&cfg(&["mcp:endpoints:getVariable"])));
assert!(!ep.contains(&cfg(&["mcp:endpoints:getResource"])));
assert!(!ep.contains(&cfg(&["mcp:all"])));
// mcp:all grants all endpoints.
assert!(cfg(&["mcp:all"]).contains(&cfg(&["mcp:endpoints:getResource"])));
}
}
+6 -44
View File
@@ -452,12 +452,12 @@ pub async fn build_client_credentials_oauth_client(
let caller_supplied_creds = !client_id.is_empty() && !client_secret.is_empty();
// Apply the resolved concrete token URL. Instance-templated providers (e.g.
// Coupa) carry an empty or `{instance}`-templated token URL in their registry
// config; the resolved value (host-pinned for instance-name connections,
// persisted on the row for refresh) is what completes it. For bring-your-own
// connections this value may instead be a caller-supplied override — safe
// because only the caller's own credentials are ever sent to it.
// Apply the server-resolved concrete token URL. Instance-templated providers
// (e.g. Coupa) carry an empty or `{instance}`-templated token URL in their
// registry config; the resolved value (host-pinned for bring-your-own,
// persisted on the row for refresh) is what completes it. The caller never
// supplies a free-form token URL: this value always comes from
// `resolve_cc_token_url_input` or a previously-resolved persisted URL.
if let Some(url) = resolved_token_url {
connect_config.token_url = url.to_string();
}
@@ -652,28 +652,6 @@ pub fn resolve_cc_token_url_input(
Ok(template.replace("{instance}", value))
}
/// Whether a built-in provider's client-credentials token URL is host-pinned via
/// an `{instance}` template (e.g. servicenow, snowflake, coupa). Such providers
/// only accept an instance name substituted into a fixed-host template, so a
/// free-form caller token URL override must be rejected for them — otherwise the
/// exchange host could be redirected, which is exactly what the template pins.
/// Fixed-host registry providers and custom (non-registry) providers return
/// `false`: an override is allowed there.
pub fn is_instance_templated_cc(connect_configs_json: &str, client_name: &str) -> bool {
serde_json::from_str::<HashMap<String, OAuthConfig>>(connect_configs_json)
.ok()
.and_then(|m| resolve_registry_config(&m, client_name))
.map(|cfg| {
cfg.connect_config_template
.as_ref()
.map(|t| t.token_url.clone())
.filter(|u| !u.is_empty())
.unwrap_or(cfg.token_url)
.contains("{instance}")
})
.unwrap_or(false)
}
/// Resolve the concrete bring-your-own client-credentials token URL for any
/// provider, never from a caller-supplied URL:
/// - **Built-in registry providers** resolve from the registry via
@@ -1372,20 +1350,4 @@ mod tests {
assert!(resolve_cc_token_url_input(CC_REGISTRY, "bad_host_tpl", Some("evil.com")).is_err());
assert!(resolve_cc_token_url_input(CC_REGISTRY, "bad_mid_tpl", Some("evil")).is_err());
}
#[test]
fn instance_templated_cc_true_for_templated_providers() {
// Host-pinned via `{instance}`: a bring-your-own token URL override must be
// refused for these (only the instance-name path may set their URL).
assert!(is_instance_templated_cc(CC_REGISTRY, "coupa"));
assert!(is_instance_templated_cc(CC_REGISTRY, "servicenow"));
}
#[test]
fn instance_templated_cc_false_for_fixed_host_and_unknown() {
// Fixed-host registry provider and custom (non-registry) provider both allow
// an override, so neither is reported as instance-templated.
assert!(!is_instance_templated_cc(CC_REGISTRY, "visma"));
assert!(!is_instance_templated_cc(CC_REGISTRY, "my_custom_thing"));
}
}
-4
View File
@@ -53,7 +53,3 @@ futures.workspace = true
chrono.workspace = true
reqwest.workspace = true
anyhow.workspace = true
base64.workspace = true
[dev-dependencies]
magic-crypt.workspace = true
-6
View File
@@ -1634,12 +1634,6 @@ async fn update_resource(
let path = path.to_path();
check_scopes(&authed, || format!("resources:write:{}", path))?;
// A rename moves the resource (and its linked variable) to ns.path, so the
// destination must also be within the token's write scope, not just the
// source path.
if let Some(npath) = ns.path.as_deref() {
check_scopes(&authed, || format!("resources:write:{}", npath))?;
}
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
&w_id,
AuditAuthorable::username(&authed),
+2 -106
View File
@@ -14,8 +14,8 @@ use windmill_common::db::DB;
use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult};
use crate::secret_backend_ext::{
delete_secret_from_backend, get_secret_value, is_external_stored_value, is_vault_stored_value,
rename_vault_secret, store_secret_value,
delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret,
store_secret_value,
};
use windmill_common::utils::{escape_ilike_pattern, BulkDeleteRequest};
use windmill_common::webhook::{WebhookMessage, WebhookShared};
@@ -25,7 +25,6 @@ use axum::{
routing::{delete, get, post},
Json, Router,
};
use base64::{engine::general_purpose::STANDARD, Engine as _};
use futures::future::try_join_all;
use hyper::StatusCode;
use serde_json::Value;
@@ -536,35 +535,6 @@ async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> {
return Ok(());
}
/// Reject a secret value flagged as already-encrypted (`already_encrypted=true`)
/// that is not actually workspace-key ciphertext — e.g. plaintext mistakenly
/// pushed as encrypted. Storing plaintext in the encrypted `value` column
/// silently bricks the variable: every later read fails to decrypt it.
///
/// The check is purely structural and never decrypts, so it cannot act as a
/// decryption/padding oracle for a caller who can write but not read secrets.
/// `encrypt` (AES-256-CBC) always yields standard base64 decoding to a non-zero
/// multiple of the 16-byte block size; anything else cannot be our ciphertext.
/// Values stored by an external backend ($vault:/$aws_sm:/$azure_kv: markers)
/// are not workspace ciphertext and are passed through untouched.
fn validate_already_encrypted_secret(path: &str, value: &str) -> Result<()> {
if is_external_stored_value(value) {
return Ok(());
}
let looks_like_ciphertext = STANDARD
.decode(value)
.map(|bytes| !bytes.is_empty() && bytes.len() % 16 == 0)
.unwrap_or(false);
if !looks_like_ciphertext {
return Err(Error::BadRequest(format!(
"Variable {path} was sent as already-encrypted (already_encrypted=true) but its \
value is not valid workspace-encrypted ciphertext. To push a plaintext secret, \
send it without already_encrypted (CLI: use --plain-secrets) so it gets encrypted."
)));
}
Ok(())
}
async fn create_variable(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -615,11 +585,6 @@ async fn create_variable(
// Use secret backend for encryption (supports both DB and Vault)
store_secret_value(&db, &w_id, &variable.path, &plain).await?
} else {
if variable.is_secret {
// already_encrypted == true: value is stored verbatim, so it must be
// ciphertext and not plaintext mislabeled as encrypted.
validate_already_encrypted_secret(&variable.path, &variable.value)?;
}
variable.value
};
@@ -1072,12 +1037,6 @@ async fn update_variable(
let path = path.to_path();
check_scopes(&authed, || format!("variables:write:{}", path))?;
// A rename moves the (possibly secret) variable to ns.path, so the
// destination must also be within the token's write scope, not just the
// source path.
if let Some(npath) = ns.path.as_deref() {
check_scopes(&authed, || format!("variables:write:{}", npath))?;
}
let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
let mut sqlb = SqlBuilder::update_table("variable");
@@ -1117,11 +1076,6 @@ async fn update_variable(
// Store at target_path (new path if renaming, otherwise current path)
store_secret_value(&db, &w_id, target_path, &plain).await?
} else {
if is_secret {
// already_encrypted == true: value is stored verbatim, so it must
// be ciphertext and not plaintext mislabeled as encrypted.
validate_already_encrypted_secret(target_path, &nvalue)?;
}
nvalue
};
sqlb.set_str("value", &value);
@@ -1553,61 +1507,3 @@ pub async fn get_value_internal<'a>(
Ok(r)
}
#[cfg(test)]
mod tests {
use super::*;
use magic_crypt::MagicCryptTrait;
#[test]
fn accepts_real_workspace_ciphertext() {
// The exact shape produced by `encrypt` (AES-256-CBC, base64).
let mc = magic_crypt::new_magic_crypt!("a-test-workspace-key", 256);
for plain in [
"",
"original-secret",
"some: plaintext\n",
"a".repeat(500).as_str(),
] {
let ciphertext = mc.encrypt_str_to_base64(plain);
assert!(
validate_already_encrypted_secret("f/x/cfg", &ciphertext).is_ok(),
"should accept genuine ciphertext for plaintext {plain:?}: {ciphertext}"
);
}
}
#[test]
fn rejects_plaintext_mislabeled_as_encrypted() {
// Plaintext mislabeled as encrypted: storing it verbatim would make the
// variable undecryptable on every read, so it must be rejected.
for plaintext in [
"some: plaintext\n",
"original-secret",
"hunter2",
"{\"a\": 1}",
"not base64!!",
" leading-space",
] {
assert!(
validate_already_encrypted_secret("f/x/cfg", plaintext).is_err(),
"should reject plaintext mislabeled as encrypted: {plaintext:?}"
);
}
}
#[test]
fn rejects_empty_and_non_block_aligned() {
// Valid base64 but not a whole number of AES blocks -> cannot be our ciphertext.
assert!(validate_already_encrypted_secret("p", "").is_err());
assert!(validate_already_encrypted_secret("p", "dGVzdA==").is_err()); // "test" -> 4 bytes
}
#[test]
fn passes_through_external_backend_markers() {
// External secret backends store $-prefixed markers, not workspace ciphertext.
for marker in ["$vault:f/x/cfg", "$aws_sm:f/x/cfg", "$azure_kv:f/x/cfg"] {
assert!(validate_already_encrypted_secret("f/x/cfg", marker).is_ok());
}
}
}
+2 -9
View File
@@ -7,7 +7,7 @@ use axum::{extract::Path, routing::post, Extension, Json, Router};
use http::StatusCode;
use sqlx::PgConnection;
use std::collections::HashSet;
use windmill_api_auth::{check_scopes, ApiAuthed};
use windmill_api_auth::ApiAuthed;
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::global_settings::HTTP_ROUTE_WORKSPACED_ROUTE;
use windmill_common::{
@@ -262,12 +262,6 @@ pub async fn create_many_http_triggers(
let mut route_path_keys = Vec::with_capacity(new_http_triggers.len());
for new_http_trigger in new_http_triggers.iter() {
// Per-item write scope, matching the single-create handler. The bulk
// endpoint must not let a path-scoped token create triggers outside it.
check_scopes(&authed, || {
format!("http_triggers:write:{}", &new_http_trigger.base.path)
})?;
handler
.validate_new(&db, &w_id, &new_http_trigger.config)
.await
@@ -379,8 +373,7 @@ impl TriggerCrud for HttpTrigger {
const TABLE_NAME: &'static str = "http_trigger";
const TRIGGER_TYPE: &'static str = "http";
const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind =
windmill_common::user_drafts::UserDraftItemKind::TriggerHttp;
const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind = windmill_common::user_drafts::UserDraftItemKind::TriggerHttp;
const SUPPORTS_SERVER_STATE: bool = false;
const SUPPORTS_TEST_CONNECTION: bool = false;
const ROUTE_PREFIX: &'static str = "/http_triggers";
@@ -4,7 +4,7 @@ use async_trait::async_trait;
use itertools::Itertools;
use serde_json::value::RawValue;
use sqlx::{types::Json as SqlxJson, PgConnection};
use windmill_api_auth::{check_scopes, ApiAuthed};
use windmill_api_auth::ApiAuthed;
use windmill_common::DB;
use windmill_common::{
db::UserDB,
@@ -15,39 +15,10 @@ use windmill_git_sync::DeployedObject;
use windmill_trigger::{Trigger, TriggerCrud, TriggerData};
use super::{
get_url_from_runnable_value, listener::InitialMessage, proxy::connect_async_with_proxy,
validate_websocket_url_for_ssrf, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest,
WebsocketTrigger,
get_url_from_runnable_value, proxy::connect_async_with_proxy, validate_websocket_url_for_ssrf,
TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, WebsocketTrigger,
};
/// A websocket_triggers:write token can configure secondary runnables that the
/// listener later executes under the trigger owner's identity: a `$flow:`/
/// `$script:` URL resolver and `initial_messages` of kind `runnable_result`.
/// That execution happens in a background task where the reconstructed authed is
/// scopeless (so its check_scopes is a no-op), so enforce run scope here, at
/// create/update time, against the API caller's token.
fn check_secondary_runnable_scopes(
authed: &ApiAuthed,
config: &WebsocketConfigRequest,
) -> Result<()> {
if let Some(rest) = config.url.strip_prefix("$flow:") {
check_scopes(authed, || format!("jobs:run:flows:{}", rest))?;
} else if let Some(rest) = config.url.strip_prefix("$script:") {
check_scopes(authed, || format!("jobs:run:scripts:{}", rest))?;
}
if let Some(messages) = config.initial_messages.as_ref() {
for msg in messages {
if let Ok(InitialMessage::RunnableResult { path, is_flow, .. }) =
serde_json::from_value::<InitialMessage>(msg.clone())
{
let kind = if is_flow { "flows" } else { "scripts" };
check_scopes(authed, || format!("jobs:run:{}:{}", kind, path))?;
}
}
}
Ok(())
}
#[async_trait]
impl TriggerCrud for WebsocketTrigger {
type TriggerConfig = WebsocketConfig;
@@ -130,7 +101,6 @@ impl TriggerCrud for WebsocketTrigger {
w_id: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
check_secondary_runnable_scopes(authed, &trigger.config)?;
let resolved_edited_by = trigger.base.resolve_edited_by(authed);
let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed);
let filters = trigger
@@ -208,7 +178,6 @@ impl TriggerCrud for WebsocketTrigger {
path: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
check_secondary_runnable_scopes(authed, &trigger.config)?;
let resolved_edited_by = trigger.base.resolve_edited_by(authed);
let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed);
let filters = trigger
@@ -509,7 +509,7 @@ impl Clone for ReturnMessageChannels {
}
#[derive(Debug, Deserialize)]
pub(crate) enum InitialMessage {
enum InitialMessage {
#[serde(rename = "raw_message")]
RawMessage(String),
#[serde(rename = "runnable_result")]
@@ -78,22 +78,4 @@ pub async fn get_datatable_resource_from_agent_http(
.await
}
/// Record a materialization outcome from an agent worker (no direct DB) via the
/// API, so `materialized_partition` state lands the same as on a Sql worker.
// Only called from the duckdb executor, which is itself `#[cfg(feature = "duckdb")]`.
#[cfg(feature = "duckdb")]
pub async fn record_materialization_from_agent_http(
client: &HttpClient,
w_id: &str,
req: &windmill_common::materialization::RecordMaterializationRequest,
) -> anyhow::Result<()> {
client
.post(
&format!("/api/w/{}/agent_workers/record_materialization", w_id),
None,
req,
)
.await
}
pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping";
@@ -13,7 +13,7 @@ use tokio::process::Command;
use uuid::Uuid;
use windmill_common::{
error,
git_sync_oss::{prepend_token_to_github_url, sanitize_git_url},
git_sync_oss::prepend_token_to_github_url,
worker::{
is_allowed_file_location, split_python_requirements, to_raw_value, write_file,
write_file_at_user_defined_location, Connection, PyVAlias, WORKER_CONFIG,
@@ -847,7 +847,7 @@ pub async fn get_git_repo_full_head_commit_hash(
.first()
.ok_or(anyhow!(
"The HEAD commit hash was not found for repo `{}`",
sanitize_git_url(&repo.url)
&repo.url
))?
.split_whitespace()
.next()
@@ -1254,12 +1254,7 @@ pub async fn handle_ansible_job(
git_ssh_cmd,
)
.await
.map_err(|e| {
anyhow!(
"Failed to clone git repo `{}`: {e}",
sanitize_git_url(&repo.url)
)
})?;
.map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?;
} else {
clone_repo(
&repo,
@@ -1274,12 +1269,7 @@ pub async fn handle_ansible_job(
git_ssh_cmd,
)
.await
.map_err(|e| {
anyhow!(
"Failed to clone git repo `{}`: {e}",
sanitize_git_url(&repo.url)
)
})?;
.map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?;
}
append_logs(
@@ -1326,7 +1316,7 @@ pub async fn handle_ansible_job(
append_logs(
&job.id,
&job.workspace_id,
format!("\nCloning {}...\n", sanitize_git_url(&repo.url)),
format!("\nCloning {}...\n", &repo.url),
conn,
)
.await;
@@ -1348,18 +1338,13 @@ pub async fn handle_ansible_job(
git_ssh_cmd,
)
.await
.map_err(|e| {
anyhow!(
"Failed to clone git repo `{}`: {e}",
sanitize_git_url(&repo.url)
)
})?;
.map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?;
} else {
if req_lockfiles.is_some() {
append_logs(
&job.id,
&job.workspace_id,
format!("Warning: `{}` is using latest commit because the lockfile didn't store a commit hash for this repo. Updates to the repo could break the deployed playbook.\n", sanitize_git_url(&repo.url)),
format!("Warning: `{}` is using latest commit because the lockfile didn't store a commit hash for this repo. Updates to the repo could break the deployed playbook.\n", &repo.url),
conn,
)
.await;
@@ -1377,22 +1362,13 @@ pub async fn handle_ansible_job(
git_ssh_cmd,
)
.await
.map_err(|e| {
anyhow!(
"Failed to clone git repo `{}`: {e}",
sanitize_git_url(&repo.url)
)
})?;
.map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?;
}
append_logs(
&job.id,
&job.workspace_id,
format!(
"Cloned {} into {}\n",
sanitize_git_url(&repo.url),
&repo.target_path
),
format!("Cloned {} into {}\n", &repo.url, &repo.target_path),
conn,
)
.await;
@@ -32,178 +32,6 @@ use crate::sql_utils::remove_comments;
use windmill_common::client::AuthedClient;
use windmill_object_store::DEFAULT_STORAGE;
// What a `// materialize` run records into `materialized_partition` once it
// finishes. `asset_path` is the full `<name>/<table>` (the asset identity);
// `partition` is "" for an unpartitioned (whole-table) materialization.
struct MaterializeExec {
asset_kind: windmill_common::assets::AssetKind,
asset_path: String,
partition: String,
}
// If `query` declares `// materialize <ducklake>`, return what to record plus,
// for the default managed mode, the rewritten managed-write SQL (in `manual`
// mode the script writes its own DDL, so the rewrite is `None`). The rewritten
// SQL contains a synthetic `ATTACH 'ducklake://<name>' AS _wm_target` that the
// normal ATTACH-transform pass resolves to real credentials — the same path as
// the user's own ATTACH. Returns `None` when there is no materialize annotation
// or the target isn't a ducklake (only ducklake is materialized in v1).
fn build_materialized_query(
query: &str,
partition_value: Option<&str>,
) -> Result<Option<(Option<String>, MaterializeExec)>> {
use windmill_parser::asset_parser::{parse_pipeline_annotations, AssetKind as PAssetKind};
use windmill_parser::sql_materialize::{
build_wrap_blocks, classify_wrap, MaterializeStrategy, TARGET_ALIAS,
};
let ann = parse_pipeline_annotations(query);
let Some(m) = ann.materialize else {
return Ok(None);
};
if m.target_kind != PAssetKind::Ducklake {
return Ok(None);
}
let partitioned = ann.partition.is_some();
let partition = partition_value.unwrap_or("").to_string();
// Partition *resolution* is enterprise; in its absence a partitioned
// materialize only runs with an explicit `partition` arg. Fail loudly rather
// than silently materialize the wrong (empty) slice.
if partitioned && partition.is_empty() {
return Err(Error::ExecutionErr(
"materialize: a `// partitioned` script ran with no resolved partition — pass an \
explicit `partition` arg, or enable enterprise partition resolution"
.to_string(),
));
}
// Convention: `ducklake://<name>/<table>` — <name> is the configured
// ducklake (resolved like a user ATTACH), <table> is the rest.
let (ducklake_name, table) = m
.target_path
.split_once('/')
.unwrap_or((m.target_path.as_str(), ""));
let meta = MaterializeExec {
asset_kind: windmill_common::assets::AssetKind::Ducklake,
asset_path: m.target_path.clone(),
partition: partition.clone(),
};
if m.manual {
// Escape hatch: the script owns its DDL; we only record state.
return Ok(Some((None, meta)));
}
if table.is_empty() {
return Err(Error::ExecutionErr(format!(
"materialize: target `ducklake://{}` has no table (use ducklake://<name>/<table>)",
m.target_path
)));
}
let mut plan = classify_wrap(query).map_err(|e| Error::ExecutionErr(e.message()))?;
// Resolve the `{partition}` token (same token `// on` asset URIs use) to the
// current partition value everywhere in the managed script, so a partitioned
// materialize can filter its source by the active slice, e.g.
// `WHERE day = {partition}`. The token is always replaced by a *complete*
// escaped SQL literal (`'…'` with `'` doubled) whether or not the author
// quoted it — so a run caller can't pass metacharacters that break out of
// the literal and alter statement boundaries. The pre-quoted form
// `'{partition}'` is matched first so it doesn't become `''…''`. Only
// meaningful when partitioned.
if partitioned {
let lit = format!("'{}'", partition.replace('\'', "''"));
let tok = windmill_common::assets::PARTITION_TOKEN;
let quoted_tok = format!("'{tok}'");
plan.output = plan.output.replace(&quoted_tok, &lit).replace(tok, &lit);
for s in plan.setup.iter_mut() {
*s = s.replace(&quoted_tok, &lit).replace(tok, &lit);
}
}
let strategy = if m.append {
MaterializeStrategy::Append
} else if let Some(uk) = m.unique_key {
MaterializeStrategy::Merge { unique_key: uk }
} else {
MaterializeStrategy::Replace
};
// Inline the partition as an escaped SQL literal (DuckLake has no bind for
// the partition column in our generated DDL).
let pval = format!("'{}'", partition.replace('\'', "''"));
let synthetic_attach = format!("ATTACH 'ducklake://{ducklake_name}' AS {TARGET_ALIAS};");
let blocks = build_wrap_blocks(
&plan,
&synthetic_attach,
table,
&m.target_path,
"_wm_partition",
&pval,
partitioned,
strategy,
);
Ok(Some((Some(blocks.join("\n")), meta)))
}
// Pull a named i64 field (`snapshot_id` / `rows`) out of the trailing summary
// read — which in wrap mode is the job result. Shape-tolerant (object / array /
// nested), returns None if absent (literal mode, or capture failed).
fn extract_i64(result: &RawValue, field: &str) -> Option<i64> {
fn find(v: &Value, field: &str) -> Option<i64> {
match v {
Value::Number(n) => n.as_i64(),
Value::Object(m) => m.get(field).and_then(|x| find(x, field)),
Value::Array(a) => a.iter().find_map(|x| find(x, field)),
_ => None,
}
}
find(&serde_json::from_str::<Value>(result.get()).ok()?, field)
}
// Best-effort record of a materialization outcome. On a Sql connection it writes
// the row directly; on an agent worker (Http, no direct DB) it posts to the API
// so state lands the same way. Never fails the job — a lost row degrades the
// grid, not the run.
async fn record_mat(
conn: &Connection,
w_id: &str,
job_id: Uuid,
meta: &MaterializeExec,
status: windmill_common::materialization::MaterializationStatus,
snapshot_id: Option<i64>,
row_count: Option<i64>,
error: Option<&str>,
) {
let req = windmill_common::materialization::RecordMaterializationRequest {
asset_kind: meta.asset_kind,
asset_path: meta.asset_path.clone(),
partition: meta.partition.clone(),
status,
snapshot_id,
row_count,
job_id: Some(job_id),
error: error.map(|e| e.to_string()),
};
let res: anyhow::Result<()> = match conn {
Connection::Sql(db) => windmill_common::materialization::record_materialization(
db,
w_id,
req.asset_kind,
&req.asset_path,
&req.partition,
req.status,
req.snapshot_id,
req.row_count,
req.job_id,
req.error.as_deref(),
)
.await
.map_err(|e| anyhow::anyhow!("{e:#}")),
Connection::Http(client) => {
crate::agent_workers::record_materialization_from_agent_http(client, w_id, &req).await
}
};
if let Err(e) = res {
tracing::warn!("failed to record materialization state: {e:#}");
}
}
pub async fn do_duckdb(
job: &MiniPulledJob,
client: &AuthedClient,
@@ -240,37 +68,7 @@ pub async fn do_duckdb(
let mut hidden_passwords = hidden_passwords.clone();
let mut bigquery_credentials = None;
// Materialization (`// materialize`): rewrite a wrap script into managed
// DDL (its synthetic target ATTACH is resolved by the transform pass
// below, like the user's own ATTACH); a literal script is left as-is.
// `materialize` also carries what to record once the run finishes.
let partition_value: Option<String> = job
.args
.as_ref()
.and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG))
.and_then(|rv| serde_json::from_str::<String>(rv.get()).ok())
.filter(|s| !s.is_empty());
let materialize = if query.contains("materialize") {
build_materialized_query(query, partition_value.as_deref())?
} else {
None
};
// Parse the signature from the ORIGINAL script: managed materialize wraps
// the trailing SELECT and strips line comments, which drops the
// `-- $name (type)` arg declarations while their `$name` references
// survive in the embedded SELECT. Parsing args here (pre-wrap) keeps them
// declared so they are still bound — and s3object args translated to
// `s3://` URIs — at run time.
let sig = parse_duckdb_sig(query)?.args;
let materialized_query;
let query: &str = match &materialize {
Some((Some(rewritten), _)) => {
materialized_query = rewritten.clone();
&materialized_query
}
_ => query,
};
let mut job_args = build_args_values(job, client, conn).await?;
let reserved_variables =
@@ -401,19 +199,6 @@ pub async fn do_duckdb(
let (result, column_order) = match result {
Ok(r) => r,
Err(e) => {
if let Some((_, meta)) = &materialize {
record_mat(
conn,
&job.workspace_id,
job.id,
meta,
windmill_common::materialization::MaterializationStatus::Failed,
None,
None,
Some(&e.to_string()),
)
.await;
}
if let Some(s3_proxy_err) = S3_PROXY_LAST_ERRORS_CACHE.get(&client.token) {
return Err(Error::ExecutionErr(format!(
"{}\n\nS3 Related Error: {}",
@@ -425,24 +210,6 @@ pub async fn do_duckdb(
}
};
if let Some((_, meta)) = &materialize {
// In wrap mode the job result is the summary read (snapshot_id +
// rows); in literal mode there is none, so both stay None.
let snapshot_id = extract_i64(&result, "snapshot_id");
let row_count = extract_i64(&result, "rows");
record_mat(
conn,
&job.workspace_id,
job.id,
meta,
windmill_common::materialization::MaterializationStatus::Materialized,
snapshot_id,
row_count,
None,
)
.await;
}
drop(bigquery_credentials);
*column_order_ref = column_order;
@@ -1112,40 +879,6 @@ mod tests {
);
}
// Managed `// materialize` may take SQL args (e.g. an s3object uploaded on
// the run form). The wrap strips line comments — including the
// `-- $name (type)` declarations — so the executor parses the signature from
// the original script (done above, before the rewrite) while the `$name`
// references survive inside the wrapped SELECT. This pins both halves of that
// contract so a regression that drops either is caught.
#[test]
fn materialize_preserves_sql_args() {
let script = "-- materialize ducklake://main/rows\n\
-- $file (s3object)\n\
SELECT * FROM read_json_auto($file)";
// The signature is recoverable from the original (un-wrapped) script.
let sig = parse_duckdb_sig(script).expect("sig parses").args;
let file_arg = sig
.iter()
.find(|a| a.name == "file")
.expect("`$file` declared");
assert_eq!(file_arg.otyp.as_deref(), Some("s3object"));
// The wrapped query still references `$file`, so the parsed sig binds it.
let (rewritten, _) = build_materialized_query(script, None)
.expect("materialize builds")
.expect("materialize present");
let rewritten = rewritten.expect("managed mode rewrites the query");
assert!(
rewritten.contains("$file"),
"wrapped query must keep the `$file` reference, got:\n{rewritten}"
);
// The declaration comment is gone (wrap strips line comments) — which is
// exactly why the sig must come from the original, not the rewrite.
assert!(!rewritten.contains("-- $file"));
}
// Tests for parse_attach_db_resource function
#[test]
fn test_parse_attach_db_resource_postgres_res_prefix() {
+27 -148
View File
@@ -44,7 +44,7 @@ use windmill_common::{
schema::{should_validate_schema, SchemaValidator},
utils::{create_directory_async, WarnAfterExt},
worker::{
is_allowed_file_location, make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT,
make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT,
MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, WINDMILL_DIR,
},
worker_group_job_stats::JobStatsMap,
@@ -4324,25 +4324,20 @@ async fn resolve_partition_for_job(
job: &MiniPulledJob,
code: &str,
conn: &Connection,
) -> error::Result<(Option<MiniPulledJob>, bool)> {
) -> error::Result<Option<MiniPulledJob>> {
use windmill_common::partition::{resolve_partition, PARTITION_ARG};
use windmill_parser::asset_parser::PartitionKind;
// Only deployed scripts participate in asset pipelines. Cheap substring
// guard so the overwhelming majority of script jobs skip the annotation
// scan; when one might be present we parse *once* here and reuse the result
// for both `in_pipeline` (→ WM_PIPELINE env, read by the wmll.ducklake SDK to
// record state) and `partition` resolution — no second parse downstream. The
// bool is whether the script is a `// pipeline` member.
if !matches!(job.kind, JobKind::Script)
|| !(code.contains("pipeline") || code.contains("partitioned"))
{
return Ok((None, false));
// Only deployed scripts participate in asset pipelines. Cheap
// substring guard so the overwhelming majority of script jobs (no
// `// partitioned` line) skip the full annotation scan on the hot
// path; a false positive only costs one extra parse, never wrong.
if !matches!(job.kind, JobKind::Script) || !code.contains("partitioned") {
return Ok(None);
}
let ann = windmill_parser::asset_parser::parse_pipeline_annotations(code);
let in_pipeline = ann.in_pipeline;
let Some(spec) = ann.partition else {
return Ok((None, in_pipeline));
let Some(spec) = windmill_parser::asset_parser::parse_pipeline_annotations(code).partition
else {
return Ok(None);
};
// Already resolved upstream — explicit run arg, backfill, or
@@ -4354,7 +4349,7 @@ async fn resolve_partition_for_job(
.is_some_and(|s| !s.is_empty())
});
if already_set {
return Ok((None, in_pipeline));
return Ok(None);
}
// `dynamic` extracts from the triggering payload (the `trigger` object
@@ -4387,7 +4382,7 @@ async fn resolve_partition_for_job(
job_id = %job.id,
"partitioned script resolved to no partition (before start anchor); running without one"
);
return Ok((None, in_pipeline));
return Ok(None);
};
// Persist back so dispatch_asset_triggers (which reads the producer's
@@ -4409,7 +4404,7 @@ async fn resolve_partition_for_job(
windmill_common::worker::to_raw_value(&value),
);
updated.args = Some(Json(map));
Ok((Some(updated), in_pipeline))
Ok(Some(updated))
}
#[tracing::instrument(level = "trace", skip_all)]
@@ -4571,8 +4566,7 @@ async fn handle_code_execution_job(
// `// partitioned` (if any) and shadow `job` with a clone whose args
// carry the resolved `partition` for the rest of execution.
let _job_with_partition;
let (resolved_job, in_pipeline) = resolve_partition_for_job(job, code, conn).await?;
let job = match resolved_job {
let job = match resolve_partition_for_job(job, code, conn).await? {
Some(j) => {
_job_with_partition = j;
&_job_with_partition
@@ -4625,153 +4619,48 @@ async fn handle_code_execution_job(
lock,
&modules,
false,
in_pipeline,
)
.await
}
/// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot
/// escape the directory it is joined onto (no `..`, no absolute root, no Windows
/// drive prefix).
fn is_contained_relative_path(path: &str) -> bool {
use std::path::Component;
std::path::Path::new(path)
.components()
.all(|c| matches!(c, Component::Normal(_) | Component::CurDir))
}
pub async fn write_module_files(
job_dir: &str,
modules: &std::collections::HashMap<String, ScriptModule>,
base_dir: Option<&str>,
) -> error::Result<()> {
// base_dir is derived from the runnable path, which on a preview run can
// carry `..` traversal (it is not the validated module-map key). Reject it
// before it is used to build any write target, otherwise a module could
// escape job_dir and write arbitrary files.
if let Some(dir) = base_dir {
if !is_contained_relative_path(dir) {
return Err(error::Error::BadRequest(format!(
"Invalid module base directory (path traversal): {dir}"
)));
}
}
for (relpath, module) in modules {
// Reject path traversal attempts in module paths (the module-map key).
if !is_contained_relative_path(relpath) {
// Reject path traversal attempts in module paths
if relpath.contains("..") {
tracing::warn!("Skipping module with path traversal: {relpath}");
continue;
}
let relpath_from_job_dir = match base_dir {
Some(dir) => format!("{}/{}", dir, relpath),
None => relpath.to_string(),
let full_path = match base_dir {
Some(dir) => format!("{}/{}/{}", job_dir, dir, relpath),
None => format!("{}/{}", job_dir, relpath),
};
// Authoritative guard: resolve the path and assert it stays inside job_dir.
let full_path = is_allowed_file_location(job_dir, &relpath_from_job_dir)?;
if let Some(parent) = full_path.parent() {
if let Some(parent) = std::path::Path::new(&full_path).parent() {
tokio::fs::create_dir_all(parent).await?;
}
// For Python modules, create __init__.py in each intermediate directory
// between base_dir and the module's parent so that relative imports work.
if let Some(dir) = base_dir {
let mut current = std::path::PathBuf::from(dir);
for component in std::path::Path::new(relpath)
.parent()
.into_iter()
.flat_map(|p| p.components())
{
let rel = std::path::Path::new(relpath);
let base = std::path::Path::new(job_dir).join(dir);
let mut current = base.clone();
for component in rel.parent().into_iter().flat_map(|p| p.components()) {
current = current.join(component);
let init_py = is_allowed_file_location(
job_dir,
&current.join("__init__.py").to_string_lossy(),
)?;
let init_py = current.join("__init__.py");
if !init_py.exists() {
tokio::fs::write(&init_py, "").await?;
}
}
}
tracing::debug!("Writing module file: {}", full_path.display());
tracing::debug!("Writing module file: {full_path}");
tokio::fs::write(&full_path, &module.content).await?;
}
Ok(())
}
#[cfg(test)]
mod write_module_files_tests {
use super::*;
use std::collections::HashMap;
use windmill_common::scripts::ScriptLang;
fn module(content: &str) -> ScriptModule {
ScriptModule { content: content.to_string(), language: ScriptLang::Python3, lock: None }
}
#[test]
fn contained_relative_path_rejects_traversal_and_absolute() {
assert!(is_contained_relative_path("u/admin/pkg"));
assert!(is_contained_relative_path("./pkg/sub"));
// A `..` in a filename is a valid name, not a traversal.
assert!(is_contained_relative_path("weird..name"));
assert!(!is_contained_relative_path("u/x/../../../etc"));
assert!(!is_contained_relative_path("../escape"));
assert!(!is_contained_relative_path("/etc/cron.d/wm"));
}
#[tokio::test]
async fn base_dir_traversal_is_rejected_and_writes_nothing() {
let job = tempfile::tempdir().unwrap();
let job_dir = job.path().to_str().unwrap();
// Sentinel just outside job_dir that a successful traversal would create.
let outside = job.path().parent().unwrap().join("wm_escaped_marker");
let mut modules = HashMap::new();
modules.insert(
"wm_escaped_marker".to_string(),
module("* * * * * root id\n"),
);
// base_dir derived from a preview path carrying `..` traversal.
let res = write_module_files(job_dir, &modules, Some("u/x/../../../../../..")).await;
assert!(res.is_err(), "traversal base_dir must be rejected");
assert!(!outside.exists(), "no file may be written outside job_dir");
}
#[tokio::test]
async fn relpath_traversal_is_skipped() {
let job = tempfile::tempdir().unwrap();
let job_dir = job.path().to_str().unwrap();
let outside = job.path().parent().unwrap().join("wm_relpath_escape.py");
let mut modules = HashMap::new();
modules.insert("../wm_relpath_escape.py".to_string(), module("x = 1"));
write_module_files(job_dir, &modules, None).await.unwrap();
assert!(!outside.exists());
}
#[tokio::test]
async fn legitimate_modules_are_written_with_init_py() {
let job = tempfile::tempdir().unwrap();
let job_dir = job.path().to_str().unwrap();
let mut modules = HashMap::new();
modules.insert("pkg/sub/mod.py".to_string(), module("VALUE = 42"));
write_module_files(job_dir, &modules, Some("u/admin"))
.await
.unwrap();
let base = job.path().join("u/admin");
assert_eq!(
std::fs::read_to_string(base.join("pkg/sub/mod.py")).unwrap(),
"VALUE = 42"
);
assert!(base.join("pkg/__init__.py").exists());
assert!(base.join("pkg/sub/__init__.py").exists());
}
}
pub async fn run_language_executor(
job: &MiniPulledJob,
conn: &Connection,
@@ -4796,9 +4685,6 @@ pub async fn run_language_executor(
lock: &Option<String>,
modules: &Option<std::collections::HashMap<String, ScriptModule>>,
run_inline: bool,
// Whether the script is a `// pipeline` member (parsed once upstream) — sets
// WM_PIPELINE so the wmll.ducklake SDK helpers record materialization state.
in_pipeline: bool,
) -> error::Result<Box<RawValue>> {
// Defense-in-depth (GHSA-wxjq-w5pj-jqhx): the entrypoint override is
// interpolated verbatim into a code position of the generated language
@@ -5161,11 +5047,6 @@ mount {{
#[allow(unused_mut)]
let mut envs = build_envs(envs.as_ref())?;
// Signal pipeline context to the script so the wmll.ducklake SDK helpers
// record materialization state (the grid/backfill) and skip it otherwise.
if in_pipeline {
envs.insert("WM_PIPELINE".to_string(), "true".to_string());
}
let Some(language) = language else {
return Err(Error::ExecutionErr(
@@ -5951,7 +5832,6 @@ pub fn init_worker_internal_server_inline_utils(
&None,
&None,
true,
false,
)
.await
})
@@ -6033,7 +5913,6 @@ pub fn init_worker_internal_server_inline_utils(
&content_info.lockfile,
&content_info.modules,
true,
false,
)
.await
})
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.737.0";
export const VERSION = "v1.733.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
+4 -74
View File
@@ -99,31 +99,6 @@ export interface VariableFile {
is_oauth?: boolean;
}
/**
* Whether `value` has the structural shape of a workspace-encrypted secret
* (the form produced by `sync pull` without --plain-secrets), as opposed to a
* plaintext value a user authored by hand.
*
* Mirrors the server guard (windmill-store/src/variables.rs): workspace
* ciphertext (AES-256-CBC, base64) is standard base64 decoding to a non-zero
* multiple of the 16-byte block size. External secret-backend markers
* ($vault:/$aws_sm:/$azure_kv:) are stored verbatim too, so they count as
* already-encrypted. This is a shape check only it never decrypts.
*/
export function looksLikeWorkspaceCiphertext(value: string): boolean {
if (
value.startsWith("$vault:") ||
value.startsWith("$aws_sm:") ||
value.startsWith("$azure_kv:")
) {
return true;
}
if (value.length === 0 || value.length % 4 !== 0) return false;
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(value)) return false;
const decodedLen = Buffer.from(value, "base64").length;
return decodedLen > 0 && decodedLen % 16 === 0;
}
export async function pushVariable(
workspace: string,
remotePath: string,
@@ -131,11 +106,6 @@ export async function pushVariable(
localVariable: VariableFile,
plainSecrets: boolean,
wsSpecific?: boolean,
// Whether a secret->non-secret downgrade may be applied. Only an authoritative
// single-file `variable push` sets this. Bulk `sync push` leaves it false: a
// pulled secret's spec value is ciphertext, and demoting it would store that
// ciphertext verbatim as a visible non-secret value.
allowSecretDowngrade: boolean = false,
): Promise<void> {
remotePath = removeType(remotePath, "variable");
log.debug(`Processing local variable ${remotePath}`);
@@ -160,26 +130,14 @@ export async function pushVariable(
log.debug(`Variable ${remotePath} is not up-to-date, updating`);
// Apply is_secret only when it differs from the remote (the value is always
// sent, so the server allows the flag change). Upgrades (non-secret->secret)
// always apply; downgrades only when explicitly allowed (single-file push) —
// see allowSecretDowngrade. `undefined` leaves the flag untouched.
let nextIsSecret: boolean | undefined = undefined;
if (localVariable.is_secret !== variable.is_secret) {
if (localVariable.is_secret) {
nextIsSecret = true;
} else if (allowSecretDowngrade) {
nextIsSecret = false;
}
}
await wmill.updateVariable({
workspace,
path: remotePath.replaceAll(SEP, "/"),
alreadyEncrypted: !plainSecrets,
requestBody: {
...localVariable,
is_secret: nextIsSecret,
is_secret:
localVariable.is_secret && !variable.is_secret ? true : undefined,
...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}),
},
});
@@ -216,40 +174,12 @@ async function push(
log.info(colors.bold.yellow("Pushing variable..."));
const local = parseFromFile(filePath) as VariableFile;
// A secret value in a single-file push is authored by the user and is
// therefore plaintext that must be encrypted server-side — unless it has the
// shape of workspace ciphertext (a value round-tripped from `sync pull`).
// Pushing plaintext as already-encrypted would brick the variable. An explicit
// --plain-secrets always forces the plaintext (encrypt) path.
let plainSecrets = opts.plainSecrets ?? false;
if (opts.plainSecrets === undefined && local.is_secret) {
if (!looksLikeWorkspaceCiphertext(local.value)) {
log.info(
colors.yellow(
"Secret value is not in encrypted form; pushing as plaintext to be encrypted server-side (pass --plain-secrets to silence)."
)
);
plainSecrets = true;
} else {
// The value has the shape of workspace ciphertext, so it's stored as-is.
// A plaintext secret that coincidentally looks like ciphertext (e.g. a
// base64 token) would be stored unreadable, so surface the assumption.
log.warn(
"Secret value looks already-encrypted; pushing it as-is. If it is a plaintext secret, re-run with --plain-secrets so it gets encrypted."
);
}
}
await pushVariable(
workspace.workspaceId,
remotePath,
undefined,
local,
plainSecrets,
undefined,
true // single-file push is authoritative: allow secret->non-secret downgrade
parseFromFile(filePath),
opts.plainSecrets ?? false
);
log.info(colors.bold.underline.green(`Variable ${remotePath} pushed`));
}
+5 -5
View File
@@ -104,25 +104,25 @@ export async function requireLogin(
// 403 means the token authenticated but lacks scope — re-issuing
// won't help. Keep this distinct from the 401 message so the user
// doesn't waste time reproducing the token.
log.infoStderr(colors.red(
log.info(colors.red(
`Permission denied: the token is valid but lacks the required scope.${bodyStr ? `\n${bodyStr}` : ""}`
));
} else if (status === 401) {
log.infoStderr(colors.red(
log.info(colors.red(
`Could not authenticate with the provided credentials. Please check your --token and --base-url and try again.${bodyStr ? `\n${bodyStr}` : ""}`
));
} else {
log.infoStderr(colors.red(
log.info(colors.red(
`Request failed (${status ?? "unknown"}): ${bodyStr}`
));
}
return process.exit(1);
}
log.infoStderr(colors.red("Could not authenticate with the provided credentials. Please check your --token and --base-url and try again."));
log.info(colors.red("Could not authenticate with the provided credentials. Please check your --token and --base-url and try again."));
return process.exit(1);
}
log.infoStderr(
log.info(
"! Could not reach API given existing credentials. Attempting to reauth..."
);
const newToken = await loginInteractive(workspace.remote);
+1 -1
View File
@@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork";
// (e.g. utils.ts) can read it without importing main.ts and creating a circular
// dependency (main → workspace → utils → main) that triggers a TDZ.
// Re-exported from main.ts for backwards compatibility.
export const VERSION = "1.737.0";
export const VERSION = "1.733.1";
+29 -29
View File
@@ -57,7 +57,7 @@ async function selectFromMultipleProfiles(
(p) => p.name === lastUsedProfileName
);
if (lastUsedProfile) {
log.infoStderr(
log.info(
colors.green(
`Using last used profile '${lastUsedProfile.name}' for ${context}`
)
@@ -69,7 +69,7 @@ async function selectFromMultipleProfiles(
// No last used or it no longer exists - prompt for selection
if (!!!process.stdin.isTTY || !!!process.stdout.isTTY) {
const selectedProfile = profiles[0];
log.infoStderr(
log.info(
colors.yellow(
`Multiple profiles found for ${context}. Using first available profile: '${selectedProfile.name}'`
)
@@ -87,7 +87,7 @@ async function selectFromMultipleProfiles(
return selectedProfile;
}
log.infoStderr(
log.info(
colors.yellow(`\nMultiple workspace profiles found for ${context}:`)
);
@@ -125,14 +125,14 @@ async function createWorkspaceProfileInteractively(
): Promise<Workspace | undefined> {
// Log appropriate message based on context
if (!context.isForked) {
log.infoStderr(
log.info(
colors.yellow(
`\nNo workspace profile found for branch '${context.rawBranch}'\n` +
`(${normalizedBaseUrl}, ${workspaceId})`
)
);
} else {
log.infoStderr(
log.info(
colors.yellow(
`\nNo workspace profile was found for this forked workspace\n` +
`(${normalizedBaseUrl}, ${workspaceId})`
@@ -141,7 +141,7 @@ async function createWorkspaceProfileInteractively(
}
if (!!!process.stdin.isTTY || !!!process.stdout.isTTY) {
log.infoStderr(
log.info(
"Not a TTY, cannot create profile interactively. Use 'wmill workspace add' first."
);
return undefined;
@@ -187,12 +187,12 @@ async function createWorkspaceProfileInteractively(
opts.configDir
);
log.infoStderr(
log.info(
colors.green(
`✓ Created profile '${profileName}' for ${workspaceId} on ${normalizedBaseUrl}`
)
);
log.infoStderr(colors.green(`✓ Profile '${profileName}' is now active`));
log.info(colors.green(`✓ Profile '${profileName}' is now active`));
return newWorkspace;
}
@@ -244,7 +244,7 @@ async function tryResolveWorkspace(
`workspace '${opts.workspace}'`,
opts.configDir
);
log.infoStderr(
log.info(
colors.green(
`Using workspace profile '${selected.name}' for workspace '${opts.workspace}' (${workspaceId} on ${normalizedBaseUrl})`
)
@@ -254,7 +254,7 @@ async function tryResolveWorkspace(
}
// No matching profile — offer to create one
log.infoStderr(
log.info(
`No profile found for workspace '${opts.workspace}' (${workspaceId} on ${normalizedBaseUrl})`
);
const ws = await createWorkspaceProfileInteractively(
@@ -309,7 +309,7 @@ export async function tryResolveBranchWorkspace(
wsEntry = config.workspaces?.[workspaceNameOverride] as WorkspaceEntryConfig | undefined;
if (wsEntry) {
wsName = workspaceNameOverride;
log.infoStderr(`Using workspace override: ${workspaceNameOverride}`);
log.info(`Using workspace override: ${workspaceNameOverride}`);
}
} else {
// Only try branch-based resolution if in a Git repository
@@ -328,7 +328,7 @@ export async function tryResolveBranchWorkspace(
const branchToLookup = originalBranchIfForked ?? rawBranch;
if (originalBranchIfForked) {
log.infoStderr(
log.info(
`Using original branch \`${originalBranchIfForked}\` for finding workspace from workspaces section in wmill.yaml`
);
}
@@ -346,7 +346,7 @@ export async function tryResolveBranchWorkspace(
if (!wsEntry.baseUrl) {
if (workspaceNameOverride) {
// User explicitly asked for this workspace but it has no baseUrl
log.warnStderr(
log.warn(
`⚠️ Workspace '${wsName}' has no baseUrl configured. Cannot resolve a profile.\n` +
` Add baseUrl to workspace '${wsName}' in wmill.yaml, or use --base-url flag.`
);
@@ -370,7 +370,7 @@ export async function tryResolveBranchWorkspace(
reason = `matched current git branch '${rawBranch}'`;
}
log.infoStderr(
log.info(
`Using workspace '${wsName}' (${reason}) → ${workspaceId} on ${baseUrl}`
);
@@ -406,7 +406,7 @@ export async function tryResolveBranchWorkspace(
if (matchingProfiles.length === 1) {
selectedProfile = matchingProfiles[0];
log.infoStderr(
log.info(
colors.green(
`Using workspace profile '${selectedProfile.name}' for workspace '${wsName}' with workspace id \`${workspaceId}\``
)
@@ -424,7 +424,7 @@ export async function tryResolveBranchWorkspace(
(p) => p.name === lastUsedName
);
if (lastUsedProfile) {
log.infoStderr(
log.info(
colors.green(
`Using workspace profile '${lastUsedProfile.name}' for workspace '${wsName}' (last used)`
)
@@ -449,7 +449,7 @@ export async function tryResolveBranchWorkspace(
opts.configDir
);
log.infoStderr(
log.info(
colors.green(
`Using workspace profile '${selectedProfile.name}' for workspace '${wsName}'`
)
@@ -459,7 +459,7 @@ export async function tryResolveBranchWorkspace(
if (workspaceIdIfForked) {
selectedProfile.name = `${selectedProfile.name}/${workspaceIdIfForked}`;
selectedProfile.workspaceId = workspaceIdIfForked;
log.infoStderr(
log.info(
`Using fork workspace \`${workspaceIdIfForked}\` (parent: \`${workspaceId}\`) from branch \`${rawBranch}\``
);
}
@@ -480,7 +480,7 @@ export async function resolveWorkspace(
try {
normalizedBaseUrl = new URL(opts.baseUrl).toString();
} catch (error) {
log.infoStderr(colors.red(`Invalid base URL: ${opts.baseUrl}`));
log.info(colors.red(`Invalid base URL: ${opts.baseUrl}`));
return process.exit(-1);
}
@@ -514,7 +514,7 @@ export async function resolveWorkspace(
if (existingWorkspace) {
if (existingWorkspace.remote !== normalizedBaseUrl) {
log.infoStderr(
log.info(
colors.red(
`Base URL mismatch: --base-url is ${normalizedBaseUrl} but workspace profile "${opts.workspace}" uses ${existingWorkspace.remote}`
)
@@ -535,7 +535,7 @@ export async function resolveWorkspace(
token: opts.token,
};
} else {
log.infoStderr(
log.info(
colors.red(
"If you specify a base URL with --base-url, you must also specify a workspace (--workspace) and token (--token)."
)
@@ -555,7 +555,7 @@ export async function resolveWorkspace(
if (workspaceNameOverride || opts.workspace || !branch || !branch.startsWith(WM_FORK_PREFIX)) {
return workspace;
} else {
log.infoStderr(
log.info(
`Found an active workspace \`${workspace.name}\` but the branch name indicates this is a forked workspace. Ignoring active workspace and trying to resolve the correct workspace from the branch name \`${branch}\`. Use --workspace to override.`
);
}
@@ -572,9 +572,9 @@ export async function resolveWorkspace(
if (suggestions.length > 0) {
msg += ` Did you mean: ${suggestions.map((s) => `"${s.name}"`).join(", ")}?`;
}
log.infoStderr(colors.red.bold(msg));
log.info(colors.red.bold(msg));
if (profiles.length > 0) {
log.infoStderr("\nAvailable workspaces:");
log.info("\nAvailable workspaces:");
new Table()
.header(["name", "remote", "workspace id"])
.padding(2)
@@ -620,12 +620,12 @@ export async function resolveWorkspace(
if (wsNames.length === 1) {
pickedWsName = wsNames[0];
log.infoStderr(
log.info(
`Auto-selected workspace '${pickedWsName}' (only workspace in config).\n` +
`Use --workspace to override or 'wmill workspace bind' to add more workspaces.`
);
} else if (process.stdin.isTTY) {
log.infoStderr(
log.info(
`Multiple workspaces configured but none matched the current context.\n` +
`Configured workspaces:\n${wsListStr}\n` +
`Use --workspace to skip this prompt.`
@@ -675,7 +675,7 @@ export async function resolveWorkspace(
try {
normalizedBaseUrl = new URL(envBaseUrl).toString();
} catch {
log.infoStderr(colors.red(`Invalid BASE_INTERNAL_URL: ${envBaseUrl}`));
log.info(colors.red(`Invalid BASE_INTERNAL_URL: ${envBaseUrl}`));
return process.exit(-1);
}
log.debug(
@@ -691,7 +691,7 @@ export async function resolveWorkspace(
return ws;
}
log.infoStderr(colors.red.bold("No workspace given and no default set. Run 'wmill workspace add' to configure one."));
log.info(colors.red.bold("No workspace given and no default set. Run 'wmill workspace add' to configure one."));
return process.exit(-1);
}
@@ -746,7 +746,7 @@ export async function tryResolveVersion(
export function validatePath(path: string): boolean {
if (!(path.startsWith("g") || path.startsWith("u") || path.startsWith("f"))) {
log.infoStderr(
log.info(
colors.red(
"Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
)
-15
View File
@@ -21,26 +21,11 @@ export function info(msg: unknown) {
console.log(`\x1b[34m${String(msg)}\x1b[39m`);
}
// Like `info` but written to stderr, for diagnostics (e.g. the workspace-profile
// banner printed on every command) that must not pollute stdout when a command's
// data output is piped or redirected (e.g. `wmill variable get path > file`).
export function infoStderr(msg: unknown) {
if (silentMode) return;
console.error(`\x1b[34m${String(msg)}\x1b[39m`);
}
export function warn(msg: unknown) {
if (silentMode) return;
console.log(`\x1b[33m${String(msg)}\x1b[39m`);
}
// Like `warn` but written to stderr; see `infoStderr` for why diagnostics must
// not land on stdout.
export function warnStderr(msg: unknown) {
if (silentMode) return;
console.error(`\x1b[33m${String(msg)}\x1b[39m`);
}
export function error(msg: unknown) {
console.error(`\x1b[31m${String(msg)}\x1b[39m`);
}
+4 -4
View File
@@ -10,7 +10,7 @@ import * as http from "node:http";
export async function loginInteractive(remote: string) {
let token: string | undefined;
if (!process.stdin.isTTY) {
log.infoStderr("Not a TTY, can't login interactively.");
log.info("Not a TTY, can't login interactively.");
return undefined;
}
if (
@@ -55,7 +55,7 @@ export async function browserLogin(
const port = await getPort.default({ port: env });
if (port == undefined) {
log.infoStderr(colors.red.underline("failed to aquire port"));
log.info(colors.red.underline("failed to aquire port"));
return undefined;
}
@@ -79,7 +79,7 @@ export async function browserLogin(
});
const url = `${baseUrl}user/cli?port=${port}`;
log.infoStderr(`Login by going to ${url}`);
log.info(`Login by going to ${url}`);
try {
open.default(url).catch((error) => {
@@ -88,7 +88,7 @@ export async function browserLogin(
);
});
log.infoStderr("Opened browser for you");
log.info("Opened browser for you");
} catch (error) {
console.error(
`Failed to open browser, please navigate to ${url}, error: ${error}`
-90
View File
@@ -1102,29 +1102,6 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
/**
* Idempotently materialize \`selectSql\` into a ducklake table for one
* partition (or the whole table when \`partition\` is omitted) — the client-side
* equivalent of the \`// materialize\` engine.
* With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
* replaces it (whole table \`CREATE OR REPLACE\`; partition → delete + insert).
* Safe to re-run for the same partition (backfill / failure-recovery).
*
* Returns a lazy statement call \`.execute()\` to run it:
* \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
*/
upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
/**
* INSERT-only materialization (no dedup/replace) for append-only tables.
* Re-running the same partition duplicates rows use only for immutable
* event-log sources.
*
* Returns a lazy statement call \`.execute()\` to run it:
* \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
*/
appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
`,
"write-script-bunnative": `---
name: write-script-bunnative
@@ -1856,29 +1833,6 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
/**
* Idempotently materialize \`selectSql\` into a ducklake table for one
* partition (or the whole table when \`partition\` is omitted) — the client-side
* equivalent of the \`// materialize\` engine.
* With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
* replaces it (whole table \`CREATE OR REPLACE\`; partition → delete + insert).
* Safe to re-run for the same partition (backfill / failure-recovery).
*
* Returns a lazy statement call \`.execute()\` to run it:
* \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
*/
upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
/**
* INSERT-only materialization (no dedup/replace) for append-only tables.
* Re-running the same partition duplicates rows use only for immutable
* event-log sources.
*
* Returns a lazy statement call \`.execute()\` to run it:
* \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
*/
appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
`,
"write-script-csharp": `---
name: write-script-csharp
@@ -2702,29 +2656,6 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
/**
* Idempotently materialize \`selectSql\` into a ducklake table for one
* partition (or the whole table when \`partition\` is omitted) — the client-side
* equivalent of the \`// materialize\` engine.
* With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
* replaces it (whole table \`CREATE OR REPLACE\`; partition → delete + insert).
* Safe to re-run for the same partition (backfill / failure-recovery).
*
* Returns a lazy statement call \`.execute()\` to run it:
* \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
*/
upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
/**
* INSERT-only materialization (no dedup/replace) for append-only tables.
* Re-running the same partition duplicates rows use only for immutable
* event-log sources.
*
* Returns a lazy statement call \`.execute()\` to run it:
* \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
*/
appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
`,
"write-script-duckdb": `---
name: write-script-duckdb
@@ -4375,27 +4306,6 @@ def stream_result(stream) -> None
# SqlQuery instance for fetching results
def query(sql: str, *args) -> SqlQuery
# Idempotently materialize the rows of \`select_sql\` into ducklake
# \`table\` for one \`partition\` (or the whole table when \`partition\` is
# None). Client-side equivalent of the \`// materialize\` engine: with
# \`unique_key\` it upserts within the slice (delete-by-key + insert);
# without it, it replaces (whole table CREATE OR REPLACE; partition
# delete the partition + insert). Re-running the same slice is safe the
# backfill / failure-recovery contract.
#
# The partition value is bound as a DuckDB arg (never string-interpolated)
# so it cannot inject SQL. \`select_sql\` is trusted (your own query).
def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)
# INSERT-only materialization (no dedup / no replace) for an immutable
# event-log table for one \`partition\`, or the whole table when
# \`partition\` is None. NOTE: unlike \`upsert_partition\`, re-running the same
# slice duplicates rows use only for append-only sources.
def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
# Read a materialized ducklake table, optionally a single partition.
def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
# Execute query and fetch results.
#
# Args:
@@ -1,42 +0,0 @@
import { expect, test } from "bun:test";
import { looksLikeWorkspaceCiphertext } from "../src/commands/variable/variable.ts";
// =============================================================================
// looksLikeWorkspaceCiphertext drives whether single-file `variable push` treats
// a secret's value as already-encrypted (store verbatim) or as plaintext to be
// encrypted server-side. It must agree with the server guard
// (validate_already_encrypted_secret in windmill-store/src/variables.rs): a value
// is "ciphertext shaped" iff it is an external-backend marker, or standard base64
// decoding to a non-zero multiple of the AES block size (16 bytes).
// =============================================================================
test("treats workspace-ciphertext-shaped values as already-encrypted", () => {
const ciphertextShaped = [
"MpYeXnSBBF7dzI6K8J89xQ==", // real magic_crypt output: 16 bytes
Buffer.alloc(16, 7).toString("base64"), // 16 bytes
Buffer.alloc(32, 7).toString("base64"), // 32 bytes
"$vault:f/x/cfg",
"$aws_sm:f/x/cfg",
"$azure_kv:f/x/cfg",
];
for (const value of ciphertextShaped) {
expect(looksLikeWorkspaceCiphertext(value)).toBe(true);
}
});
test("treats hand-authored plaintext as NOT already-encrypted", () => {
const plaintext = [
"some: plaintext\n", // space, colon, newline
"original-secret", // hyphen, not length % 4
"hunter2",
'{"a": 1}',
"", // empty
"dGVzdA==", // valid base64 but decodes to 4 bytes (not % 16)
Buffer.alloc(17, 7).toString("base64"), // 17 bytes (not % 16)
"$omething-plain", // starts with $ but is not a real backend marker
];
for (const value of plaintext) {
expect(looksLikeWorkspaceCiphertext(value)).toBe(false);
}
});
-97
View File
@@ -219,103 +219,6 @@ describe("variable", () => {
});
});
test("push encrypts a plaintext secret value (no --plain-secrets) and round-trips", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
const uniqueId = Date.now();
const varPath = `f/test/sec_push_${uniqueId}`;
// Existing secret variable (server-encrypted).
const createResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: varPath,
value: "original-secret",
is_secret: true,
description: "",
}),
}
);
expect(createResp.status).toBeLessThan(300);
await createResp.text();
// A hand-authored spec file: plaintext value, is_secret: true. Pushing it
// without --plain-secrets must encrypt the value server-side, not store the
// plaintext verbatim as ciphertext (which would make every read fail).
const specPath = join(tempDir, "v.yaml");
await writeFile(
specPath,
`value: |\n some: plaintext\nis_secret: true\ndescription: ""\n`,
"utf-8"
);
const pushResult = await backend.runCLICommand(
["variable", "push", specPath, varPath],
tempDir
);
expect(pushResult.code).toEqual(0);
// The value must decrypt cleanly to the pushed plaintext.
const apiResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/get/${varPath}?decrypt_secret=true`
);
expect(apiResp.status).toEqual(200);
const varData = await apiResp.json();
expect(varData.is_secret).toBe(true);
expect(varData.value).toBe("some: plaintext\n");
});
});
test("push flips is_secret from true to false", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
const uniqueId = Date.now();
const varPath = `f/test/sec_down_${uniqueId}`;
const createResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: varPath,
value: "original-secret",
is_secret: true,
description: "",
}),
}
);
expect(createResp.status).toBeLessThan(300);
await createResp.text();
const specPath = join(tempDir, "v_down.yaml");
await writeFile(
specPath,
`value: "now-public"\nis_secret: false\ndescription: ""\n`,
"utf-8"
);
const pushResult = await backend.runCLICommand(
["variable", "push", specPath, varPath],
tempDir
);
expect(pushResult.code).toEqual(0);
const apiResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/get/${varPath}?decrypt_secret=true`
);
expect(apiResp.status).toEqual(200);
const varData = await apiResp.json();
expect(varData.is_secret).toBe(false);
expect(varData.value).toBe("now-public");
});
});
test("pull retrieves variables into local files", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
-250
View File
@@ -1,250 +0,0 @@
# DuckLake-native materialization
Design sketch for "managed, versioned, incremental" assets built on the
DuckLake substrate. This is a companion to [`pipelines-vs-dbt.md`](./pipelines-vs-dbt.md)
and extends its **Path C (hybrid, partition-first)** recommendation. The new
contribution here is leveraging DuckLake's snapshot/time-travel layer, which
the earlier doc's incremental deep-dive did not use. The annotation grammar is
reconciled with that doc — `// partitioned` + `// unique_key` + `// append`
stay canonical; nothing here forks a competing vocabulary.
## The core reframe
dbt had to *build* a materialization engine (compile SQL → `CREATE TABLE AS` /
incremental `MERGE` / SCD2 snapshots) because the warehouse gives it nothing
but raw SQL over a mutable table. DuckLake hands us, at the storage layer, the
four things that engine exists to provide:
| Capability | Source |
|---|---|
| ACID multi-statement transactions | DuckLake |
| A snapshot per commit + time-travel (`AT (VERSION => n)` / `AT (TIMESTAMP => ...)`) | DuckLake |
| Physical partitioning + pruning (`ALTER TABLE … SET PARTITIONED BY (…)`) | DuckLake |
| Schema evolution tracked in the catalog DB | DuckLake |
Windmill already attaches DuckLake fully — see `transform_attach_ducklake`
(`backend/windmill-worker/src/duckdb_executor.rs:661`), which rewrites a user's
`ATTACH 'ducklake://name' AS dl` into the real
`ATTACH 'ducklake:postgres:…' AS dl (DATA_PATH 's3://…', OVERRIDE_DATA_PATH
TRUE, AUTOMATIC_MIGRATION TRUE)` at `duckdb_executor.rs:730`. But today every
write is a destructive overwrite and **all four capabilities above are thrown
away** — snapshots are never surfaced, partitioning is purely an orchestration
concept disconnected from the physical layout.
So the materialization engine we need is a thin layer — a write-strategy
wrapper + snapshot capture — not a dbt rebuild. This is exactly the
"buy A's 80% without B's dialect-rewriting tax" tradeoff `pipelines-vs-dbt.md`
argued for; DuckLake is what makes the remaining 20% (versioning, reproducible
reads, materialization history) nearly free instead of a second project.
## Annotation grammar (final)
One self-documenting line; managed-by-default. Strategy options live *on* the
`materialize` line (they have no meaning without it), while `// partitioned`
stays separate because it is cross-cutting (cascade + scheduling + materialize).
```
// materialize ducklake://analytics/orders_daily → managed, replace (default)
// materialize ducklake://analytics/orders_daily key=order_id → managed, merge
// materialize ducklake://analytics/orders_daily append → managed, append
// materialize manual ducklake://analytics/orders_daily → track-only escape hatch
```
- **managed (default)** — the script is *setup + one trailing `SELECT`*; Windmill
generates the write DDL, captures the DuckLake snapshot, and records state.
DuckDB-only; validated at deploy (a non-SELECT script is rejected with a clear
error pointing to the `wmll.ducklake` helpers).
- **`manual`** — escape hatch: the script writes its own DDL; Windmill only
records state (no snapshot capture, no idempotency guarantee). Rare; explicit.
- **`key=<col>`** → MERGE (dedup within slice); **`append`** → INSERT-only;
neither → DELETE-by-partition + INSERT (replace). `append` wins over `key` if
both are given (deploy warning).
- **`// partitioned <kind>`** — unit of work + state + backfill (separate;
cross-cutting). Polyglot / multi-statement writes use the `wmll.ducklake`
helpers instead of `// materialize`.
There is no `wrap` keyword — `materialize` *is* "manage the write," so it was
redundant; the only reason for it was to carve out the weak track-only mode,
which is now the explicit `manual` opt-out.
DuckLake snapshots are **orthogonal to all of the above** — they apply to every
strategy automatically because every write is a DuckLake commit. The user never
annotates for versioning; they get it.
## Executor codegen
### The seam
`run_duckdb` already splits the script into statement blocks and rewrites
custom `ATTACH` blocks in a single pass before execution
(`duckdb_executor.rs:114-160`):
```rust
let query_block_list = parse_sql_blocks(&query, true);
// each block: remove_comments → if ducklake/datatable ATTACH, expand; else passthrough
```
All blocks run in order on one DuckDB connection. Materialize has two modes:
1. **Managed (default).** The user writes *setup + one trailing `SELECT`*. Windmill
replaces that SELECT with generated statements, wrapped in an *explicit
DuckLake transaction it controls* — never textual `BEGIN/COMMIT` injected
around the user's other statements (fragile across their own `ATTACH`s and
multi-statement SQL). For a partitioned `replace` the SELECT block expands to:
```sql
-- generated for: // partitioned daily ; target = dl.orders_daily ; partition = '2026-06-19'
CREATE TABLE IF NOT EXISTS dl.orders_daily AS
SELECT *, CAST(NULL AS VARCHAR) AS _wm_partition FROM (<user_select>) WHERE false; -- first-run bootstrap
ALTER TABLE dl.orders_daily SET PARTITIONED BY (_wm_partition);
BEGIN TRANSACTION;
DELETE FROM dl.orders_daily WHERE _wm_partition = '2026-06-19';
INSERT INTO dl.orders_daily SELECT *, '2026-06-19' AS _wm_partition FROM (<user_select>);
COMMIT;
```
The strategy variants are all DELETE+INSERT-shaped — no `MERGE INTO`, which
DuckLake can't reliably run on a fresh partition (it 404s writing the first
rows):
- **whole-table replace** (no `// partitioned`) → a single `CREATE OR REPLACE
TABLE … AS <user_select>` (handles schema changes, still snapshots).
- **`key=<col>`** → `DELETE FROM … WHERE [<partition> AND] <col> IN (SELECT
<col> FROM (<user_select>))` then `INSERT` (upsert within the slice).
- **`append`** → the `DELETE` is dropped (insert-only).
2. **`manual`.** The user writes their own DDL inside their own `BEGIN … COMMIT`;
Windmill injects nothing into the body and only records state (no snapshot
capture, no idempotency guarantee).
The `_wm_partition` column is the physical link the orchestration layer lacks: on
first materialize Windmill runs `ALTER TABLE … SET PARTITIONED BY (_wm_partition)`
so DuckLake prunes on read and DELETE-by-partition rewrites only that partition's
Parquet files.
> Storage: DuckLake writes go to `s3://_default_/` through the windmill S3 proxy
> (`/api/w/{ws}/s3_proxy`, gated behind the `parquet` + `private` features). The
> proxy must sign the SigV4 canonical URI with **single** percent-encoding — the
> SigV4 default (`Double`) 401s Hive-partition keys like `_wm_partition=2026-06-19`
> (the `=` double-encodes to `%253D` vs the client's `%3D`).
### Run summary capture
After the generated blocks, Windmill appends one read block — it is both the job's
result (a useful preview rendered as the materialized table) and the row it records:
```sql
SELECT 'ducklake://<name>/<table>' AS materialized,
'<partition>' AS partition, -- only when partitioned
(SELECT count(*) FROM <target> [WHERE _wm_partition = '<partition>']) AS rows,
(SELECT max(snapshot_id) FROM ducklake_snapshots('<target>')) AS snapshot_id;
```
The `snapshot_id` and `rows` are persisted as `materialized_partition` metadata.
One extra round-trip per materialization, no new infra.
## Metadata schema
Extends the `materialized_partitions` table proposed in `pipelines-vs-dbt.md`
§"First implementation slice" with the DuckLake snapshot id:
```
materialized_partition (
workspace_id TEXT,
asset_kind TEXT, -- 'ducklake'
asset_path TEXT, -- 'analytics/orders_daily'
partition TEXT, -- '2026-06-19' (NULL for unpartitioned)
snapshot_id BIGINT, -- DuckLake snapshot produced by this materialize
row_count BIGINT,
job_id UUID,
materialized_at TIMESTAMPTZ,
PRIMARY KEY (workspace_id, asset_kind, asset_path, partition)
)
```
This one table drives four things at once:
- **Observability** — "last materialized: snapshot 42, 1.2M rows, 09:14" per
asset node (closes the Dagster-catalog gap from the v1-readiness review).
- **Run-stale / gap detection** — which partitions exist, which are missing.
- **Backfill** — the missing/failed set *is* the backfill worklist.
- **Snapshot pinning** — see below.
## Reproducibility — the beyond-dbt part
Because every materialization records the snapshot it produced, a downstream
consumer can read the *exact* upstream snapshot its run saw:
```sql
FROM dl.orders_daily AT (VERSION => $WM_UPSTREAM_SNAPSHOT)
```
The cascade already threads a `trigger` blob (producer path, partition) to each
subscriber; add the producer's captured `snapshot_id` to it, and a consumer's
read is pinned to the upstream state at dispatch time. That makes the *whole
pipeline* reproducible and time-travelable — something dbt has no native answer
for (dbt models are always "whatever's in the warehouse now"). It also gives
rollback (re-point an asset to snapshot N) and "what did this table look like at
the failing run" debugging, for free off the same captured ids.
This is the differentiator worth leaning on. It is not catch-up to dbt; it is a
capability dbt structurally cannot offer, and DuckLake gives it to us at the
cost of recording one integer per run.
It also means **we do not build SCD2 snapshots** (gap #4 in `pipelines-vs-dbt.md`):
DuckLake time-travel is a strictly better answer for most of what dbt's
`{% snapshot %}` is used for. One fewer engine to write.
## Scoping decision: DuckLake vs DataTable
**Make DuckLake the materialization/versioning substrate; keep DataTable as the
live operational table with no versioning.** DataTable is plain Postgres
(`transform_attach_datatable`, `duckdb_executor.rs:742`) — no native snapshots
or time-travel — so giving *it* the versioned/incremental story means building
MVCC-on-top ourselves (history tables, SCD2), precisely the complexity this
DuckLake approach exists to avoid. Clean split:
- `ducklake://` → analytics, versioned, reproducible, backfillable.
- `datatable://` → mutable app/operational state; partition idempotency via
DELETE+INSERT still works, but no snapshot/time-travel layer.
Don't try to give both the full treatment for v1.
## v1 slice (smallest viable)
1. **Partition runtime context** — resolve `(value, start, end)` and surface as
`WM_PARTITION*` bind/env (Path C step 1; partly built per the
pipeline-partition-runtime work).
2. **Physical partition wiring**`_wm_partition` column + `SET PARTITIONED BY`
on first materialize for `ducklake://` targets.
3. **Strategy templates** — DELETE+INSERT default (`CREATE OR REPLACE` for the
whole table); delete-by-key + insert when `key=<col>`; INSERT-only when
`append`. Managed `// materialize` wraps a single-SELECT DuckDB script behind
these templates; `// materialize manual` opts out.
4. **Snapshot + metadata capture** — append `ducklake_snapshots` read, persist
`materialized_partition` rows.
5. **Surface it** — last-materialized/snapshot/row-count on the asset node;
missing-partition set feeds the backfill UI.
6. *v1.x* — snapshot pinning across the cascade (`$WM_UPSTREAM_SNAPSHOT`),
rollback, time-travel read helper.
Steps 15 are a thin annotation+template layer plus one metadata table and one
extra read per run. They deliver managed/incremental/versioned assets,
idempotent partitioned materialization, the backfill substrate, and
materialization observability together — and stay recognizably Windmill-shaped.
## Open decisions
These ride on top of the six in `pipelines-vs-dbt.md` §"Decisions either path
forces"; DuckLake-specific:
1. **Bootstrap of `SET PARTITIONED BY`.** First-materialize detection — table
absent vs. present-but-unpartitioned. Idempotent re-apply.
2. **Snapshot retention / compaction.** DuckLake snapshots accumulate; when do
we expire old ones, and does pinning hold a snapshot alive past retention?
3. **Pin scope.** Pin only direct producers, or the full transitive upstream
set per run? Storage and "stale pin" semantics differ.
4. **Managed multi-statement.** *Resolved:* managed `// materialize` accepts
setup statements (ATTACH/SET/…) followed by exactly one trailing SELECT, and
rejects anything else at deploy with a clear error pointing to
`// materialize manual`. The classifier (`sql_materialize.rs`) is the single
source of truth.
-5
View File
@@ -280,11 +280,6 @@ introspection per substrate is its own project.
Ships A's 80% case first without committing to B's dialect-rewriting tax.
Wrapping becomes opt-in convenience for users who want dbt-style ergonomics.
> See [`ducklake-materialization.md`](./ducklake-materialization.md) for the
> DuckLake-native realization of this path: how snapshots make the assets
> versioned/reproducible for free, the executor codegen seam, and the
> materialization-metadata schema.
### Decisions either path forces
1. **Partition window provenance.** Scheduler tick? Trigger event time
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@windmill-labs/components",
"version": "1.737.0",
"version": "1.733.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@windmill-labs/components",
"version": "1.737.0",
"version": "1.733.1",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@windmill-labs/components",
"version": "1.737.0",
"version": "1.733.1",
"scripts": {
"dev": "vite dev",
"dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev",
+2 -102
View File
@@ -1,4 +1,4 @@
<!doctype html>
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
@@ -23,106 +23,6 @@
}
</style>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<script>
// WIN-2006: when the app is rendered inside an opaque-origin (sandboxed,
// no allow-same-origin) iframe, Web Storage access throws a SecurityError.
// Install an in-memory shim before the SvelteKit app boots so the app (and
// Windmill SPA code running inside the frame) keeps working within the
// session. No-op in normal real-origin contexts, so zero blast radius.
;(function () {
// `onOp` (localStorage only) relays each mutation up to the embedder,
// which backs a single store shared across all apps. sessionStorage
// stays in-memory/session-only.
function shim(onOp) {
var m = {}
return {
__hydrate: function (obj) {
if (obj) for (var k in obj) m['' + k] = '' + obj[k]
},
getItem: function (k) {
k = '' + k
return Object.prototype.hasOwnProperty.call(m, k) ? m[k] : null
},
setItem: function (k, v) {
m['' + k] = '' + v
if (onOp) onOp({ op: 'set', key: '' + k, value: '' + v })
},
removeItem: function (k) {
delete m['' + k]
if (onOp) onOp({ op: 'remove', key: '' + k })
},
clear: function () {
for (var k in m) delete m[k]
if (onOp) onOp({ op: 'clear' })
},
key: function (i) {
var ks = Object.keys(m)
return i < ks.length ? ks[i] : null
},
get length() {
return Object.keys(m).length
}
}
}
try {
window.localStorage.getItem('__wm_probe__')
} catch (e) {
var framed = window.parent !== window
function relayOp(o) {
if (framed)
try {
window.parent.postMessage(
{ type: 'wm_ls_op', op: o.op, key: o.key, value: o.value },
'*'
)
} catch (_) {}
}
var ls = shim(framed ? relayOp : null)
try {
Object.defineProperty(window, 'localStorage', { value: ls, configurable: true })
} catch (_) {}
try {
Object.defineProperty(window, 'sessionStorage', {
value: shim(null),
configurable: true
})
} catch (_) {}
// Persistence: ask the embedder for the shared store and hydrate the
// in-memory localStorage when it arrives.
if (framed) {
window.addEventListener('message', function (ev) {
if (ev.source === window.parent && ev.data && ev.data.type === 'wm_ls_hydrate') {
ls.__hydrate(ev.data.data)
}
})
try {
window.parent.postMessage({ type: 'wm_ls_req' }, '*')
} catch (_) {}
}
// `document.cookie` also throws in an opaque origin; back it with an
// in-memory jar so reads don't crash apps. This is NOT the real
// session cookie (unreachable here) — just an isolated client-side store.
try {
var jar = {}
Object.defineProperty(Document.prototype, 'cookie', {
configurable: true,
get: function () {
return Object.keys(jar)
.map(function (k) {
return k + '=' + jar[k]
})
.join('; ')
},
set: function (v) {
var p = String(v).split(';')[0]
var i = p.indexOf('=')
if (i > -1) jar[p.slice(0, i).trim()] = p.slice(i + 1).trim()
}
})
} catch (_) {}
}
})()
</script>
%sveltekit.head%
</head>
<body data-sveltekit-preload-code="viewport" class="outline-none focus:outline-none">
@@ -156,7 +56,7 @@
/>
<polygon
class="st3"
points="136.93,132.47 116.46,167.93 73.82,241.78 130.71,241.78 144.9,217.2 180.13,156.18 193.82,132.46
points="136.93,132.47 116.46,167.93 73.82,241.78 130.71,241.78 144.9,217.2 180.13,156.18 193.82,132.46
"
/>
<polygon
@@ -162,9 +162,6 @@
let clientId = $state('')
let clientSecret = $state('')
let ccInstance = $state('')
/** Bring-your-own resource-level token endpoint override (optional). Only sent
* for non-instance-templated providers, where it isn't host-pinned. */
let tokenUrl = $state('')
let resourceTypeInfo: ResourceType | undefined = $state(undefined)
let resourceTypeNotFound = $state(false)
@@ -225,7 +222,6 @@
clientId = ''
clientSecret = ''
ccInstance = ''
tokenUrl = ''
scopes = []
}
@@ -606,13 +602,7 @@
scopes: scopes,
cc_client_id: trimmedClientId,
cc_client_secret: trimmedClientSecret,
// Instance-templated providers are host-pinned via the instance
// name; only other providers accept a free-form token URL override.
...(needsInstance
? { cc_instance: trimmedInstance }
: tokenUrl.trim()
? { cc_token_url: tokenUrl.trim() }
: {})
...(needsInstance ? { cc_instance: trimmedInstance } : {})
}
})
@@ -744,13 +734,10 @@
accountData.cc_client_id = clientId.trim()
accountData.cc_client_secret = clientSecret.trim()
// Instance-templated providers send an instance name; the backend
// resolves and stores the host-pinned token URL. Other providers may
// send an optional token URL override (stored for refresh); without
// it the token URL comes from the registry/instance config.
// resolves and stores the host-pinned token URL. Other registry
// providers need nothing more (token URL comes from the registry).
if (ccInstanceMeta) {
accountData.cc_instance = ccInstance.trim()
} else if (tokenUrl.trim()) {
accountData.cc_token_url = tokenUrl.trim()
}
}
@@ -1196,23 +1183,6 @@
bind:value={ccInstance}
/>
</label>
{:else}
<label class="flex flex-col gap-1">
<span class="text-xs font-semibold text-emphasis"
>Token URL override (optional)</span
>
<div class="text-xs text-secondary font-normal">
Override the provider's token endpoint for this resource, stored with the
connection and reused on token refresh
</div>
<TextInput
inputProps={{
type: 'url',
placeholder: 'https://provider.example.com/oauth/token'
}}
bind:value={tokenUrl}
/>
</label>
{/if}
</form>
{/if}

Some files were not shown because too many files have changed in this diff Show More