diff --git a/.github/scripts/check-docs-links.mjs b/.github/scripts/check-docs-links.mjs new file mode 100644 index 0000000000..122d098197 --- /dev/null +++ b/.github/scripts/check-docs-links.mjs @@ -0,0 +1,126 @@ +// Extracts every windmill.dev/docs link referenced in the frontend source and +// verifies none of them 404. Run: `node .github/scripts/check-docs-links.mjs`. +// Used by the check-docs-links GitHub workflow (release / manual trigger only). + +import { readdir, readFile } from 'node:fs/promises' +import { join, extname } from 'node:path' + +const ROOT = 'frontend/src' +const EXTS = new Set(['.ts', '.js', '.svelte', '.mjs', '.cjs']) +const DOCS_RE = /https?:\/\/(?:www\.)?windmill\.dev\/docs\/[^\s"'`)>\]}]*/g +// `const someBaseUrl = 'https://www.windmill.dev/docs/...'` used later as `${someBaseUrl}/foo` +const BASE_RE = /(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*['"`](https?:\/\/(?:www\.)?windmill\.dev\/docs\/[^'"`]+)['"`]/g + +const CONCURRENCY = 24 +const TIMEOUT_MS = 20000 +const RETRIES = 2 + +async function walk(dir) { + const out = [] + for (const entry of await readdir(dir, { withFileTypes: true })) { + const p = join(dir, entry.name) + if (entry.isDirectory()) { + if (entry.name === 'node_modules' || entry.name === '.svelte-kit') continue + out.push(...(await walk(p))) + } else if (EXTS.has(extname(entry.name))) { + out.push(p) + } + } + return out +} + +// url (no fragment) -> Set of source files it appears in +const urls = new Map() +const unresolved = [] + +function record(url, file) { + const clean = url + .replace(/\\.*$/, '') // cut at an escape sequence embedded in a string literal (e.g. \n) + .replace(/#.*$/, '') // drop anchor fragment — irrelevant to a 404 check + .replace(/[.,;:'")\]]+$/, '') + if (!clean) return + // A `{`/`${` means the URL is built from an unresolved template/interpolation var. + if (clean.includes('{')) { + unresolved.push(`${clean} (${file})`) + return + } + if (!urls.has(clean)) urls.set(clean, new Set()) + urls.get(clean).add(file) +} + +for (const file of await walk(ROOT)) { + let content = await readFile(file, 'utf8') + // Inline file-local base-url constants so `${base}/page` template literals resolve. + const bases = [] + for (const m of content.matchAll(BASE_RE)) bases.push({ name: m[1], value: m[2], decl: m[0] }) + for (const { name, value } of bases) { + content = content.replaceAll('${' + name + '}', value) + } + // Blank each base declaration so a prefix-only base (no index page of its own, + // e.g. .../app_configuration_settings) isn't checked as a standalone link. + // A genuinely bare `${base}` usage was already inlined above, so it's still covered. + for (const { decl } of bases) content = content.replace(decl, '') + for (const m of content.matchAll(DOCS_RE)) record(m[0], file) +} + +const allUrls = [...urls.keys()].sort() +console.log(`Found ${allUrls.length} distinct docs links across ${ROOT}`) +if (unresolved.length) { + console.log(`\n⚠️ ${unresolved.length} link(s) built from an unrecognized base URL — skipped (register the base const so they can be checked):`) + for (const u of [...new Set(unresolved)].sort()) console.log(` ${u}`) +} + +async function check(url) { + for (let attempt = 0; attempt <= RETRIES; attempt++) { + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS) + try { + let res = await fetch(url, { + method: 'HEAD', + redirect: 'follow', + signal: ctrl.signal, + headers: { 'user-agent': 'windmill-docs-link-check' } + }) + // Some hosts reject HEAD — fall back to GET. + if (res.status === 405 || res.status === 501) { + res = await fetch(url, { + method: 'GET', + redirect: 'follow', + signal: ctrl.signal, + headers: { 'user-agent': 'windmill-docs-link-check' } + }) + } + clearTimeout(timer) + return { url, status: res.status, ok: res.status < 400 } + } catch (err) { + clearTimeout(timer) + if (attempt === RETRIES) return { url, status: 0, ok: false, error: String(err?.message || err) } + await new Promise((r) => setTimeout(r, 500 * (attempt + 1))) + } + } +} + +// Simple concurrency pool. +const results = [] +let idx = 0 +async function worker() { + while (idx < allUrls.length) { + const url = allUrls[idx++] + results.push(await check(url)) + } +} +await Promise.all(Array.from({ length: CONCURRENCY }, worker)) + +const failures = results.filter((r) => !r.ok) +if (failures.length === 0) { + console.log(`\n✅ All ${allUrls.length} docs links are reachable.`) + process.exit(0) +} + +console.log(`\n❌ ${failures.length} broken docs link(s):`) +for (const f of failures.sort((a, b) => a.url.localeCompare(b.url))) { + console.log(`\n ${f.url}`) + console.log(` status: ${f.error ? `error (${f.error})` : f.status}`) + for (const file of urls.get(f.url)) console.log(` ↳ ${file}`) +} +process.exit(1) diff --git a/.github/workflows/check-docs-links.yml b/.github/workflows/check-docs-links.yml new file mode 100644 index 0000000000..ec17c06771 --- /dev/null +++ b/.github/workflows/check-docs-links.yml @@ -0,0 +1,23 @@ +name: Check frontend docs links + +on: + push: + tags: + - "v*" + workflow_dispatch: + +jobs: + check-docs-links: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + frontend/src + .github/scripts + - uses: actions/setup-node@v4 + with: + node-version: "22.x" + - name: Verify docs links are not 404 + run: node .github/scripts/check-docs-links.mjs diff --git a/.github/workflows/check-org-membership.yml b/.github/workflows/check-org-membership.yml deleted file mode 100644 index eb338d3188..0000000000 --- a/.github/workflows/check-org-membership.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Check Organization Membership - -on: - workflow_call: - inputs: - commenter: - required: false - type: string - default: '' - description: 'The username to check. Auto-detected from the event context if not provided.' - organization: - required: false - type: string - default: 'windmill-labs' - description: 'The organization to check membership for' - trusted_bot: - required: false - type: string - default: 'windmill-internal-app[bot]' - description: 'The trusted bot username to allow' - secrets: - access_token: - required: true - description: 'The access token to use for org membership check' - outputs: - is_member: - description: 'Whether the user is an organization member or trusted bot' - value: ${{ jobs.check-membership.outputs.is_member }} - -jobs: - check-membership: - runs-on: ubicloud-standard-2 - outputs: - is_member: ${{ steps.check-membership.outputs.is_member }} - steps: - - name: Determine commenter - id: determine-commenter - run: | - COMMENTER="${{ inputs.commenter }}" - if [[ -z "$COMMENTER" ]]; then - if [[ "${{ github.event_name }}" == "issue_comment" || \ - "${{ github.event_name }}" == "pull_request_review_comment" ]]; then - COMMENTER="${{ github.event.comment.user.login }}" - elif [[ "${{ github.event_name }}" == "pull_request_review" ]]; then - COMMENTER="${{ github.event.review.user.login }}" - else - COMMENTER="${{ github.event.issue.user.login }}" - fi - fi - echo "commenter=$COMMENTER" >> $GITHUB_OUTPUT - - - name: Check organization membership - id: check-membership - env: - ORG_ACCESS_TOKEN: ${{ secrets.access_token }} - COMMENTER: ${{ steps.determine-commenter.outputs.commenter }} - ORG: ${{ inputs.organization }} - TRUSTED_BOT: ${{ inputs.trusted_bot }} - run: | - # 1. Allow the trusted bot straight away - if [[ "$COMMENTER" == "$TRUSTED_BOT" ]]; then - echo "is_member=true" >> $GITHUB_OUTPUT - exit 0 - fi - - # 2. Disallow other bots - if [[ "${COMMENTER}" =~ \[bot\]$ ]]; then - echo "is_member=false" >> $GITHUB_OUTPUT - exit 0 - fi - - # 3. Otherwise check if the user is a member of the organization - STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token $ORG_ACCESS_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/orgs/$ORG/members/$COMMENTER") - - if [ "$STATUS" -eq 204 ]; then - echo "is_member=true" >> $GITHUB_OUTPUT - else - echo "is_member=false" >> $GITHUB_OUTPUT - fi \ No newline at end of file diff --git a/.github/workflows/check-write-access.yml b/.github/workflows/check-write-access.yml new file mode 100644 index 0000000000..5e9494e74a --- /dev/null +++ b/.github/workflows/check-write-access.yml @@ -0,0 +1,66 @@ +name: Check Write Access + +# Authorizes a user to trigger privileged command workflows (/review, /ai, /plan, +# /updatesqlx, ...). The webhook author_association reports PRIVATE org members as +# CONTRIBUTOR/NONE (only public members show as MEMBER), so command jobs can't gate on +# it alone. This mints the internal GitHub App token — which can see private members — +# and confirms the user is a member or has write access to the repo. The app token is +# minted fresh per run, so unlike the old ORG_ACCESS_TOKEN PAT it never expires. + +on: + workflow_call: + inputs: + username: + required: true + type: string + description: 'The user whose access to verify' + trusted_bot: + required: false + type: string + default: 'windmill-internal-app[bot]' + description: 'A bot login that is always authorized' + outputs: + authorized: + description: 'true if the user is the trusted bot, an org member, or has repo write access' + value: ${{ jobs.check.outputs.authorized }} + +jobs: + check: + runs-on: ubuntu-latest + outputs: + authorized: ${{ steps.check.outputs.authorized }} + steps: + - name: Mint internal app token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Resolve authorization + id: check + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + USERNAME: ${{ inputs.username }} + TRUSTED_BOT: ${{ inputs.trusted_bot }} + REPO: ${{ github.repository }} + run: | + if [ "$USERNAME" = "$TRUSTED_BOT" ]; then + echo "authorized=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + ORG="${REPO%%/*}" + # Org membership resolves private members too (204 = member, 404 = not). + if gh api "orgs/$ORG/members/$USERNAME" --silent 2>/dev/null; then + echo "authorized=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Fallback: effective repo permission (also covers outside collaborators). + PERM=$(gh api "repos/$REPO/collaborators/$USERNAME/permission" --jq '.permission' 2>/dev/null || echo none) + if [ "$PERM" = "admin" ] || [ "$PERM" = "write" ]; then + echo "authorized=true" >> "$GITHUB_OUTPUT" + else + echo "authorized=false" >> "$GITHUB_OUTPUT" + echo "$USERNAME is neither the trusted bot, an org member, nor a repo writer." + fi diff --git a/.github/workflows/claude-plan.yml b/.github/workflows/claude-plan.yml index c63e3fe1aa..ef25554f83 100644 --- a/.github/workflows/claude-plan.yml +++ b/.github/workflows/claude-plan.yml @@ -11,20 +11,24 @@ on: types: [submitted] jobs: - check-membership: + # author_association misses private org members; check-access resolves them via the + # internal app token. Both are OR'd below so public members still pass instantly. + check-access: if: | (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/plan')) || (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/plan')) || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/plan')) || (github.event_name == 'issues' && contains(github.event.issue.body, '/plan')) - uses: ./.github/workflows/check-org-membership.yml - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + uses: ./.github/workflows/check-write-access.yml + with: + username: ${{ github.event.comment.user.login || github.event.review.user.login || github.event.issue.user.login }} + secrets: inherit claude-plan-action: - needs: check-membership + needs: [check-access] if: | - needs.check-membership.outputs.is_member == 'true' + needs.check-access.outputs.authorized == 'true' || + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association || github.event.review.author_association || github.event.issue.author_association) runs-on: ubicloud-standard-4 timeout-minutes: 20 permissions: diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index f5db652084..115115dac6 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -11,20 +11,24 @@ on: types: [submitted] jobs: - check-membership: + # author_association misses private org members; check-access resolves them via the + # internal app token. Both are OR'd below so public members still pass instantly. + check-access: if: | (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/ai') && !startsWith(github.event.comment.body, '/ai-fast')) || (github.event_name == 'pull_request_review_comment' && startsWith(github.event.comment.body, '/ai') && !startsWith(github.event.comment.body, '/ai-fast')) || (github.event_name == 'pull_request_review' && startsWith(github.event.review.body, '/ai') && !startsWith(github.event.review.body, '/ai-fast')) || (github.event_name == 'issues' && startsWith(github.event.issue.body, '/ai') && !startsWith(github.event.issue.body, '/ai-fast')) - uses: ./.github/workflows/check-org-membership.yml - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + uses: ./.github/workflows/check-write-access.yml + with: + username: ${{ github.event.comment.user.login || github.event.review.user.login || github.event.issue.user.login }} + secrets: inherit claude-code-action: - needs: check-membership + needs: [check-access] if: | - needs.check-membership.outputs.is_member == 'true' + needs.check-access.outputs.authorized == 'true' || + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association || github.event.review.author_association || github.event.issue.author_association) runs-on: ubicloud-standard-8 permissions: contents: write diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 66cc4b5f10..ea622914bc 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -32,27 +32,19 @@ concurrency: cancel-in-progress: true jobs: - check-membership: - if: github.event_name == 'pull_request' - uses: ./.github/workflows/check-org-membership.yml - with: - commenter: ${{ github.event.pull_request.user.login }} - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} - codex-review: - needs: check-membership runs-on: ubicloud-standard-2 timeout-minutes: 30 + # A non-fork PR (head.repo.fork == false) can only be opened by someone with push + # access to this repo, so fork==false already enforces write access. Do NOT re-add + # an author_association gate: the pull_request webhook payload reports private org + # members as CONTRIBUTOR/NONE (only public members show as MEMBER), which silently + # skips auto-review for every private member. if: | - always() && + github.event_name == 'workflow_call' || ( - needs.check-membership.result == 'skipped' || - (needs.check-membership.result == 'success' && needs.check-membership.outputs.is_member == 'true') - ) && - ( - github.event_name == 'workflow_call' || - (github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false) + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.fork == false ) permissions: contents: read diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index f1cc380563..3443b7e649 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -5,21 +5,22 @@ on: types: [created] jobs: - check-membership: - if: >- - github.event.issue.pull_request && ( - startsWith(github.event.comment.body, '/updatesqlx') || - startsWith(github.event.comment.body, '/demo') || - startsWith(github.event.comment.body, '/eeref') || - startsWith(github.event.comment.body, '/docs') - ) - uses: ./.github/workflows/check-org-membership.yml - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + # /command comments can come from anyone; author_association misses private org + # members, so check-access resolves them via the internal app token. Runs once and is + # OR'd into each job's guard (public members still pass on author_association alone). + check-access: + if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/') + uses: ./.github/workflows/check-write-access.yml + with: + username: ${{ github.event.comment.user.login }} + secrets: inherit update-sqlx: - needs: check-membership - if: needs.check-membership.outputs.is_member == 'true' && startsWith(github.event.comment.body, '/updatesqlx') + needs: [check-access] + if: >- + github.event.issue.pull_request && + (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true') && + startsWith(github.event.comment.body, '/updatesqlx') runs-on: ubicloud-standard-8 permissions: contents: write @@ -147,8 +148,11 @@ jobs: }) demo: - needs: check-membership - if: needs.check-membership.outputs.is_member == 'true' && startsWith(github.event.comment.body, '/demo') + needs: [check-access] + if: >- + github.event.issue.pull_request && + (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true') && + startsWith(github.event.comment.body, '/demo') runs-on: ubicloud-standard-2 permissions: contents: read @@ -227,8 +231,11 @@ jobs: fi update-ee-ref: - needs: check-membership - if: needs.check-membership.outputs.is_member == 'true' && startsWith(github.event.comment.body, '/eeref') + needs: [check-access] + if: >- + github.event.issue.pull_request && + (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true') && + startsWith(github.event.comment.body, '/eeref') runs-on: ubicloud-standard-2 permissions: contents: write @@ -313,8 +320,11 @@ jobs: }) update-docs: - needs: check-membership - if: needs.check-membership.outputs.is_member == 'true' && startsWith(github.event.comment.body, '/docs') + needs: [check-access] + if: >- + github.event.issue.pull_request && + (contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true') && + startsWith(github.event.comment.body, '/docs') runs-on: ubicloud-standard-2 permissions: contents: read diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index 9fbe43e9f0..72553b0d83 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -30,27 +30,19 @@ concurrency: cancel-in-progress: true jobs: - check-membership: - if: github.event_name == 'pull_request' - uses: ./.github/workflows/check-org-membership.yml - with: - commenter: ${{ github.event.pull_request.user.login }} - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} - pi-review: - needs: check-membership runs-on: ubicloud-standard-2 timeout-minutes: 30 + # A non-fork PR (head.repo.fork == false) can only be opened by someone with push + # access to this repo, so fork==false already enforces write access. Do NOT re-add + # an author_association gate: the pull_request webhook payload reports private org + # members as CONTRIBUTOR/NONE (only public members show as MEMBER), which silently + # skips auto-review for every private member. if: | - always() && + github.event_name == 'workflow_call' || ( - needs.check-membership.result == 'skipped' || - (needs.check-membership.result == 'success' && needs.check-membership.outputs.is_member == 'true') - ) && - ( - github.event_name == 'workflow_call' || - (github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false) + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.fork == false ) permissions: contents: read diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index 4d722df4af..eb634977d1 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -30,26 +30,18 @@ concurrency: cancel-in-progress: true jobs: - check-membership: - if: github.event_name == 'pull_request' - uses: ./.github/workflows/check-org-membership.yml - with: - commenter: ${{ github.event.pull_request.user.login }} - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} - auto-review: - needs: check-membership runs-on: ubuntu-latest + # A non-fork PR (head.repo.fork == false) can only be opened by someone with push + # access to this repo, so fork==false already enforces write access. Do NOT re-add + # an author_association gate: the pull_request webhook payload reports private org + # members as CONTRIBUTOR/NONE (only public members show as MEMBER), which silently + # skips auto-review for every private member. if: | - always() && + github.event_name == 'workflow_call' || ( - needs.check-membership.result == 'skipped' || - (needs.check-membership.result == 'success' && needs.check-membership.outputs.is_member == 'true') - ) && - ( - github.event_name == 'workflow_call' || - (github.event.pull_request.draft == false || github.event.pull_request.ready_for_review == true) + (github.event.pull_request.draft == false || github.event.pull_request.ready_for_review == true) && + github.event.pull_request.head.repo.fork == false ) permissions: contents: read diff --git a/.github/workflows/pr-review-commands.yml b/.github/workflows/pr-review-commands.yml index ba55bfea2f..ef93274d7e 100644 --- a/.github/workflows/pr-review-commands.yml +++ b/.github/workflows/pr-review-commands.yml @@ -42,16 +42,24 @@ jobs: ;; esac - check-membership: - needs: parse + # author_association misses private org members; check-access resolves them via the + # internal app token. Both are OR'd so public members still pass instantly. + check-access: + needs: [parse] if: needs.parse.outputs.command != '' - uses: ./.github/workflows/check-org-membership.yml - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + uses: ./.github/workflows/check-write-access.yml + with: + username: ${{ github.event.comment.user.login }} + secrets: inherit acknowledge: - needs: [parse, check-membership] - if: needs.parse.outputs.command != '' && needs.check-membership.outputs.is_member == 'true' + needs: [parse, check-access] + if: | + needs.parse.outputs.command != '' && + ( + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || + needs.check-access.outputs.authorized == 'true' + ) runs-on: ubuntu-latest permissions: issues: write @@ -68,9 +76,12 @@ jobs: -f content=eyes >/dev/null claude: - needs: [parse, check-membership] + needs: [parse, check-access] if: | - needs.check-membership.outputs.is_member == 'true' && + ( + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || + needs.check-access.outputs.authorized == 'true' + ) && (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'claude') permissions: contents: read @@ -86,9 +97,12 @@ jobs: WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} codex: - needs: [parse, check-membership] + needs: [parse, check-access] if: | - needs.check-membership.outputs.is_member == 'true' && + ( + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || + needs.check-access.outputs.authorized == 'true' + ) && (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'codex') permissions: contents: read @@ -105,9 +119,12 @@ jobs: WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} pi: - needs: [parse, check-membership] + needs: [parse, check-access] if: | - needs.check-membership.outputs.is_member == 'true' && + ( + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || + needs.check-access.outputs.authorized == 'true' + ) && (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'pi') permissions: contents: read diff --git a/AGENTS.md b/AGENTS.md index aead237e59..83b68f7d36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,8 @@ Open-source platform for internal tools, workflows, API integrations, background ## Dev Environment - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) +- **DuckDB local jobs**: before running DuckDB scripts locally, build the FFI shared library with `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. Re-run it after clean builds or when `backend/target/debug/libwindmill_duckdb_ffi_internal.*` is missing. The bundled DuckDB compile (~2min) is cached in a per-user dir shared across worktrees, so a fresh worktree reuses it and the build is near-instant. +- **Data pipelines (DuckLake) from source**: a plain `cargo run` (even `--features quickjs`) advertises a `duckdb` worker tag but **cannot** execute DuckDB scripts and has **no** working S3 proxy (DuckLake writes 404). Build CE DuckLake with `cargo run --features quickjs,duckdb,parquet,private` (add `,python` for Python scripts, `,enterprise,license` for EE) **and** build the FFI (bullet above). See `backend/CLAUDE.md` → "Running data pipelines (DuckLake) from source" for the exact feature sets and the two feature-gate gotchas. - **Frontend**: `REMOTE=http://localhost:8000 npm run dev` from `frontend/` (port 3000+) - **DB**: `psql postgres://postgres:changeme@localhost:5432/windmill` - **Login**: `admin@windmill.dev` / `changeme` diff --git a/CHANGELOG.md b/CHANGELOG.md index 361b6fbd91..069b259d4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,167 @@ # Changelog +## [1.750.0](https://github.com/windmill-labs/windmill/compare/v1.749.0...v1.750.0) (2026-07-06) + + +### Features + +* chat-scoped session changes bar + unified diff drawer ([#9762](https://github.com/windmill-labs/windmill/issues/9762)) ([a6c0b37](https://github.com/windmill-labs/windmill/commit/a6c0b3756be78ca3fadc7bad6bae98c0887fd538)) +* **pipelines:** require data uploads before running a pipeline ([#9953](https://github.com/windmill-labs/windmill/issues/9953)) ([a1c5b7a](https://github.com/windmill-labs/windmill/commit/a1c5b7aa3ed2841f09f4f148ded5c5b5ef10fd3d)) +* **pipelines:** wm_partition macro for grain-agnostic partition filters ([#9950](https://github.com/windmill-labs/windmill/issues/9950)) ([43044c2](https://github.com/windmill-labs/windmill/commit/43044c2e28139b1dbde6844c781a821f8de68f58)) + + +### Bug Fixes + +* **ai:** test key routes Azure Foundry Claude models via Anthropic Messages API ([#9956](https://github.com/windmill-labs/windmill/issues/9956)) ([ea19cc9](https://github.com/windmill-labs/windmill/commit/ea19cc9dc459bd259e27f7fcc29601a010c5f8f0)) +* **cli:** HD-1 test_edges + HD-2 scd2 _current write in --local pipeline graph ([#9947](https://github.com/windmill-labs/windmill/issues/9947)) ([ad6f23d](https://github.com/windmill-labs/windmill/commit/ad6f23d6bfcf1056bcb6d8c6b552114e88177328)) +* **pipelines:** make node & pipeline-level run affordances always visible ([#9948](https://github.com/windmill-labs/windmill/issues/9948)) ([6eabb96](https://github.com/windmill-labs/windmill/commit/6eabb96ae78fb966f9916f907bb693d569b04c0b)) +* read chat drafts via own-draft route so drawer-kind drafts deploy ([#9913](https://github.com/windmill-labs/windmill/issues/9913)) ([056ebdb](https://github.com/windmill-labs/windmill/commit/056ebdb03543a93094c80ca354c117236cd8d6c8)) +* resolve extensionless bun relative imports on windows loader ([#9949](https://github.com/windmill-labs/windmill/issues/9949)) ([bf96621](https://github.com/windmill-labs/windmill/commit/bf9662172ad7e0ff53d39adc338fd7886672c8f9)) + +## [1.749.0](https://github.com/windmill-labs/windmill/compare/v1.748.0...v1.749.0) (2026-07-05) + + +### Features + +* **pipelines:** mid-DAG selective execution (dbt `model+`) for pipeline runs ([#9945](https://github.com/windmill-labs/windmill/issues/9945)) ([2d3a773](https://github.com/windmill-labs/windmill/commit/2d3a77344104a587548f23f1b614ceffd52a5778)) +* **pipelines:** partition run-arg picker + first-run setup signpost ([#9943](https://github.com/windmill-labs/windmill/issues/9943)) ([475b072](https://github.com/windmill-labs/windmill/commit/475b072987b33d50111f5251a5f69f4245f930ae)) +* **pipelines:** self-teaching custom data_test errors + scaffold ([#9937](https://github.com/windmill-labs/windmill/issues/9937)) ([0ad174f](https://github.com/windmill-labs/windmill/commit/0ad174fa490e17eeb26280b5bdfd62956dfed9ff)) + + +### Bug Fixes + +* **cli:** macro-library parity in --local pipeline graph + read-only run --dry-run ([#9942](https://github.com/windmill-labs/windmill/issues/9942)) ([e3f4303](https://github.com/windmill-labs/windmill/commit/e3f43033cafcdb5df253aeb55ce93e599b2584d2)) +* **datatable:** self-teaching error for unresolved datatable:// references ([#9941](https://github.com/windmill-labs/windmill/issues/9941)) ([55451db](https://github.com/windmill-labs/windmill/commit/55451db009e3060c21948ece2c97e102a3c9b171)) +* **object-storage:** remove 20-file bucket-browser listing cap in CE ([#9935](https://github.com/windmill-labs/windmill/issues/9935)) ([22452ce](https://github.com/windmill-labs/windmill/commit/22452ce54034a9bea8f7d48946818fd148b938c0)) +* **pipelines:** dedup guard for keyed merge + deploy-time SCD2 validation ([#9936](https://github.com/windmill-labs/windmill/issues/9936)) ([52ce805](https://github.com/windmill-labs/windmill/commit/52ce805f619747af4f998cde7819a164c754205a)) +* **pipelines:** link SCD2 <dim>_current view to its producer across all graph surfaces ([#9933](https://github.com/windmill-labs/windmill/issues/9933)) ([574d3ac](https://github.com/windmill-labs/windmill/commit/574d3ac9ff5015b5d3f53040c9d4dfbfd161a076)) +* **pipelines:** order data_test relationships refs before the tested script in a cascade ([#9934](https://github.com/windmill-labs/windmill/issues/9934)) ([46be39d](https://github.com/windmill-labs/windmill/commit/46be39dfb7fbfb2b70e61819d6065b45810c41c9)) +* **pipelines:** pipeline-level run control, tables label, data-test rollback + fork badges ([#9944](https://github.com/windmill-labs/windmill/issues/9944)) ([6ae8dd3](https://github.com/windmill-labs/windmill/commit/6ae8dd37b1de930ab17344cebf7c28385c6cfdba)) +* rebuild windows bun loader main.ts filter from forward-slash cdir ([#9946](https://github.com/windmill-labs/windmill/issues/9946)) ([a582e04](https://github.com/windmill-labs/windmill/commit/a582e04bf40cf685f88bceaf88e3d24bde3d420a)) + +## [1.748.0](https://github.com/windmill-labs/windmill/compare/v1.747.0...v1.748.0) (2026-07-05) + + +### Features + +* **ai-agent:** support reasoning effort in AI agent workflow steps ([#9886](https://github.com/windmill-labs/windmill/issues/9886)) ([a368d49](https://github.com/windmill-labs/windmill/commit/a368d49bd8786a2dca6771f2051f1d44d1b2363d)) +* **ducklake:** scheduled lake maintenance (expiry, compaction, orphan cleanup) ([#9916](https://github.com/windmill-labs/windmill/issues/9916)) ([3352150](https://github.com/windmill-labs/windmill/commit/33521505dbc34f22b575d21fda1cc76d698a8840)) +* **pipelines:** asset freshness — fresh/stale badge (CE) + watchdog (EE) ([#9909](https://github.com/windmill-labs/windmill/issues/9909)) ([5d7fb6d](https://github.com/windmill-labs/windmill/commit/5d7fb6deca3e02e89d77e5d3856483beb8b8bfeb)) +* **pipelines:** capture violating-row samples for data tests ([#9919](https://github.com/windmill-labs/windmill/issues/9919)) ([d4b4374](https://github.com/windmill-labs/windmill/commit/d4b4374de8f8a7875b050c16d1236fcd0355812b)) +* **pipelines:** fork data environments for ducklake materialization (dev data) ([#9915](https://github.com/windmill-labs/windmill/issues/9915)) ([39eb9de](https://github.com/windmill-labs/windmill/commit/39eb9de1bce400109c130a081807e40e995ae068)) +* **pipelines:** on_schema_change write guardrails + data_test deploy validation ([#9930](https://github.com/windmill-labs/windmill/issues/9930)) ([377c02e](https://github.com/windmill-labs/windmill/commit/377c02ec47389e64b7ef5cbbae0de05df648266d)) +* **pipelines:** record upstream snapshot ids on cascade-dispatched jobs ([#9910](https://github.com/windmill-labs/windmill/issues/9910)) ([af36498](https://github.com/windmill-labs/windmill/commit/af36498432e643108308e1c03b5d986d0f0f8888)) +* **pipelines:** schema contracts — save-time consumer checks vs captured schemas ([#9917](https://github.com/windmill-labs/windmill/issues/9917)) ([42e11c6](https://github.com/windmill-labs/windmill/commit/42e11c6570b62ffaa86598438fa8ddf462c4035f)) +* **pipeline:** write-audit-publish for materialization data tests ([#9911](https://github.com/windmill-labs/windmill/issues/9911)) ([dce247c](https://github.com/windmill-labs/windmill/commit/dce247c6d2678a2c95bd728027e17ae3965638e2)) +* **sdk:** enforce s3:// URIs for string S3 params + ingestion (EL) docs ([#9912](https://github.com/windmill-labs/windmill/issues/9912)) ([5ad2de9](https://github.com/windmill-labs/windmill/commit/5ad2de91a26b312bf27124ceca16ef331621bde8)) + + +### Bug Fixes + +* **cli:** pipeline + workspace UX batch (init/bind stub, run errors, macro libs, lock-job report, upgrade errors) ([#9929](https://github.com/windmill-labs/windmill/issues/9929)) ([28a6b08](https://github.com/windmill-labs/windmill/commit/28a6b086c842105298f236baa0a61868f71a5eb1)) +* **cli:** publish all windmill-parser-wasm-* deps so local pipeline graph keeps write edges ([#9926](https://github.com/windmill-labs/windmill/issues/9926)) ([744a759](https://github.com/windmill-labs/windmill/commit/744a7597edaf3ca9a7fd2b21a34fb33457913a64)) +* **pipelines:** activity-axis label clarity + select failed node on cascade failure ([#9931](https://github.com/windmill-labs/windmill/issues/9931)) ([5769b60](https://github.com/windmill-labs/windmill/commit/5769b6036cf14b0cb424c5b3d9d878c600a5652e)) + +## [1.747.0](https://github.com/windmill-labs/windmill/compare/v1.746.0...v1.747.0) (2026-07-03) + + +### Features + +* **frontend:** add federatedTokenFile field to instance object storage Azure config ([#9904](https://github.com/windmill-labs/windmill/issues/9904)) ([ae85d27](https://github.com/windmill-labs/windmill/commit/ae85d274371a24c5badb6081f00deeb409123252)) + + +### Bug Fixes + +* **ai:** route Azure Foundry Claude models via Anthropic Messages API ([#9908](https://github.com/windmill-labs/windmill/issues/9908)) ([d600c7e](https://github.com/windmill-labs/windmill/commit/d600c7ecfe305533798e82e8d05e5f2f297f9b54)) +* **forks:** clone only the current raw-app bundle, via server-side copy ([#9899](https://github.com/windmill-labs/windmill/issues/9899)) ([5c521d8](https://github.com/windmill-labs/windmill/commit/5c521d808a2b5d6d6bb7cf3da17fb2addc53fdf4)) +* **kafka:** set https.ca.location=probe for OAUTHBEARER OIDC token endpoint ([#9897](https://github.com/windmill-labs/windmill/issues/9897)) ([1b6065f](https://github.com/windmill-labs/windmill/commit/1b6065fa9201fd548c4b2ef199f1009200645929)) +* prevent truncated tool call args from bricking AI chat sessions ([#9902](https://github.com/windmill-labs/windmill/issues/9902)) ([4ba17d0](https://github.com/windmill-labs/windmill/commit/4ba17d0f9cd70489f89c84f982a0c8f0062fed1a)) +* strip NUL characters from app values at save time ([#9903](https://github.com/windmill-labs/windmill/issues/9903)) ([3ec1f16](https://github.com/windmill-labs/windmill/commit/3ec1f164be9c8c6c40e003188ce593a963c65a43)) + +## [1.746.0](https://github.com/windmill-labs/windmill/compare/v1.745.0...v1.746.0) (2026-07-02) + + +### Features + +* **ai:** add Azure AI Foundry as a native AI provider ([#9879](https://github.com/windmill-labs/windmill/issues/9879)) ([d9b080f](https://github.com/windmill-labs/windmill/commit/d9b080f57fa0be144cefa773d39742c45b40f043)) +* **frontend:** group compare & deploy items by folder ([#9880](https://github.com/windmill-labs/windmill/issues/9880)) ([7b04820](https://github.com/windmill-labs/windmill/commit/7b04820f8ef8c7f02f79dd4239a877f667d23e6a)) +* **frontend:** pipelines index page and sql editor hint ([#9881](https://github.com/windmill-labs/windmill/issues/9881)) ([20351a6](https://github.com/windmill-labs/windmill/commit/20351a6b4c262184c5f815eeb5de007ab1eaf4a0)) +* **pipeline:** backfill a range of partitions from the asset drawer ([#9885](https://github.com/windmill-labs/windmill/issues/9885)) ([53bbb92](https://github.com/windmill-labs/windmill/commit/53bbb92953178eb6d0017818ef870f3cb2399dfd)) +* **pipelines:** workspace duckdb macro libraries (// macros / // use) ([#9890](https://github.com/windmill-labs/windmill/issues/9890)) ([84141ad](https://github.com/windmill-labs/windmill/commit/84141add1ddf35c2573e3c213366ce7c5f1f2258)) +* **s3:** replace CE 50MB upload cap with 10GiB workspace storage quota ([#9874](https://github.com/windmill-labs/windmill/issues/9874)) ([af01e90](https://github.com/windmill-labs/windmill/commit/af01e90b5c65d1b1cfacf4433f8cff7effe73768)) +* support workspace forks on cloud using parent workspace limits ([#9864](https://github.com/windmill-labs/windmill/issues/9864)) ([7c7d747](https://github.com/windmill-labs/windmill/commit/7c7d7474cc86a4052272032f281cc4d7a85db37b)) + + +### Bug Fixes + +* **duckdb:** auto-declare partition arg for `// partitioned` scripts ([#9878](https://github.com/windmill-labs/windmill/issues/9878)) ([b883adb](https://github.com/windmill-labs/windmill/commit/b883adbc0011073da592dc5b39e1b79db492c83c)) +* **frontend:** home New submenus fall back below, hugging the right edge ([#9894](https://github.com/windmill-labs/windmill/issues/9894)) ([186ac49](https://github.com/windmill-labs/windmill/commit/186ac4933b79aed57fce23ebcf3b525fcfd1c474)) +* **frontend:** show inline workspace name editor on general settings (Fixes GIT-911) ([#9892](https://github.com/windmill-labs/windmill/issues/9892)) ([a49c087](https://github.com/windmill-labs/windmill/commit/a49c0871d7ab2aaf78a7713b8a786ead937434da)) +* **frontend:** stack cron field and cron builder button on narrow screens ([#9871](https://github.com/windmill-labs/windmill/issues/9871)) ([7989795](https://github.com/windmill-labs/windmill/commit/79897950e7646b00d92a28a009174d91c705b251)) +* invalidate bun bundle cache on transitive relative-import changes ([#9891](https://github.com/windmill-labs/windmill/issues/9891)) ([d15033c](https://github.com/windmill-labs/windmill/commit/d15033cde6a474b548ebbaf18ff02223fc21f701)) +* make SMTP username and password optional in frontend validation ([#9895](https://github.com/windmill-labs/windmill/issues/9895)) ([37bb574](https://github.com/windmill-labs/windmill/commit/37bb57474e8336823bb31527f2a708ef41cd39c4)) +* **parsers:** infer py s3 assets from S3Object constructor and dict forms ([#9877](https://github.com/windmill-labs/windmill/issues/9877)) ([659642e](https://github.com/windmill-labs/windmill/commit/659642e4889361f86e8addb038cda62fc3471006)) +* pipeline dogfooding fixes — SCD2 data-test scope, --partition, s3object upload binding ([#9875](https://github.com/windmill-labs/windmill/issues/9875)) ([d65f58c](https://github.com/windmill-labs/windmill/commit/d65f58c388d88fff71cda22dfa21aecdae70c450)) +* polish pipeline graph view (layout, viewport, minimap, lineage, timestamps) ([#9883](https://github.com/windmill-labs/windmill/issues/9883)) ([b92a86b](https://github.com/windmill-labs/windmill/commit/b92a86b8b3a60b877540c3a7f0ffefe36ccbb053)) +* stale AI chat context picker after workspace item changes ([#9893](https://github.com/windmill-labs/windmill/issues/9893)) ([5af91a6](https://github.com/windmill-labs/windmill/commit/5af91a677cad88faccba702e3556fc4fb7b6e640)) +* **triggers:** retry transient websocket connect failures before disabling ([#9887](https://github.com/windmill-labs/windmill/issues/9887)) ([7894507](https://github.com/windmill-labs/windmill/commit/789450731b0a3c8dffa336f7bfc3f3de528c09fb)) + +## [1.745.0](https://github.com/windmill-labs/windmill/compare/v1.744.0...v1.745.0) (2026-07-01) + + +### Features + +* **forks:** partial-visibility deploy + surface hidden items ([#9868](https://github.com/windmill-labs/windmill/issues/9868)) ([20cd1a0](https://github.com/windmill-labs/windmill/commit/20cd1a02d582c0715bedacce52cc5c1e1e8d70ca)) +* **frontend:** add zoom and download to Mermaid graphs ([#9859](https://github.com/windmill-labs/windmill/issues/9859)) ([289017b](https://github.com/windmill-labs/windmill/commit/289017bcb28c049c8258b2ffd7da0ec3e6ef120b)) +* use derived username instead of email for non-member superadmins ([#9857](https://github.com/windmill-labs/windmill/issues/9857)) ([76a9523](https://github.com/windmill-labs/windmill/commit/76a95230095ca3f43c9dc9eecde0e9de6520242f)) + + +### Bug Fixes + +* **cli:** correct misleading delete-fork command description ([#9870](https://github.com/windmill-labs/windmill/issues/9870)) ([a73b14d](https://github.com/windmill-labs/windmill/commit/a73b14d902d759226d0af2f2faf9bdd6588e358c)) +* **folders:** allow dots and at-signs in folder owner validation ([#9856](https://github.com/windmill-labs/windmill/issues/9856)) ([383c705](https://github.com/windmill-labs/windmill/commit/383c70523bf81c5c07784a4379ef6b1c93ff86e5)) +* **forks:** require admin of both sides for the compare visibility guard ([#9869](https://github.com/windmill-labs/windmill/issues/9869)) ([7363d2c](https://github.com/windmill-labs/windmill/commit/7363d2c217cb04391f03b2f9958be70a9d0b5325)) +* **forks:** reset diff tally on trigger delete + guard compare visibility for admins ([#9866](https://github.com/windmill-labs/windmill/issues/9866)) ([6a6f129](https://github.com/windmill-labs/windmill/commit/6a6f12960e29c314d11ad541519c71412f42567b)) +* **jobs:** give flow dynselect a path and its worker tag, like scripts ([#9867](https://github.com/windmill-labs/windmill/issues/9867)) ([1a9debb](https://github.com/windmill-labs/windmill/commit/1a9debb689f756f38db085d1360c8fe7691ada48)) +* **offboarding:** make global reassignment per-workspace and optional ([#9863](https://github.com/windmill-labs/windmill/issues/9863)) ([3586164](https://github.com/windmill-labs/windmill/commit/35861641f807a02b5c205608fb592e20ee7cad7f)) + +## [1.744.0](https://github.com/windmill-labs/windmill/compare/v1.743.0...v1.744.0) (2026-07-01) + + +### Features + +* add copy-to-clipboard button to rendered Mermaid diagrams in AI chat ([#9838](https://github.com/windmill-labs/windmill/issues/9838)) ([a27e814](https://github.com/windmill-labs/windmill/commit/a27e814a03c615259381eaf684aa90d56569b0af)) +* add dev workspaces paired with a lockable prod workspace ([#9793](https://github.com/windmill-labs/windmill/issues/9793)) ([b4b0c6a](https://github.com/windmill-labs/windmill/commit/b4b0c6a93e52152251fadefe319773faf42549b2)) +* **ansible:** support repo-provided ansible.cfg in delegate_to_git_repo ([#9851](https://github.com/windmill-labs/windmill/issues/9851)) ([68bf0da](https://github.com/windmill-labs/windmill/commit/68bf0daf5815307cda6ce23214dd5159b6aa33b4)) +* **licensing:** enforce offline license seat cap ([#9845](https://github.com/windmill-labs/windmill/issues/9845)) ([83f3d7f](https://github.com/windmill-labs/windmill/commit/83f3d7f910b331c09f60cc9ff556728afa3dec07)) +* **object-store:** make GCS service account key optional for Workload Identity ([#9842](https://github.com/windmill-labs/windmill/issues/9842)) ([83ed011](https://github.com/windmill-labs/windmill/commit/83ed011e264f20ffa66a7bf933f2fe3615cf6b67)) +* **pipeline:** local development for data pipelines (CLI --local + pipeline dev preview) ([#9840](https://github.com/windmill-labs/windmill/issues/9840)) ([74f579e](https://github.com/windmill-labs/windmill/commit/74f579e6d9ef08e74460f904a4c22ed9d6a3b5b0)) +* **pipelines:** add managed SCD2 history materialize strategy ([#9850](https://github.com/windmill-labs/windmill/issues/9850)) ([5a66127](https://github.com/windmill-labs/windmill/commit/5a661279a3690e2393b9b16996f5d1a5a509259c)) + + +### Bug Fixes + +* **ai-chat:** replay anthropic turns verbatim to keep thinking valid ([#9843](https://github.com/windmill-labs/windmill/issues/9843)) ([a37a144](https://github.com/windmill-labs/windmill/commit/a37a144e81cf6b3de935688a617e9d0e1756004a)) +* grant dispatch_event table to windmill roles ([#9852](https://github.com/windmill-labs/windmill/issues/9852)) ([f05b50d](https://github.com/windmill-labs/windmill/commit/f05b50d29ac2fdbb808a97057fb92c8e425b4a2f)) +* grant workspace_diff, materialized_partition, debounce_stale_data to windmill roles ([#9853](https://github.com/windmill-labs/windmill/issues/9853)) ([293647d](https://github.com/windmill-labs/windmill/commit/293647de4c13cb8468cbd81ff1924cba90e164b4)) +* honor verify-ca/verify-full sslmode for postgres connections ([#9835](https://github.com/windmill-labs/windmill/issues/9835)) ([bf6be96](https://github.com/windmill-labs/windmill/commit/bf6be967fa8c74e1299cf63f813c1cfa34b97f3e)) +* **mcp:** stop double-escaping string query params in build_query_string ([#9855](https://github.com/windmill-labs/windmill/issues/9855)) ([1c46f89](https://github.com/windmill-labs/windmill/commit/1c46f899ca03edf62053f4f14d65b4eabff4255d)) +* **s3_proxy:** preserve URL-encoding on Hive-partition proxy writes ([#9848](https://github.com/windmill-labs/windmill/issues/9848)) ([6b79bdd](https://github.com/windmill-labs/windmill/commit/6b79bddd42fe55f891c17cb71a7e36ee31337bac)) +* validate workspace name length (max 50 chars) on create and fork ([#9854](https://github.com/windmill-labs/windmill/issues/9854)) ([b52972d](https://github.com/windmill-labs/windmill/commit/b52972d0de89004e98d18241d238ca028e4eecba)) + +## [1.743.0](https://github.com/windmill-labs/windmill/compare/v1.742.0...v1.743.0) (2026-06-29) + + +### Features + +* **home:** redesign create-new popover and home header ([#9827](https://github.com/windmill-labs/windmill/issues/9827)) ([2493eaf](https://github.com/windmill-labs/windmill/commit/2493eaf031f30072637a297398674e761f039005)) +* **pipeline:** AI-chat data-pipeline editor (route + in-session) + home surfacing ([#9805](https://github.com/windmill-labs/windmill/issues/9805)) ([c910278](https://github.com/windmill-labs/windmill/commit/c91027824be1f1f49cdd14148baf6aad092a1dd0)) + + +### Bug Fixes + +* **gcp:** require token verification for authenticated push delivery ([#9834](https://github.com/windmill-labs/windmill/issues/9834)) ([9b65161](https://github.com/windmill-labs/windmill/commit/9b65161c643bf3f120d2ebd82f786c17233a971b)) + ## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 379eb6d447..f9023449a2 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -11,6 +11,7 @@ import type { DataTableTables, DataTableTableSchema, GetDraftForUserResponse, + GetOwnDraftResponse, ListDraftsResponse, ScriptLang, UpdateDraftResponse, @@ -294,8 +295,8 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { /** * In-memory stand-in for the per-user draft backend (`DraftService`). The global * AI chat now persists and reads drafts through the backend DB instead of an - * in-tab `UserDraft` cell, so the eval mocks the three draft endpoints it - * exercises (`updateDraft` / `getDraftForUser` / `listDrafts`) and keeps the + * in-tab `UserDraft` cell, so the eval mocks the draft endpoints it exercises + * (`updateDraft` / `getOwnDraft` / `getDraftForUser` / `listDrafts`) and keeps the * saved values here, keyed by workspace + draft kind + storage path. Mirrors the * semantics of the production unit test's mock in * `frontend/src/lib/components/copilot/chat/global/core.test.ts`. @@ -379,6 +380,20 @@ export function getBenchmarkDraftForUser(input: { return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } } +/** Mirror `DraftService.getOwnDraft`: `null` (200) when absent — unlike + * `getDraftForUser`, absence is not an error on this route. */ +export function getBenchmarkOwnDraft(input: { + workspace: string + kind: UserDraftItemKind + path: string +}): GetOwnDraftResponse { + const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) + if (!entry) { + return null + } + return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } +} + /** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { return [...benchmarkDrafts.values()] diff --git a/ai_evals/adapters/frontend/mockBackendDrafts.test.ts b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts index a720de5e43..0ab79d216e 100644 --- a/ai_evals/adapters/frontend/mockBackendDrafts.test.ts +++ b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { clearBenchmarkDrafts, getBenchmarkDraftForUser, + getBenchmarkOwnDraft, listBenchmarkDrafts, resetBenchmarkMockBackend, seedBenchmarkDraft, @@ -55,6 +56,27 @@ describe('mockBackend drafts', () => { expect(() => getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'variable', path: 'f/evals/token' })).toThrow() }) + it('returns null from getOwnDraft when no draft exists', () => { + expect( + getBenchmarkOwnDraft({ workspace: WORKSPACE, kind: 'trigger_schedule', path: 'u/evals/missing' }) + ).toBeNull() + }) + + // The global chat hydrates drawer-kind drafts (schedule/trigger/resource/variable) + // through getOwnDraft — getDraftForUser rejects those kinds as private. + it('hydrates a saved drawer-kind draft through getOwnDraft', () => { + const value = { path: 'u/evals/nightly', schedule: '0 0 9 * * *' } + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'trigger_schedule', + path: 'u/evals/nightly', + requestBody: { value } + }) + expect( + getBenchmarkOwnDraft({ workspace: WORKSPACE, kind: 'trigger_schedule', path: 'u/evals/nightly' })?.value + ).toEqual(value) + }) + it('throws a 404-shaped error when no draft exists', () => { try { getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/missing' }) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index dcaa1d2ca4..92e33414df 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -40,6 +40,7 @@ vi.mock('$lib/gen', async () => { getBenchmarkDraftForUser, getBenchmarkFlowByPath, getBenchmarkJobLogs, + getBenchmarkOwnDraft, getBenchmarkScriptByHash, getBenchmarkScriptByPath, hasBenchmarkWorkspace, @@ -86,6 +87,10 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? getBenchmarkDraftForUser(data) : actual.DraftService.getDraftForUser(data), + getOwnDraft: async (data: { workspace: string; kind: any; path: string }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkOwnDraft(data) + : actual.DraftService.getOwnDraft(data), listDrafts: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) ? listBenchmarkDrafts(data.workspace) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 31b097867b..66e5fd3cb7 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1223,6 +1223,66 @@ - places it in team_a (writable by this non-admin user) and not team_b (read-only) - leaves the result as a draft only +- id: global-test-pipeline-create-node + prompt: |- + Set up the first step of a data pipeline at `f/evals/global/orders_ingest`. + On a schedule, it should pull raw orders and land them in a managed DuckLake + table so later steps can build on it. Keep it as an AI draft only — don't + deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/orders_ingest + valueIncludes: + - pipeline + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - builds a data pipeline node as a script (not a flow) + - marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`) + - declares a schedule trigger and writes its output to a managed DuckLake table + - leaves the result as an AI draft and does not deploy or save it + +- id: global-test-pipeline-two-node-chain + prompt: |- + Build a small data pipeline in the `f/evals/global` folder: one step that + ingests orders into a DuckLake table, and a second step that reads that table + and writes a daily order-count rollup table. Wire the second step to run off + the first step's output. Keep everything as drafts — don't deploy. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 14 + validate: + draftCountAtLeast: 2 + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates two data pipeline nodes as scripts (not a flow) in f/evals/global + - both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) + - the first ingests orders into a DuckLake table + - the second reads that same table and writes a daily rollup, wired to the first step's output asset + - leaves both as AI drafts without deploying + - id: global-path5-create-folder-then-draft prompt: |- Create a new shared folder called "analytics" for our data work, then draft a diff --git a/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json b/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json deleted file mode 100644 index cde17acf76..0000000000 --- a/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941" -} diff --git a/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json b/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json deleted file mode 100644 index 5ca75fe782..0000000000 --- a/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM asset\n WHERE (workspace_id, path, kind) IN (\n SELECT workspace_id, path, kind FROM (\n SELECT a.workspace_id, a.path, a.kind, a.usage_kind, ROW_NUMBER() OVER (\n PARTITION BY a.workspace_id, a.path, a.kind\n ORDER BY a.created_at DESC\n ) as rn,\n limits.max_n\n FROM asset a\n INNER JOIN (\n SELECT * FROM UNNEST(\n $1::varchar[], \n $2::varchar[], \n $3::asset_kind[],\n $4::int[]\n ) AS t(workspace_id, path, kind, max_n)\n ) limits\n ON a.workspace_id = limits.workspace_id \n AND a.path = limits.path \n AND a.kind = limits.kind\n WHERE a.usage_kind = 'job'\n ) ranked\n WHERE rn > max_n\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "VarcharArray", - "VarcharArray", - { - "Custom": { - "name": "asset_kind[]", - "kind": { - "Array": { - "Custom": { - "name": "asset_kind", - "kind": { - "Enum": [ - "s3object", - "resource", - "variable", - "ducklake", - "datatable", - "volume" - ] - } - } - } - } - } - }, - "Int4Array" - ] - }, - "nullable": [] - }, - "hash": "02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0" -} diff --git a/backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json b/backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json similarity index 51% rename from backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json rename to backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json index 6fd7d38f69..036e8e4b57 100644 --- a/backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json +++ b/backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2", + "query": "UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e" + "hash": "0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b" } diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index a779aa0e95..6efb66005d 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -35,7 +35,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json b/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json new file mode 100644 index 0000000000..906c710fdf --- /dev/null +++ b/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7" +} diff --git a/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json b/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json new file mode 100644 index 0000000000..238522a3ff --- /dev/null +++ b/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json @@ -0,0 +1,77 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (mp.asset_kind, mp.asset_path)\n mp.asset_kind AS \"asset_kind: AssetKind\", mp.asset_path,\n mp.snapshot_id AS \"snapshot_id!\", mp.partition\n FROM materialized_partition mp\n JOIN unnest($2::ASSET_KIND[], $3::text[]) AS u(kind, path)\n ON mp.asset_kind = u.kind AND mp.asset_path = u.path\n WHERE mp.workspace_id = $1\n AND mp.status = 'materialized' AND mp.snapshot_id IS NOT NULL\n ORDER BY mp.asset_kind, mp.asset_path, mp.snapshot_id DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "snapshot_id!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "partition", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "TextArray" + ] + }, + "nullable": [ + false, + false, + true, + false + ] + }, + "hash": "0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285" +} diff --git a/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json new file mode 100644 index 0000000000..0dbcf04c6e --- /dev/null +++ b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4)\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946" +} diff --git a/backend/.sqlx/query-0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b.json b/backend/.sqlx/query-0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b.json new file mode 100644 index 0000000000..c43b9e53a1 --- /dev/null +++ b/backend/.sqlx/query-0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, archived, schema_validation, ws_error_handler_muted, deleted, extra_perms)\n VALUES ('test-workspace', 'f/restricted/item', 314159, 'def main(): return 1', '', '', 'python3', 'test-user', NOW(), false, false, false, false, '{}'::jsonb)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b" +} diff --git a/backend/.sqlx/query-12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7.json b/backend/.sqlx/query-12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7.json new file mode 100644 index 0000000000..f2faa9f48a --- /dev/null +++ b/backend/.sqlx/query-12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, params, body, is_table_macro, provider_path FROM macro_definition WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "params", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "body", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "is_table_macro", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "provider_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false + ] + }, + "hash": "12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7" +} diff --git a/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json b/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json new file mode 100644 index 0000000000..c19df789d4 --- /dev/null +++ b/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, edited_by, edited_at, schedule, enabled, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n )\n SELECT\n $1, path, edited_by, edited_at, schedule, FALSE, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n FROM schedule WHERE workspace_id = $2 AND path NOT LIKE $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb" +} diff --git a/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json b/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json deleted file mode 100644 index 911d6c3b07..0000000000 --- a/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n CASE WHEN q.running\n THEN $3::text::jsonb\n ELSE $4::text::jsonb\n END,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n VALUES ($5, $1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "x", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346" -} diff --git a/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json b/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json new file mode 100644 index 0000000000..f548727abb --- /dev/null +++ b/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a" +} diff --git a/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json b/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json new file mode 100644 index 0000000000..058018a8ee --- /dev/null +++ b/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\", is_dev_workspace\n FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_parent!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_dev_workspace", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531" +} diff --git a/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json new file mode 100644 index 0000000000..708922a0cb --- /dev/null +++ b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, storage, inflight_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, upload_id)\n DO UPDATE SET inflight_bytes = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f" +} diff --git a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json b/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json deleted file mode 100644 index 75957a4f2f..0000000000 --- a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - { - "Custom": { - "name": "asset_usage_kind", - "kind": { - "Enum": [ - "script", - "flow", - "job" - ] - } - } - } - ] - }, - "nullable": [] - }, - "hash": "1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6" -} diff --git a/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json b/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json new file mode 100644 index 0000000000..b84cd8a14e --- /dev/null +++ b/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, name)\n DO UPDATE SET rules = EXCLUDED.rules,\n bypass_groups = EXCLUDED.bypass_groups,\n bypass_users = EXCLUDED.bypass_users\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int4", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242" +} diff --git a/backend/.sqlx/query-21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9.json b/backend/.sqlx/query-21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9.json new file mode 100644 index 0000000000..1a3d7ef0e4 --- /dev/null +++ b/backend/.sqlx/query-21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT consumer_path AS \"consumer_path!\", macro_name AS \"macro_name!\"\n FROM macro_usage\n WHERE workspace_id = $1\n AND ($2::text IS NULL OR consumer_path LIKE $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "consumer_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "macro_name!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9" +} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json index a53d131a3f..704883d4f1 100644 --- a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -38,7 +38,9 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset", + "freshness" ] } } @@ -75,7 +77,9 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035.json b/backend/.sqlx/query-246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035.json new file mode 100644 index 0000000000..f828b906cd --- /dev/null +++ b/backend/.sqlx/query-246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_definition WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035" +} diff --git a/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json b/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json new file mode 100644 index 0000000000..5f74d57491 --- /dev/null +++ b/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s)\n SELECT $2, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s\n FROM script_trigger WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46" +} diff --git a/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json new file mode 100644 index 0000000000..508eb32ed6 --- /dev/null +++ b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT COALESCE(MAX(depth), 0)::bigint AS \"depth!\" FROM chain\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "depth!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2" +} diff --git a/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json b/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json new file mode 100644 index 0000000000..a1e3edbbe2 --- /dev/null +++ b/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23" +} diff --git a/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json b/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json deleted file mode 100644 index 642723decf..0000000000 --- a/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) as \"c!\" FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "c!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4" -} diff --git a/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json b/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json new file mode 100644 index 0000000000..3ad3666413 --- /dev/null +++ b/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5\n FROM workspace WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee" +} diff --git a/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json new file mode 100644 index 0000000000..44d9f8f36d --- /dev/null +++ b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS \"height!\" FROM tree\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d" +} diff --git a/backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json b/backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json similarity index 71% rename from backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json rename to backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json index 50ed7549e8..6163a0f9fb 100644 --- a/backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json +++ b/backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2", + "query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM ws_specific ws\n WHERE ws.path = workspace_diff.path\n AND ws.item_kind = workspace_diff.kind\n AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id)\n )", "describe": { "columns": [ { @@ -55,5 +55,5 @@ true ] }, - "hash": "0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1" + "hash": "2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf" } diff --git a/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json b/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json new file mode 100644 index 0000000000..71615c6466 --- /dev/null +++ b/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'resource', $2::varchar\n WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1" +} diff --git a/backend/.sqlx/query-3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680.json b/backend/.sqlx/query-3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680.json new file mode 100644 index 0000000000..0624b86e0c --- /dev/null +++ b/backend/.sqlx/query-3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_diff\n (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork)\n VALUES ('test-workspace', 'wm-fork-guard-test', 'f/restricted/item', 'script', 1, 0, true, true, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680" +} diff --git a/backend/.sqlx/query-dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9.json b/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json similarity index 59% rename from backend/.sqlx/query-dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9.json rename to backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json index 6ae3f60e49..dfbc2b75e6 100644 --- a/backend/.sqlx/query-dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9.json +++ b/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT email FROM usr WHERE workspace_id = $1 AND username = $2", + "query": "SELECT path FROM schedule WHERE workspace_id = $1 AND path LIKE $2", "describe": { "columns": [ { "ordinal": 0, - "name": "email", + "name": "path", "type_info": "Varchar" } ], @@ -19,5 +19,5 @@ false ] }, - "hash": "dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9" + "hash": "333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba" } diff --git a/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json b/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json new file mode 100644 index 0000000000..1ca539c46d --- /dev/null +++ b/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"is_fork!\" FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fork!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd" +} diff --git a/backend/.sqlx/query-3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b.json b/backend/.sqlx/query-3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b.json new file mode 100644 index 0000000000..b814575234 --- /dev/null +++ b/backend/.sqlx/query-3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM skip_workspace_diff_tally WHERE workspace_id IN ('test-workspace', 'wm-fork-guard-test')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b" +} diff --git a/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json new file mode 100644 index 0000000000..16063b9a3f --- /dev/null +++ b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7" +} diff --git a/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json b/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json new file mode 100644 index 0000000000..d05abd4f3a --- /dev/null +++ b/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path AS \"asset_path!\", usage_path AS \"producer_path!\"\n FROM asset\n WHERE workspace_id = $1 AND kind = 'ducklake' AND path = ANY($2)\n AND usage_kind = 'script' AND usage_access_type IN ('w', 'rw')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "producer_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419" +} diff --git a/backend/.sqlx/query-36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4.json b/backend/.sqlx/query-36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4.json new file mode 100644 index 0000000000..d95cc6dc75 --- /dev/null +++ b/backend/.sqlx/query-36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name)\n SELECT $2, consumer_path, macro_name\n FROM macro_usage WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4" +} diff --git a/backend/.sqlx/query-3703847d0cef9df3a480af8b6a5c21b6af282c68f06ce037a11f703284d8980d.json b/backend/.sqlx/query-3703847d0cef9df3a480af8b6a5c21b6af282c68f06ce037a11f703284d8980d.json new file mode 100644 index 0000000000..3a784490ef --- /dev/null +++ b/backend/.sqlx/query-3703847d0cef9df3a480af8b6a5c21b6af282c68f06ce037a11f703284d8980d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT trigger_ref AS \"trigger_ref!\"\n FROM script_trigger\n WHERE workspace_id = $1\n AND runnable_path = $2\n AND trigger_kind = 'asset'\n AND runnable_kind = 'script'\n ORDER BY trigger_ref", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3703847d0cef9df3a480af8b6a5c21b6af282c68f06ce037a11f703284d8980d" +} diff --git a/backend/.sqlx/query-3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2.json b/backend/.sqlx/query-3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2.json new file mode 100644 index 0000000000..dcb81f26b8 --- /dev/null +++ b/backend/.sqlx/query-3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1 AND (consumer_path = $2 OR macro_name IN (SELECT name FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2" +} diff --git a/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json new file mode 100644 index 0000000000..291dc00310 --- /dev/null +++ b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2 AND created_at > now() - ($3::text)::interval)::bigint as \"other_reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b" +} diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 3568d1723e..f5ee767768 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -128,7 +128,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json new file mode 100644 index 0000000000..137af7638b --- /dev/null +++ b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129" +} diff --git a/backend/.sqlx/query-3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f.json b/backend/.sqlx/query-3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f.json new file mode 100644 index 0000000000..667907e290 --- /dev/null +++ b/backend/.sqlx/query-3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_definition (workspace_id, name, provider_path, params, body, is_table_macro)\n SELECT $2, name, provider_path, params, body, is_table_macro\n FROM macro_definition WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f" +} diff --git a/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json b/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json new file mode 100644 index 0000000000..60dbee76ab --- /dev/null +++ b/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT p.path AS \"path!\",\n (SELECT c.completed_at\n FROM v2_job j\n JOIN v2_job_completed c ON c.id = j.id\n WHERE j.workspace_id = $1\n AND j.runnable_path = p.path\n AND j.parent_job IS NULL\n -- No 'singlestepflow': flows may share a script's path, and\n -- a same-path flow run must not read as the script being\n -- fresh (false-fresh). Script retries land as native\n -- 'script' jobs; only the rare flow-wrapper fallback is\n -- missed, which errs stale. Kept in lockstep with the\n -- freshness watchdog's queries (freshness_watchdog_ee).\n AND j.kind IN ('script', 'preview')\n AND c.status = 'success'\n ORDER BY j.created_at DESC\n LIMIT 1) AS last_success_at\n FROM unnest($2::text[]) AS p(path)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "last_success_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed" +} diff --git a/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json b/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json new file mode 100644 index 0000000000..4599073796 --- /dev/null +++ b/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ducklake_name AS \"ducklake_name!\", metadata_schema AS \"metadata_schema!\",\n catalog AS \"catalog!\", storage AS \"storage!\",\n storage_ref AS \"storage_ref!\", data_path AS \"data_path!\",\n schema_dropped AS \"schema_dropped!\"\n FROM fork_ducklake_namespace WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "metadata_schema!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "catalog!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "storage!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "storage_ref!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "data_path!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "schema_dropped!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9" +} diff --git a/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json b/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json deleted file mode 100644 index 31c0ab4982..0000000000 --- a/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca" -} diff --git a/backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json b/backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json similarity index 57% rename from backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json rename to backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json index 4fe298370b..c12e0cdac4 100644 --- a/backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json +++ b/backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id)\n VALUES ($1, $2, $3, $4)", + "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id, is_dev_workspace)\n VALUES ($1, $2, $3, $4, $5)", "describe": { "columns": [], "parameters": { @@ -8,10 +8,11 @@ "Varchar", "Varchar", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c" + "hash": "41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056" } diff --git a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json b/backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json similarity index 76% rename from backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json rename to backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json index 0265d7d2b5..76eab1d117 100644 --- a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json +++ b/backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ", + "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ON CONFLICT DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078" + "hash": "42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7" } diff --git a/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json b/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json new file mode 100644 index 0000000000..b554ef8b55 --- /dev/null +++ b/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, content\n FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND archived = false AND deleted = false\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722" +} diff --git a/backend/.sqlx/query-447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2.json b/backend/.sqlx/query-447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2.json new file mode 100644 index 0000000000..1146134a34 --- /dev/null +++ b/backend/.sqlx/query-447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_definition (workspace_id, name, provider_path, params, body, is_table_macro) VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2" +} diff --git a/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json b/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json deleted file mode 100644 index 9e7ffed082..0000000000 --- a/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings)\n VALUES ($1, $2, $3)\n ON CONFLICT (hash)\n DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a" -} diff --git a/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json new file mode 100644 index 0000000000..cb6bb20397 --- /dev/null +++ b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COUNT(DISTINCT id) AS \"count!\" FROM tree WHERE id != $1 AND NOT deleted\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba" +} diff --git a/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json b/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json new file mode 100644 index 0000000000..5b3e70acbf --- /dev/null +++ b/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.runnable_path = $2\n AND j.parent_job IS NULL\n AND j.kind IN ('script', 'preview')\n AND (q.running = true OR q.scheduled_for <= now())\n ) AS \"in_flight!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "in_flight!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd" +} diff --git a/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json b/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json new file mode 100644 index 0000000000..6dc5fdb9a6 --- /dev/null +++ b/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE pipeline_freshness_state\n SET attempts = attempts + 1,\n last_push_at = now(),\n next_attempt_at = now()\n + (LEAST($3::bigint, $4::bigint * (1::bigint << LEAST(attempts + 1, 20)))::text\n || ' seconds')::interval\n WHERE workspace_id = $1 AND script_path = $2 AND next_attempt_at <= now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f" +} diff --git a/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json new file mode 100644 index 0000000000..18af751ca0 --- /dev/null +++ b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('workspace_multipart_inflight'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc" +} diff --git a/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json b/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json new file mode 100644 index 0000000000..5f114c0156 --- /dev/null +++ b/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH potential AS (\n SELECT email, operator FROM usr WHERE is_service_account IS false\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user\n WHERE email NOT IN (SELECT email FROM password WHERE disabled IS true)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44" +} diff --git a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json index 7d950f6d8f..7474818bf3 100644 --- a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json +++ b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json @@ -80,7 +80,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json b/backend/.sqlx/query-4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501.json similarity index 74% rename from backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json rename to backend/.sqlx/query-4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501.json index 3470285737..64330e91cf 100644 --- a/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json +++ b/backend/.sqlx/query-4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ORDER BY path, created_at DESC\n ", + "query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND (content LIKE '%' || $2 || '%' OR path = ANY($3))\n ORDER BY path, created_at DESC\n ", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "parameters": { "Left": [ "Text", - "Text" + "Text", + "TextArray" ] }, "nullable": [ @@ -25,5 +26,5 @@ false ] }, - "hash": "394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db" + "hash": "4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501" } diff --git a/backend/.sqlx/query-4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7.json b/backend/.sqlx/query-4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7.json new file mode 100644 index 0000000000..06b8754d5f --- /dev/null +++ b/backend/.sqlx/query-4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO notify_event (channel, payload) VALUES ('notify_macro_registry_change', $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7" +} diff --git a/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json b/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json new file mode 100644 index 0000000000..8e9e6a8713 --- /dev/null +++ b/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a" +} diff --git a/backend/.sqlx/query-6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b.json b/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json similarity index 66% rename from backend/.sqlx/query-6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b.json rename to backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json index 1e39bfdaab..bdb39d134f 100644 --- a/backend/.sqlx/query-6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b.json +++ b/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n ORDER BY d.email NULLS LAST", + "query": "SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n LEFT JOIN password p\n ON p.email = d.email\n AND p.super_admin = true\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n ORDER BY d.email NULLS LAST", "describe": { "columns": [ { @@ -60,5 +60,5 @@ false ] }, - "hash": "6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b" + "hash": "4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d" } diff --git a/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json b/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json deleted file mode 100644 index d165e78ad2..0000000000 --- a/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f" -} diff --git a/backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json b/backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json similarity index 57% rename from backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json rename to backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json index fce125c6d5..35b70a239c 100644 --- a/backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json +++ b/backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t", + "query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1 AND is_service_account IS false\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9" + "hash": "5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72" } diff --git a/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json b/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json new file mode 100644 index 0000000000..a8facb9d1b --- /dev/null +++ b/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, enabled, email, permissioned_as, summary, tag, cron_version\n ) VALUES ($1, $2, $3, 'Etc/UTC', $4, $2, false, true, $5, $6, $7, 'duckdb', 'v2')\n ON CONFLICT (workspace_id, path) DO UPDATE SET\n schedule = EXCLUDED.schedule,\n timezone = EXCLUDED.timezone,\n edited_by = EXCLUDED.edited_by,\n edited_at = now(),\n script_path = EXCLUDED.script_path,\n is_flow = false,\n enabled = true,\n email = EXCLUDED.email,\n permissioned_as = EXCLUDED.permissioned_as,\n summary = EXCLUDED.summary,\n tag = EXCLUDED.tag,\n cron_version = EXCLUDED.cron_version,\n error = NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe" +} diff --git a/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json b/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json new file mode 100644 index 0000000000..8bce508e75 --- /dev/null +++ b/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2) AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109" +} diff --git a/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json b/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json new file mode 100644 index 0000000000..580666f07d --- /dev/null +++ b/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 FROM v2_job j\n JOIN v2_job_completed c ON c.id = j.id\n WHERE j.workspace_id = $1\n AND j.runnable_path = $2\n AND j.parent_job IS NULL\n AND j.kind IN ('script', 'preview')\n AND c.status = 'success'\n AND c.completed_at > now() - ($3::bigint::text || ' seconds')::interval\n ) AS \"fresh!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "fresh!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89" +} diff --git a/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json b/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json deleted file mode 100644 index f634fc2d4b..0000000000 --- a/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH mine AS (\n SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1\n ), claim_self AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE id = $1 AND consumed_at IS NULL\n RETURNING debounce_batch\n ), claim_rest AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE debounce_batch = (SELECT debounce_batch FROM claim_self)\n AND id <> $1 AND consumed_at IS NULL\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM mine) AS \"had_row!\",\n (SELECT debounce_batch FROM claim_self) AS claimed_batch,\n (SELECT consumed_by FROM mine) AS prev_consumed_by,\n ARRAY(SELECT id FROM claim_rest) AS \"claimed_ids!\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "had_row!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "claimed_batch", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "prev_consumed_by", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "claimed_ids!", - "type_info": "UuidArray" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89" -} diff --git a/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json new file mode 100644 index 0000000000..cd2dd23319 --- /dev/null +++ b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3))::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce" +} diff --git a/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json new file mode 100644 index 0000000000..be54b4a25d --- /dev/null +++ b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2" +} diff --git a/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json b/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json new file mode 100644 index 0000000000..18e0936d3a --- /dev/null +++ b/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO pipeline_freshness_state\n (workspace_id, script_path, attempts, last_push_at, next_attempt_at)\n VALUES ($1, $2, 1, now(),\n now() + (LEAST($3::bigint, $4::bigint * 2)::text || ' seconds')::interval)\n ON CONFLICT (workspace_id, script_path) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c" +} diff --git a/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json b/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json new file mode 100644 index 0000000000..1bcb7e6e53 --- /dev/null +++ b/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH ancestor AS (\n SELECT wid, ord FROM unnest($1::text[]) WITH ORDINALITY AS a(wid, ord)\n ), anc_mat AS (\n -- Per table, the nearest ancestor (lowest ord) that materialized it.\n SELECT DISTINCT ON (mp.asset_path) mp.asset_path, a.wid, a.ord\n FROM materialized_partition mp\n JOIN ancestor a ON a.wid = mp.workspace_id\n WHERE mp.asset_kind = 'ducklake' AND mp.status = 'materialized'\n AND split_part(mp.asset_path, '/', 1) = $3 AND mp.asset_path LIKE '%/%'\n ORDER BY mp.asset_path, a.ord\n ), fork_mat AS (\n -- Fork-OWNED assets: anything whose physical table exists in the fork\n -- namespace, not just clean materializations. A committed write whose data\n -- tests failed afterwards records status='failed' WITH a snapshot — its table\n -- is real, and a defer view emitted over it would silently yield to it\n -- (CREATE VIEW IF NOT EXISTS) while claiming the read defers to the parent.\n SELECT DISTINCT asset_path FROM materialized_partition\n WHERE workspace_id = $2 AND asset_kind = 'ducklake'\n AND (status = 'materialized' OR snapshot_id IS NOT NULL)\n ), latest_schema AS (\n SELECT DISTINCT ON (workspace_id, asset_path) workspace_id, asset_path, columns\n FROM materialized_asset_schema\n WHERE workspace_id = ANY($1) AND asset_kind = 'ducklake'\n ORDER BY workspace_id, asset_path, version DESC\n )\n SELECT am.asset_path AS \"asset_path!\",\n am.ord AS \"ord!\",\n COALESCE(EXISTS (\n SELECT 1 FROM jsonb_array_elements(ls.columns) e\n WHERE e->>'name' = 'is_current'\n ), false) AS \"has_current!\"\n FROM anc_mat am\n LEFT JOIN latest_schema ls\n ON ls.asset_path = am.asset_path AND ls.workspace_id = am.wid\n WHERE am.asset_path NOT IN (SELECT asset_path FROM fork_mat)\n ORDER BY am.asset_path\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "ord!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "has_current!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541" +} diff --git a/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json b/backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json similarity index 63% rename from backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json rename to backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json index bb80e8d19a..a9cdc35dce 100644 --- a/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json +++ b/backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", "describe": { "columns": [ { @@ -30,11 +30,16 @@ }, { "ordinal": 5, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 6, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 6, + "ordinal": 7, "name": "disabled", "type_info": "Bool" } @@ -50,9 +55,10 @@ false, true, true, + false, null, false ] }, - "hash": "c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6" + "hash": "63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19" } diff --git a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json deleted file mode 100644 index cb20ec2ffb..0000000000 --- a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b" -} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json index 08ebe6bba5..fcc16e9a7c 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json @@ -161,7 +161,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e.json b/backend/.sqlx/query-689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e.json new file mode 100644 index 0000000000..83e2711cf5 --- /dev/null +++ b/backend/.sqlx/query-689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name AS \"name!\", provider_path AS \"provider_path!\",\n params AS \"params!\", is_table_macro AS \"is_table_macro!\"\n FROM macro_definition\n WHERE workspace_id = $1\n ORDER BY provider_path, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "provider_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "params!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "is_table_macro!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e" +} diff --git a/backend/.sqlx/query-6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97.json b/backend/.sqlx/query-6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97.json new file mode 100644 index 0000000000..4dfdcaab97 --- /dev/null +++ b/backend/.sqlx/query-6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name FROM macro_definition WHERE workspace_id = $1 AND provider_path != $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97" +} diff --git a/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json b/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json new file mode 100644 index 0000000000..9a0bbf46ef --- /dev/null +++ b/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET ducklake = jsonb_set(ducklake, ARRAY['ducklakes', $2, 'fork_behavior'], '\"shared\"')\n WHERE workspace_id = $1 AND ducklake->'ducklakes' ? $2\n RETURNING 1 AS \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7" +} diff --git a/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json b/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json new file mode 100644 index 0000000000..4e8c7acee2 --- /dev/null +++ b/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE pipeline_freshness_state SET next_attempt_at = now() - interval '1 second'\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f" +} diff --git a/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json b/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json deleted file mode 100644 index e885c72039..0000000000 --- a/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '1 hour'\n RETURNING 1\n ) SELECT count(*) FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18" -} diff --git a/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json new file mode 100644 index 0000000000..acfa08d0bf --- /dev/null +++ b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT id AS \"id!\" FROM tree WHERE id != $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d" +} diff --git a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json deleted file mode 100644 index c5266c66b5..0000000000 --- a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT runnable_path AS \"runnable_path!\", kind::text AS \"kind!\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "runnable_path!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "kind!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc" -} diff --git a/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json b/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json new file mode 100644 index 0000000000..b6b0f06be5 --- /dev/null +++ b/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ducklake->'ducklakes' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb" +} diff --git a/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json b/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json new file mode 100644 index 0000000000..f425d1eebd --- /dev/null +++ b/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\" FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_parent!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c" +} diff --git a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json b/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json deleted file mode 100644 index f7d3361d99..0000000000 --- a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523" -} diff --git a/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json b/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json new file mode 100644 index 0000000000..e1cf9bc17f --- /dev/null +++ b/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376" +} diff --git a/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json b/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json new file mode 100644 index 0000000000..5c8ab6fb5b --- /dev/null +++ b/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT ON (workspace_id, path)\n workspace_id AS \"workspace_id!\", path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND content ILIKE '%freshness%'\n -- Workspace archival stops all execution but leaves script rows\n -- intact for unarchival; without this the watchdog would keep\n -- resurrecting runs in a workspace the admin shut down.\n AND EXISTS (SELECT 1 FROM workspace w\n WHERE w.id = script.workspace_id AND w.deleted = false)\n ORDER BY workspace_id, path, created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "content!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a" +} diff --git a/backend/.sqlx/query-7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d.json b/backend/.sqlx/query-7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d.json new file mode 100644 index 0000000000..f5fec50340 --- /dev/null +++ b/backend/.sqlx/query-7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE macro_usage SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d" +} diff --git a/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json new file mode 100644 index 0000000000..e171d44af5 --- /dev/null +++ b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8" +} diff --git a/backend/.sqlx/query-80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2.json b/backend/.sqlx/query-80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2.json new file mode 100644 index 0000000000..384ea81940 --- /dev/null +++ b/backend/.sqlx/query-80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by)\n VALUES ('test-workspace', 'restricted', 'restricted', ARRAY['u/test-user']::varchar[], '{\"u/test-user\": true}'::jsonb, '', 'test-user')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2" +} diff --git a/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json b/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json deleted file mode 100644 index 7a0c5578b0..0000000000 --- a/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', '1970-01-01T00:00:00+00:00')\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23" -} diff --git a/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json b/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json new file mode 100644 index 0000000000..3f6a4a8136 --- /dev/null +++ b/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM pipeline_freshness_state s\n WHERE NOT EXISTS (\n SELECT 1 FROM unnest($1::text[], $2::text[]) AS w(workspace_id, script_path)\n WHERE w.workspace_id = s.workspace_id AND w.script_path = s.script_path\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b" +} diff --git a/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json b/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json deleted file mode 100644 index c0573a8fab..0000000000 --- a/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace SELECT $1, $2, owner, false, premium FROM workspace WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745" -} diff --git a/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json b/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json new file mode 100644 index 0000000000..673becc2ff --- /dev/null +++ b/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "deleted", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38" +} diff --git a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json deleted file mode 100644 index 96587ed1d0..0000000000 --- a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT q.runnable_settings_handle\n FROM v2_job j JOIN v2_job_queue q ON q.id = j.id\n WHERE j.workspace_id = $1 AND j.runnable_path = $2\n AND j.trigger_kind = 'asset'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "runnable_settings_handle", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2" -} diff --git a/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json b/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json new file mode 100644 index 0000000000..0859d3e071 --- /dev/null +++ b/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec" +} diff --git a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json similarity index 63% rename from backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json rename to backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json index 17d372249b..5aacf1a295 100644 --- a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json +++ b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n snapshot_id = EXCLUDED.snapshot_id,\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", + "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n -- A failed run records no snapshot, but must not erase the last\n -- committed one: a physical table from an earlier commit (or from a\n -- committed write whose data tests then failed) still exists, and\n -- fork defer/graph state keys on that evidence.\n snapshot_id = COALESCE(EXCLUDED.snapshot_id, materialized_partition.snapshot_id),\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", "describe": { "columns": [], "parameters": { @@ -43,5 +43,5 @@ }, "nullable": [] }, - "hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19" + "hash": "853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424" } diff --git a/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json new file mode 100644 index 0000000000..7250aff598 --- /dev/null +++ b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at < now() - ($2::text)::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f" +} diff --git a/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json b/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json new file mode 100644 index 0000000000..5741a1aa5b --- /dev/null +++ b/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND schedule.path NOT LIKE $4\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f" +} diff --git a/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json b/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json new file mode 100644 index 0000000000..829083e0ec --- /dev/null +++ b/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.ducklake->'ducklakes' AS ducklake_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e" +} diff --git a/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json b/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json new file mode 100644 index 0000000000..9fc8cf1336 --- /dev/null +++ b/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM schedule WHERE workspace_id = $1 AND starts_with(path, $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3" +} diff --git a/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json b/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json new file mode 100644 index 0000000000..efd12114f3 --- /dev/null +++ b/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n -- Fork rows also count when a snapshot ever committed (a failed run\n -- preserves it): the physical table exists, so reads hit the FORK's\n -- data — showing 'deferred' would misstate what a query returns.\n -- Ancestor rows still require a clean materialization.\n SELECT DISTINCT asset_path AS \"asset_path!\", workspace_id AS \"workspace_id!\"\n FROM materialized_partition\n WHERE (workspace_id = $1 OR workspace_id = ANY($2))\n AND asset_kind = 'ducklake'\n AND (status = 'materialized'\n OR (workspace_id = $1 AND snapshot_id IS NOT NULL))\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444" +} diff --git a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json b/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json deleted file mode 100644 index f65ea6d2d4..0000000000 --- a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b" -} diff --git a/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json b/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json new file mode 100644 index 0000000000..3f159707b6 --- /dev/null +++ b/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'variable', $2::varchar\n WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c" +} diff --git a/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json b/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json deleted file mode 100644 index 569a5122ba..0000000000 --- a/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status = 'success' AS \"success!\"\n FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "success!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Uuid", - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b" -} diff --git a/backend/.sqlx/query-919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad.json b/backend/.sqlx/query-919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad.json new file mode 100644 index 0000000000..34c18e2633 --- /dev/null +++ b/backend/.sqlx/query-919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag FROM flow WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad" +} diff --git a/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json new file mode 100644 index 0000000000..c0207b6bc7 --- /dev/null +++ b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, part_id, storage, part_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, upload_id, part_id)\n DO UPDATE SET part_bytes = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a" +} diff --git a/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json new file mode 100644 index 0000000000..253fe79753 --- /dev/null +++ b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n -- reservation of every OTHER in-flight upload\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2\n AND created_at > now() - ($4::text)::interval\n GROUP BY upload_id) t)::bigint as \"other_reserved!\",\n -- this upload's already-recorded parts, excluding the part being (re)uploaded\n (SELECT COALESCE(SUM(part_bytes), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 AND part_id <> $3)::bigint as \"this_other_parts!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "this_other_parts!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c" +} diff --git a/backend/.sqlx/query-9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c.json b/backend/.sqlx/query-9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c.json new file mode 100644 index 0000000000..07a14677e4 --- /dev/null +++ b/backend/.sqlx/query-9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c" +} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index ad9e57801e..ab01730c02 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -35,7 +35,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770.json b/backend/.sqlx/query-9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770.json new file mode 100644 index 0000000000..64dfe64198 --- /dev/null +++ b/backend/.sqlx/query-9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM password WHERE username IS NOT NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770" +} diff --git a/backend/.sqlx/query-a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb.json b/backend/.sqlx/query-a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb.json new file mode 100644 index 0000000000..9ddc929622 --- /dev/null +++ b/backend/.sqlx/query-a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, provider_path FROM macro_definition WHERE workspace_id = $1 AND name = ANY($2) LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "provider_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb" +} diff --git a/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json b/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json new file mode 100644 index 0000000000..e3e8e3f2ad --- /dev/null +++ b/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path LIKE $2 AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d" +} diff --git a/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json b/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json new file mode 100644 index 0000000000..2b9b0619a1 --- /dev/null +++ b/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0" +} diff --git a/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json new file mode 100644 index 0000000000..0c3fdb8d0a --- /dev/null +++ b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_xact_lock(hashtext('workspace_storage_usage'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_xact_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03" +} diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index 9a21f228ea..405904604a 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -191,7 +191,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json b/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json deleted file mode 100644 index 819928ddc1..0000000000 --- a/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END\n FROM workspace WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d" -} diff --git a/backend/.sqlx/query-a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5.json b/backend/.sqlx/query-a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5.json new file mode 100644 index 0000000000..986638f649 --- /dev/null +++ b/backend/.sqlx/query-a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1 AND (consumer_path = $2 OR consumer_path = $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5" +} diff --git a/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json new file mode 100644 index 0000000000..691016a610 --- /dev/null +++ b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') as \"key!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d" +} diff --git a/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json b/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json new file mode 100644 index 0000000000..21e0f4613b --- /dev/null +++ b/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND starts_with(path, $2) AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9" +} diff --git a/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json new file mode 100644 index 0000000000..0f1bcc2271 --- /dev/null +++ b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT inflight_bytes, target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inflight_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f" +} diff --git a/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json b/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json new file mode 100644 index 0000000000..5d20f83f9d --- /dev/null +++ b/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077" +} diff --git a/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json b/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json new file mode 100644 index 0000000000..b022524bbb --- /dev/null +++ b/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e" +} diff --git a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json deleted file mode 100644 index d12d305b92..0000000000 --- a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT path AS \"path!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b" -} diff --git a/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json new file mode 100644 index 0000000000..1148679230 --- /dev/null +++ b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(SUM(bytes), 0)::bigint as \"total!\",\n COALESCE(MIN(computed_at) < now() - interval '10 minutes', true) as \"stale!\"\n FROM workspace_storage_usage WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683" +} diff --git a/backend/.sqlx/query-ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360.json b/backend/.sqlx/query-ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360.json new file mode 100644 index 0000000000..e543db2a30 --- /dev/null +++ b/backend/.sqlx/query-ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360" +} diff --git a/backend/.sqlx/query-ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3.json b/backend/.sqlx/query-ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3.json new file mode 100644 index 0000000000..1b4082b18a --- /dev/null +++ b/backend/.sqlx/query-ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(\n (SELECT email FROM usr WHERE workspace_id = $1 AND username = $2),\n (SELECT email FROM password WHERE (username = $2 OR email = $2) AND super_admin = true)\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3" +} diff --git a/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json b/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json new file mode 100644 index 0000000000..d5b10b13b9 --- /dev/null +++ b/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT attempts FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "attempts", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index 3efa843923..092d15e592 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -166,7 +166,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03.json b/backend/.sqlx/query-b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03.json new file mode 100644 index 0000000000..545a9c161b --- /dev/null +++ b/backend/.sqlx/query-b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_definition WHERE workspace_id = $1 AND (provider_path = $2 OR provider_path = $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03" +} diff --git a/backend/.sqlx/query-b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6.json b/backend/.sqlx/query-b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6.json new file mode 100644 index 0000000000..c6022d3013 --- /dev/null +++ b/backend/.sqlx/query-b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_diff\n (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork)\n VALUES\n ('test-workspace', 'wm-fork-test-workspace', 'f/rt/ghost', 'http_trigger', 1, 0, true, false, true),\n ('test-workspace', 'wm-fork-test-workspace', 'f/rt/ghost_behind', 'http_trigger', 0, 1, true, true, false)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6" +} diff --git a/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json new file mode 100644 index 0000000000..94cf77ebc5 --- /dev/null +++ b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)\n SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via\n FROM usr WHERE workspace_id = $2\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142" +} diff --git a/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json new file mode 100644 index 0000000000..96608e1bc2 --- /dev/null +++ b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, GREATEST($3::bigint, 0), to_timestamp(0))\n ON CONFLICT (workspace_id, storage)\n DO UPDATE SET bytes = GREATEST(workspace_storage_usage.bytes + $3::bigint, 0)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616" +} diff --git a/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json b/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json new file mode 100644 index 0000000000..ce77275faf --- /dev/null +++ b/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pipeline_freshness_state (workspace_id, script_path, attempts) VALUES ($1, $2, 3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43" +} diff --git a/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json new file mode 100644 index 0000000000..514024eb8d --- /dev/null +++ b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b" +} diff --git a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json index f6ff25a4bf..419ab26383 100644 --- a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json +++ b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json @@ -80,7 +80,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 8dc66064dc..b9d33a6b5f 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -111,7 +111,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json b/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json new file mode 100644 index 0000000000..35d3ace40e --- /dev/null +++ b/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_diff SET has_changes = NULL\n WHERE path = $2 AND kind = $3\n AND ($1 IN (source_workspace_id, fork_workspace_id))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db" +} diff --git a/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json b/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json new file mode 100644 index 0000000000..a1f7783645 --- /dev/null +++ b/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM workspace\n WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de" +} diff --git a/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json b/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json new file mode 100644 index 0000000000..7cfc1f109f --- /dev/null +++ b/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (asset_path)\n asset_path, version, columns AS \"columns: Json>\", captured_at\n FROM materialized_asset_schema\n WHERE workspace_id = $1 AND asset_kind = 'ducklake' AND asset_path = ANY($2)\n ORDER BY asset_path, version DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "columns: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "captured_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896" +} diff --git a/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json b/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json new file mode 100644 index 0000000000..d316c16fff --- /dev/null +++ b/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, edited_by, edited_at, schedule, enabled, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n )\n SELECT\n $1, path, edited_by, edited_at, schedule, FALSE, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n FROM schedule WHERE workspace_id = $2 AND NOT starts_with(path, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0" +} diff --git a/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json b/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json deleted file mode 100644 index 7361b645b5..0000000000 --- a/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch, consumed_at) VALUES\n ($1, nextval('debounce_batch_seq'), now() - interval '20 minutes'),\n ($2, nextval('debounce_batch_seq'), now() - interval '1 minute'),\n ($3, nextval('debounce_batch_seq'), NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed" -} diff --git a/backend/.sqlx/query-c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05.json b/backend/.sqlx/query-c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05.json new file mode 100644 index 0000000000..c1e5495204 --- /dev/null +++ b/backend/.sqlx/query-c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1 AND macro_name = ANY($2) AND consumer_path != $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05" +} diff --git a/backend/.sqlx/query-cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3.json b/backend/.sqlx/query-cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3.json new file mode 100644 index 0000000000..8f0841180b --- /dev/null +++ b/backend/.sqlx/query-cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT m.name AS \"name!\", m.params AS \"params!\", m.body AS \"body!\",\n m.is_table_macro AS \"is_table_macro!\", m.provider_path AS \"provider_path!\"\n FROM macro_definition m\n WHERE m.workspace_id = $1\n AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = m.workspace_id\n AND s.path = m.provider_path\n AND s.archived = false\n AND s.deleted = false\n )\n ORDER BY m.provider_path, m.name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "params!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "body!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "is_table_macro!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "provider_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false + ] + }, + "hash": "cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3" +} diff --git a/backend/.sqlx/query-cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345.json b/backend/.sqlx/query-cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345.json new file mode 100644 index 0000000000..1419a1a20c --- /dev/null +++ b/backend/.sqlx/query-cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE macro_definition SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345" +} diff --git a/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json b/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json new file mode 100644 index 0000000000..0fac858572 --- /dev/null +++ b/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE depth > 0 ORDER BY depth\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f" +} diff --git a/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json new file mode 100644 index 0000000000..abb128f463 --- /dev/null +++ b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT storage, bytes, computed_at FROM workspace_storage_usage\n WHERE workspace_id = $1 ORDER BY storage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "storage", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "computed_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b" +} diff --git a/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json b/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json new file mode 100644 index 0000000000..83f7394250 --- /dev/null +++ b/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE fork_ducklake_namespace SET schema_dropped = true\n WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3\n AND storage = $4 AND storage_ref = $5 AND data_path = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3" +} diff --git a/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json b/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json deleted file mode 100644 index e900b0f9e0..0000000000 --- a/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH ids AS (\n SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )\n ) SELECT args->>$2 FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266" -} diff --git a/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json b/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json new file mode 100644 index 0000000000..f705dbc337 --- /dev/null +++ b/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET ducklake = jsonb_set(ducklake, '{ducklakes}', (\n SELECT COALESCE(jsonb_object_agg(key, value - 'fork_behavior'), '{}'::jsonb)\n FROM jsonb_each(ducklake->'ducklakes')\n ))\n WHERE workspace_id = $1 AND jsonb_typeof(ducklake->'ducklakes') = 'object'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7" +} diff --git a/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json b/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json new file mode 100644 index 0000000000..cbf96dad52 --- /dev/null +++ b/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pipeline_freshness_state (workspace_id, script_path) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706" +} diff --git a/backend/.sqlx/query-d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3.json b/backend/.sqlx/query-d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3.json new file mode 100644 index 0000000000..fe92251342 --- /dev/null +++ b/backend/.sqlx/query-d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, email, username, is_admin, role)\n VALUES ('wm-fork-guard-test', 'test2@windmill.dev', 'test-user-2', true, 'Admin')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3" +} diff --git a/backend/.sqlx/query-d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785.json b/backend/.sqlx/query-d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785.json new file mode 100644 index 0000000000..deed0f1cdb --- /dev/null +++ b/backend/.sqlx/query-d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785" +} diff --git a/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json new file mode 100644 index 0000000000..1fd1fbaece --- /dev/null +++ b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_storage_usage WHERE workspace_id = $1 AND storage != ALL($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a" +} diff --git a/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json b/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json new file mode 100644 index 0000000000..9f8b86b4e6 --- /dev/null +++ b/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2\n AND NOT EXISTS (\n SELECT 1 FROM workspace\n WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5" +} diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index d5365ffe94..d97c02d26b 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -111,7 +111,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json index 33a5534b42..052d83fcd9 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json @@ -251,7 +251,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json new file mode 100644 index 0000000000..5609adaad8 --- /dev/null +++ b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3)\n GROUP BY upload_id) t)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c" +} diff --git a/backend/.sqlx/query-d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e.json b/backend/.sqlx/query-d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e.json new file mode 100644 index 0000000000..3a5531b9a6 --- /dev/null +++ b/backend/.sqlx/query-d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT usage_path FROM asset\n WHERE workspace_id = $1\n AND kind = 'ducklake'\n AND path = $2\n AND usage_kind = 'script'\n AND usage_access_type IN ('w', 'rw')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e" +} diff --git a/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json b/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json new file mode 100644 index 0000000000..4a511c2d6e --- /dev/null +++ b/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996" +} diff --git a/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json b/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json new file mode 100644 index 0000000000..48a5c693ac --- /dev/null +++ b/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8" +} diff --git a/backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json b/backend/.sqlx/query-dc60d43814e2b5220db648677dbb3656e27f1c962c2f0438e0cc06530661fbf8.json similarity index 53% rename from backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json rename to backend/.sqlx/query-dc60d43814e2b5220db648677dbb3656e27f1c962c2f0438e0cc06530661fbf8.json index f2398bf143..0d36677337 100644 --- a/backend/.sqlx/query-caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439.json +++ b/backend/.sqlx/query-dc60d43814e2b5220db648677dbb3656e27f1c962c2f0438e0cc06530661fbf8.json @@ -1,17 +1,16 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT ws.datatable->'datatables'->$2 AS config\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "query": "\n SELECT ws.datatable->'datatables' AS datatables\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "config", + "name": "datatables", "type_info": "Jsonb" } ], "parameters": { "Left": [ - "Text", "Text" ] }, @@ -19,5 +18,5 @@ null ] }, - "hash": "caf4dc1046769f410d1277cdd3747763e0d674e8a1d9f3512629cb5a9e5a3439" + "hash": "dc60d43814e2b5220db648677dbb3656e27f1c962c2f0438e0cc06530661fbf8" } diff --git a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json deleted file mode 100644 index 2ba317edc6..0000000000 --- a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_path = $2\n AND usage_access_type IN ('w', 'rw')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "kind!: AssetKind", - "type_info": { - "Custom": { - "name": "asset_kind", - "kind": { - "Enum": [ - "s3object", - "resource", - "variable", - "ducklake", - "datatable", - "volume" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "path!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b" -} diff --git a/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json b/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json deleted file mode 100644 index 67ce9d8719..0000000000 --- a/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Varchar", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef" -} diff --git a/backend/.sqlx/query-e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a.json b/backend/.sqlx/query-e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a.json new file mode 100644 index 0000000000..c3db56398e --- /dev/null +++ b/backend/.sqlx/query-e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND language = 'duckdb'::script_lang\n AND archived = false\n AND deleted = false\n AND path != $2\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "content!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a" +} diff --git a/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json b/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json deleted file mode 100644 index f38c023cb3..0000000000 --- a/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH potential AS (\n SELECT email, operator FROM usr\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "operators!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455" -} diff --git a/backend/.sqlx/query-e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c.json b/backend/.sqlx/query-e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c.json new file mode 100644 index 0000000000..4609e7adf4 --- /dev/null +++ b/backend/.sqlx/query-e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2) AS \"e!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "e!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c" +} diff --git a/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json b/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json new file mode 100644 index 0000000000..37bd25eb98 --- /dev/null +++ b/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO fork_ducklake_namespace\n (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path)\n DO UPDATE SET schema_dropped = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 470c651020..a35373a959 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -191,7 +191,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json b/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json new file mode 100644 index 0000000000..8c80c14ea4 --- /dev/null +++ b/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", created_by AS \"created_by!\",\n args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'freshness'\n ORDER BY created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false, + true + ] + }, + "hash": "e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a" +} diff --git a/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json b/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json deleted file mode 100644 index a1b52e81fd..0000000000 --- a/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status = 'success' AS \"success!\",\n result AS \"result: Json>\",\n started_at AS \"started_at!\"FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "success!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "result: Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "started_at!", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Uuid", - "Int8" - ] - }, - "nullable": [ - null, - true, - true - ] - }, - "hash": "e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976" -} diff --git a/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json new file mode 100644 index 0000000000..4f0731f5e4 --- /dev/null +++ b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, $3, now())\n ON CONFLICT (workspace_id, storage) DO UPDATE SET bytes = $3, computed_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c" +} diff --git a/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json b/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json deleted file mode 100644 index 146c7f05b3..0000000000 --- a/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT concurrency_settings, debouncing_settings FROM runnable_settings WHERE hash = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "concurrency_settings", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "debouncing_settings", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01" -} diff --git a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json new file mode 100644 index 0000000000..1930427b94 --- /dev/null +++ b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "developers!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b" +} diff --git a/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json b/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json new file mode 100644 index 0000000000..844dfb3e47 --- /dev/null +++ b/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c" +} diff --git a/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json b/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json new file mode 100644 index 0000000000..8257722c5c --- /dev/null +++ b/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) AS \"count!\" FROM pipeline_freshness_state WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472" +} diff --git a/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json b/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json new file mode 100644 index 0000000000..7049fafe37 --- /dev/null +++ b/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n SELECT $2, path, kind, usage_access_type, usage_path, usage_kind, columns\n FROM asset WHERE workspace_id = $1 AND usage_kind IN ('script', 'flow')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1" +} diff --git a/backend/.sqlx/query-f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc.json b/backend/.sqlx/query-f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc.json new file mode 100644 index 0000000000..9e32600de9 --- /dev/null +++ b/backend/.sqlx/query-f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc" +} diff --git a/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json b/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json new file mode 100644 index 0000000000..7f967b60a5 --- /dev/null +++ b/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d" +} diff --git a/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json b/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json new file mode 100644 index 0000000000..ad17a60330 --- /dev/null +++ b/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM fork_ducklake_namespace\n WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3\n AND storage = $4 AND storage_ref = $5 AND data_path = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588" +} diff --git a/backend/.sqlx/query-fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98.json b/backend/.sqlx/query-fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98.json new file mode 100644 index 0000000000..626c541556 --- /dev/null +++ b/backend/.sqlx/query-fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98" +} diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 9ff2c0be90..24cf2cb837 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -6,3 +6,57 @@ - **DB schema**: `backend/summarized_schema.txt` - **API routes entry point**: `windmill-api/src/lib.rs` - **OpenAPI spec**: `windmill-api/openapi.yaml` +- **DuckDB local jobs**: build the dynamic FFI library before running DuckDB scripts locally: + ```bash + cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh + ``` + Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. + The bundled DuckDB compile (~2min) is cached in a per-user dir shared across + worktrees (keyed by the crate's `Cargo.lock`), so a fresh worktree reuses it and + the build is near-instant — you don't pay the full compile per worktree. Editing + the FFI crate's own source falls back to an isolated per-worktree `./target`. +- **Running data pipelines (DuckLake) from source**: see the section below — a plain build + advertises the `duckdb` tag but cannot execute DuckDB scripts and has no working S3 proxy. + +## Running data pipelines (DuckLake) from source + +DuckLake pipelines need **both** the right cargo features **and** the prebuilt DuckDB FFI. A +plain `cargo run` (or `cargo run --features quickjs`) does **not** suffice, and the failure modes +are silent-ish, so agents lose time. Verify feature names against `backend/Cargo.toml` `[features]`. + +**Feature sets** (run from `backend/`): + +| Goal | Command | +|---|---| +| CE DuckLake (DuckDB scripts + S3 proxy) | `cargo run --features quickjs,duckdb,parquet,private` | +| + Python scripts | add `,python` | +| EE features (WAP, partitioning, forks, …) | add `,enterprise,license` | + +`enterprise` already pulls in `license`, but list both when you want the license-gated paths. +`quickjs` is for JS eval, not DuckLake per se — keep it if your baseline build had it. + +**Before running any DuckDB script**, build the FFI (see the bullet above): +`cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. + +**Two gotchas that a wrong feature set produces:** + +1. **`duckdb` tag advertised, feature missing.** The `duckdb` worker tag is in the *unconditional* + default tag list (`windmill-common/src/worker.rs`, `DEFAULT_TAGS`), so a worker advertises it even + without the `duckdb` feature. Jobs then dispatch but fail at execution with + `"Duck DB requires the duckdb feature to be enabled"` (`windmill-worker/src/worker.rs`). Fix: + compile with `--features duckdb`. +2. **DuckLake writes 404 (no S3 proxy).** The workspace S3 proxy (`/w/{ws}/s3_proxy/*`) that + DuckLake uses for reads/writes only mounts the real service under + `#[cfg(all(feature = "private", feature = "parquet"))]` (`windmill-api/src/s3_proxy_oss.rs`); + otherwise it's an empty router and every proxied request 404s. Fix: compile with **both** + `private` and `parquet`. + +## Cloud vs self-hosted gating + +The `cloud` cargo feature is compiled into **all** EE builds, so `#[cfg(feature = "cloud")]` is **not** a "cloud-only" runtime gate — it only means the code is present. The real gate for behavior specific to the managed cloud (app.windmill.dev) is the runtime flag `*CLOUD_HOSTED` (`windmill_common::worker::CLOUD_HOSTED`, from the `CLOUD_HOSTED` env var; note it's loaded from `.env` via `dotenv`, so it won't show in `/proc//environ` — check the running behavior, not the exec env). + +Cloud-only logic must be behind `if *CLOUD_HOSTED { ... }`: feature-gate the helper so it compiles, then **runtime-gate the call**. `#[cfg(feature = "cloud")]` on its own is only sufficient for: +- pure helper/struct definitions (they only run when a gated caller invokes them), +- code already inside an `if *CLOUD_HOSTED { ... }` block, +- handlers that early-return on `!*CLOUD_HOSTED`, +- idempotent no-ops that are harmless off-cloud (e.g. cache invalidation). diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7072389c21..dcf45d0b1c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -237,9 +237,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" @@ -517,7 +517,7 @@ dependencies = [ "futures-core", "libc", "portable-atomic", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "tokio", "tokio-stream", "xattr", @@ -775,9 +775,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -785,14 +785,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1530,7 +1531,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "shlex 1.3.0", "syn 2.0.118", ] @@ -1550,7 +1551,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "shlex 1.3.0", "syn 2.0.118", ] @@ -2056,9 +2057,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "jobserver", @@ -2601,9 +2602,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.89+curl-8.20.0" +version = "0.4.90+curl-8.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22" +checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" dependencies = [ "cc", "libc", @@ -4211,7 +4212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf592ae6a864437e98ef9c6ae7936b822077e9d038a3a48ee081ab92313afad4" dependencies = [ "num-bigint", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "swc_atoms", "swc_common", "swc_ecma_ast", @@ -5071,9 +5072,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -5552,7 +5555,7 @@ dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "triomphe", ] @@ -5660,9 +5663,9 @@ dependencies = [ [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" @@ -6085,9 +6088,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -6234,11 +6237,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -6626,14 +6629,14 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "bitflags 2.13.0", "libc", "plain", - "redox_syscall 0.8.1", + "redox_syscall 0.9.0", ] [[package]] @@ -7245,7 +7248,7 @@ dependencies = [ "native-tls", "pem 3.0.6", "percent-encoding", - "rand 0.10.1", + "rand 0.10.2", "serde", "socket2 0.6.4", "thiserror 2.0.18", @@ -7562,9 +7565,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -8360,9 +8363,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" dependencies = [ "memchr", "ucd-trie", @@ -8370,9 +8373,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" dependencies = [ "pest", "pest_generator", @@ -8380,9 +8383,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" dependencies = [ "pest", "pest_meta", @@ -8393,12 +8396,11 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" dependencies = [ "pest", - "sha2 0.10.9", ] [[package]] @@ -8983,7 +8985,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustls 0.23.35", "socket2 0.6.4", "thiserror 2.0.18", @@ -8994,17 +8996,18 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.0", + "rand 0.10.2", + "rand_pcg", "ring 0.17.14", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustls 0.23.35", "rustls-pki-types", "slab", @@ -9016,16 +9019,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9092,9 +9095,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -9183,6 +9186,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -9308,9 +9320,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ "bitflags 2.13.0", ] @@ -9558,7 +9570,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc05fbf560421a0357a750cbe78c7ca19d4923918490daabba313d5dbc871e47" dependencies = [ - "rand 0.10.1", + "rand 0.10.2", ] [[package]] @@ -9842,9 +9854,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -9998,9 +10010,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -10888,7 +10900,7 @@ dependencies = [ "data-encoding", "debugid", "if_chain", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "unicode-id-start", @@ -11321,7 +11333,7 @@ dependencies = [ "allocator-api2", "bumpalo", "hashbrown 0.14.5", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", ] [[package]] @@ -11350,7 +11362,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "siphasher 0.3.11", "swc_atoms", @@ -11399,7 +11411,7 @@ dependencies = [ "num-bigint", "once_cell", "phf 0.11.3", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "string_enum", "swc_atoms", @@ -11420,7 +11432,7 @@ dependencies = [ "num-bigint", "once_cell", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "ryu-js", "serde", "swc_allocator", @@ -11454,7 +11466,7 @@ dependencies = [ "either", "num-bigint", "phf 0.11.3", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "seq-macro", "serde", "smallvec", @@ -11474,7 +11486,7 @@ checksum = "c675d14700c92f12585049b22b02356f1e142f4b0c32a4d0eb4b7a968a4c0c1e" dependencies = [ "anyhow", "pathdiff", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11508,7 +11520,7 @@ dependencies = [ "once_cell", "par-core", "phf 0.11.3", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11551,7 +11563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39b3b34f6a28348416174912009d09994ab71c867682ec78d641a9feb3a96b4e" dependencies = [ "either", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11572,7 +11584,7 @@ dependencies = [ "bytes-str", "indexmap 2.14.0", "once_cell", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "sha1", "string_enum", @@ -11593,7 +11605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3872c006ccfdcc19f1cf5c01c15915a69964ba7982c9f581cdb7e727e77b9a2c" dependencies = [ "bytes-str", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11614,7 +11626,7 @@ dependencies = [ "num_cpus", "once_cell", "par-core", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "ryu-js", "swc_atoms", "swc_common", @@ -11672,7 +11684,7 @@ dependencies = [ "data-encoding", "debugid", "if_chain", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "unicode-id-start", @@ -11869,7 +11881,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "sketches-ddsketch", @@ -12188,9 +12200,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -12208,9 +12220,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -12922,9 +12934,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" dependencies = [ "serde", "stable_deref_trait", @@ -13321,9 +13333,9 @@ checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" [[package]] name = "utf8-width" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" +checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a" [[package]] name = "utf8_iter" @@ -13734,7 +13746,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-nats", @@ -13816,7 +13828,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.742.0" +version = "1.750.0" dependencies = [ "async-stream", "async-trait", @@ -13849,7 +13861,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13862,7 +13874,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "argon2", @@ -14000,7 +14012,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14023,7 +14035,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14038,7 +14050,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14064,7 +14076,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.742.0" +version = "1.750.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14074,7 +14086,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14091,7 +14103,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14113,7 +14125,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14136,7 +14148,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14152,7 +14164,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14173,7 +14185,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14194,7 +14206,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14220,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-nats", @@ -14243,7 +14255,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14268,7 +14280,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14286,7 +14298,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14308,7 +14320,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14328,7 +14340,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14365,7 +14377,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14393,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.742.0" +version = "1.750.0" dependencies = [ "lazy_static", "serde", @@ -14405,7 +14417,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.742.0" +version = "1.750.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14430,7 +14442,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14444,10 +14456,11 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.742.0" +version = "1.750.0" dependencies = [ "axum 0.8.9", "chrono", + "futures", "hex", "http 1.4.2", "hyper 1.10.1", @@ -14477,7 +14490,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.742.0" +version = "1.750.0" dependencies = [ "chrono", "lazy_static", @@ -14491,7 +14504,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14510,7 +14523,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.750.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14612,7 +14625,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.742.0" +version = "1.750.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14631,7 +14644,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.742.0" +version = "1.750.0" dependencies = [ "regex", "serde", @@ -14646,7 +14659,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14670,7 +14683,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "futures", @@ -14687,7 +14700,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.750.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14703,7 +14716,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -14724,7 +14737,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -14755,7 +14768,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "arc-swap", @@ -14780,7 +14793,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-stream", @@ -14814,7 +14827,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "futures", @@ -14832,7 +14845,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.750.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14841,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -14853,7 +14866,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde_json", @@ -14865,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "gosyn", @@ -14877,7 +14890,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -14889,7 +14902,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde_json", @@ -14901,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "nu-parser", @@ -14912,7 +14925,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14923,7 +14936,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14935,7 +14948,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14946,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-recursion", @@ -14968,7 +14981,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde_json", @@ -14980,7 +14993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -14994,7 +15007,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15011,7 +15024,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -15024,7 +15037,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde", @@ -15036,7 +15049,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -15054,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15070,7 +15083,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15086,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde", @@ -15097,7 +15110,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-recursion", @@ -15136,7 +15149,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "const_format", @@ -15175,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.742.0" +version = "1.750.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15186,7 +15199,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-recursion", @@ -15220,7 +15233,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15244,7 +15257,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15277,7 +15290,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15310,7 +15323,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15330,7 +15343,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15364,7 +15377,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15400,7 +15413,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15423,7 +15436,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15460,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-nats", @@ -15471,7 +15484,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15506,7 +15519,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15534,7 +15547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-trait", @@ -15559,7 +15572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15578,7 +15591,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-once-cell", @@ -15688,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.742.0" +version = "1.750.0" dependencies = [ "bytes", "futures", @@ -16506,9 +16519,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977347db8caa080403f6b6b7c1cda9479a8e869316f7e13a59b19076a40f94e3" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" [[package]] name = "zmij" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 45682215cf..b2b2ceff9c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.742.0" +version = "1.750.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.742.0" +version = "1.750.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -112,7 +112,7 @@ strip = "none" default = [] private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] -enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise"] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"] local_reports = ["windmill-common/local_reports"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] stripe = ["windmill-api/stripe"] diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ce3db8ebcb..087459205e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -95352c13c4c82247d8cfd80936f9203aeb079802 +0de2412ff0734b11e12ba378c9bcc373ff9ae800 diff --git a/backend/migrations/20260624161218_dev_workspace.down.sql b/backend/migrations/20260624161218_dev_workspace.down.sql new file mode 100644 index 0000000000..45b8e65f67 --- /dev/null +++ b/backend/migrations/20260624161218_dev_workspace.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE workspace DROP CONSTRAINT IF EXISTS workspace_dev_requires_parent; +DROP INDEX IF EXISTS workspace_canonical_dev_idx; +ALTER TABLE workspace DROP COLUMN is_dev_workspace; diff --git a/backend/migrations/20260624161218_dev_workspace.up.sql b/backend/migrations/20260624161218_dev_workspace.up.sql new file mode 100644 index 0000000000..a0a252eb7c --- /dev/null +++ b/backend/migrations/20260624161218_dev_workspace.up.sql @@ -0,0 +1,14 @@ +-- A dev workspace is a fork (parent_workspace_id set) that is the standing editable +-- environment paired with its parent ("prod"), as opposed to a throwaway fork. +ALTER TABLE workspace ADD COLUMN is_dev_workspace BOOLEAN NOT NULL DEFAULT false; + +-- At most one active canonical dev workspace per parent (one editable source per prod). +-- Excludes soft-deleted (archived) workspaces so a new dev can replace an archived one. +CREATE UNIQUE INDEX workspace_canonical_dev_idx ON workspace (parent_workspace_id) + WHERE is_dev_workspace AND deleted = false; + +-- A dev workspace is a fork, so it must have a parent. Enforce the invariant at the schema level so +-- no path (or manual write) can persist a "root dev workspace". No backfill needed: the column is +-- added above with default false, so no existing row can violate this at creation time. +ALTER TABLE workspace ADD CONSTRAINT workspace_dev_requires_parent + CHECK (NOT is_dev_workspace OR parent_workspace_id IS NOT NULL); diff --git a/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql new file mode 100644 index 0000000000..2278fbfb2e --- /dev/null +++ b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql @@ -0,0 +1,4 @@ +REVOKE ALL ON dispatch_event FROM windmill_user; +REVOKE ALL ON dispatch_event FROM windmill_admin; +REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_user; +REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_admin; diff --git a/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql new file mode 100644 index 0000000000..5d0b4650ce --- /dev/null +++ b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql @@ -0,0 +1,15 @@ +-- The dispatch_event table (migration 20260523055641_dispatch_event) was +-- created relying on ALTER DEFAULT PRIVILEGES to grant access to windmill_user +-- and windmill_admin. Those default privileges only apply to objects created by +-- the role that set them (migration 20250205131523), so deployments whose +-- migration runner is a different role leave dispatch_event ungranted. Direct +-- writes run as the invoking role -- the dispatcher insert (asset_dispatch.rs) +-- and the DELETE in delete_jobs (windmill-common/src/jobs.rs), reached whenever +-- a job's side rows are reaped, e.g. on schedule disable -- and fail with +-- "permission denied for table dispatch_event". Grant explicitly to guarantee +-- access regardless of who ran the migrations (same fix as notify_event in +-- 20260619091631 and script_trigger in 20260619112847). +GRANT ALL ON dispatch_event TO windmill_user; +GRANT ALL ON dispatch_event TO windmill_admin; +GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_admin; diff --git a/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql new file mode 100644 index 0000000000..b73484e053 --- /dev/null +++ b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql @@ -0,0 +1,6 @@ +REVOKE ALL ON workspace_diff FROM windmill_user; +REVOKE ALL ON workspace_diff FROM windmill_admin; +REVOKE ALL ON materialized_partition FROM windmill_user; +REVOKE ALL ON materialized_partition FROM windmill_admin; +REVOKE ALL ON debounce_stale_data FROM windmill_user; +REVOKE ALL ON debounce_stale_data FROM windmill_admin; diff --git a/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql new file mode 100644 index 0000000000..9205934600 --- /dev/null +++ b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql @@ -0,0 +1,22 @@ +-- Same grant gap fixed for notify_event (20260619091631), script_trigger +-- (20260619112847), and dispatch_event: tables created after the one-time +-- GRANT ALL in 20250205131523 rely on ALTER DEFAULT PRIVILEGES, which only +-- applies to objects created by the role that set them. On deployments whose +-- migration runner is a different role, these tables end up ungranted, and +-- writes that run under the RLS role (a transaction opened via +-- user_db.begin(&authed) -> SET LOCAL ROLE windmill_user/windmill_admin) fail +-- with "permission denied for table ". +-- +-- Each table below has a confirmed write on a user_db transaction: +-- * workspace_diff -- UPDATE in set_ws_specific (workspaces.rs) +-- * materialized_partition -- INSERT via record_materialization (assets API) +-- * debounce_stale_data -- DELETE in resume_suspended_trigger_jobs +-- (global_handler.rs), the same tx that reaps a +-- job's side rows +-- None has a sequence, so only table grants are needed. +GRANT ALL ON workspace_diff TO windmill_user; +GRANT ALL ON workspace_diff TO windmill_admin; +GRANT ALL ON materialized_partition TO windmill_user; +GRANT ALL ON materialized_partition TO windmill_admin; +GRANT ALL ON debounce_stale_data TO windmill_user; +GRANT ALL ON debounce_stale_data TO windmill_admin; diff --git a/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.down.sql b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.down.sql new file mode 100644 index 0000000000..2e3e434bcc --- /dev/null +++ b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.down.sql @@ -0,0 +1,3 @@ +-- Irreversible data backfill: once realigned to the derived username, a favorite +-- is indistinguishable from one legitimately created under that username, so there +-- is nothing safe to revert. No-op. diff --git a/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.up.sql b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.up.sql new file mode 100644 index 0000000000..b06f63d7af --- /dev/null +++ b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.up.sql @@ -0,0 +1,19 @@ +-- A superadmin acting in a workspace they are not a member of used to be +-- identified by their raw email (so favorites were stored with usr = email). +-- They are now identified by their instance-derived username (password.username), +-- so realign those pre-existing favorites to keep them visible. Only email-keyed +-- rows are ever a superadmin's (members always store a non-email username), and +-- the anti-join skips rows that would collide with an already-derived favorite. +UPDATE favorite f +SET usr = p.username +FROM password p +WHERE f.usr = p.email + AND p.super_admin = true + AND p.username IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM favorite f2 + WHERE f2.workspace_id = f.workspace_id + AND f2.usr = p.username + AND f2.favorite_kind = f.favorite_kind + AND f2.path = f.path + ); diff --git a/backend/migrations/20260702064737_workspace_storage_usage.down.sql b/backend/migrations/20260702064737_workspace_storage_usage.down.sql new file mode 100644 index 0000000000..e32480a602 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_storage_usage; diff --git a/backend/migrations/20260702064737_workspace_storage_usage.up.sql b/backend/migrations/20260702064737_workspace_storage_usage.up.sql new file mode 100644 index 0000000000..d7274e8742 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.up.sql @@ -0,0 +1,17 @@ +-- Cached per-(workspace, storage) byte usage of workspace object storage, +-- refreshed by listing the storage location and adjusted optimistically as +-- uploads complete. Read on every workspace-storage write in CE builds to +-- enforce the storage quota, and by the storage_usage endpoint in all builds. +CREATE TABLE workspace_storage_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + storage VARCHAR(255) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, storage) +); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role +-- that set them (same gap as workspace_diff, notify_event, script_trigger). +GRANT ALL ON workspace_storage_usage TO windmill_user; +GRANT ALL ON workspace_storage_usage TO windmill_admin; diff --git a/backend/migrations/20260702095830_duckdb_macro_registry.down.sql b/backend/migrations/20260702095830_duckdb_macro_registry.down.sql new file mode 100644 index 0000000000..6f48e71d0f --- /dev/null +++ b/backend/migrations/20260702095830_duckdb_macro_registry.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS macro_usage; +DROP TABLE IF EXISTS macro_definition; diff --git a/backend/migrations/20260702095830_duckdb_macro_registry.up.sql b/backend/migrations/20260702095830_duckdb_macro_registry.up.sql new file mode 100644 index 0000000000..ccc4bcc8cc --- /dev/null +++ b/backend/migrations/20260702095830_duckdb_macro_registry.up.sql @@ -0,0 +1,31 @@ +-- Workspace DuckDB macro registry: one row per macro defined by a deployed +-- `// macros` library script. Names are workspace-unique (macros are injected +-- unqualified into consumer jobs). +CREATE TABLE macro_definition ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + provider_path VARCHAR(510) NOT NULL, + params TEXT NOT NULL, + body TEXT NOT NULL, + is_table_macro BOOLEAN NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, name) +); +CREATE INDEX idx_macro_definition_provider ON macro_definition (workspace_id, provider_path); + +-- Deploy-recorded consumer→macro edges, for the asset graph only (the worker +-- re-detects calls live at job time). +CREATE TABLE macro_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + consumer_path VARCHAR(510) NOT NULL, + macro_name VARCHAR(255) NOT NULL, + PRIMARY KEY (workspace_id, consumer_path, macro_name) +); +CREATE INDEX idx_macro_usage_name ON macro_usage (workspace_id, macro_name); + +-- Both tables are written on user_db transactions (SET LOCAL ROLE); the +-- one-time GRANT ALL migration predates them, so explicit grants are required. +GRANT ALL ON macro_definition TO windmill_user; +GRANT ALL ON macro_definition TO windmill_admin; +GRANT ALL ON macro_usage TO windmill_user; +GRANT ALL ON macro_usage TO windmill_admin; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql new file mode 100644 index 0000000000..e72edd0b12 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_multipart_inflight; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql new file mode 100644 index 0000000000..9768eaf842 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql @@ -0,0 +1,32 @@ +-- Reservation for the parts of in-flight (initiated but not yet completed) +-- multipart uploads to workspace object storage. Uncommitted parts occupy +-- object-store capacity but are invisible to the list-based storage recount +-- until completion, so CE folds this reservation into the remaining quota to +-- bound abandoned uploads. One row per uploaded part so a re-uploaded part +-- (same part_id) replaces rather than double-counts; a part is recorded only +-- after its upstream upload succeeds. Rows are removed on successful complete +-- and lazily expired after a TTL (abort/abandon rely on the TTL, which matches +-- when the object store reaps the uncommitted parts). +-- part_id - S3 part number or Azure block id (string) +-- part_bytes - size of that part +-- target_existing_size - size of the object the upload will overwrite (0 if new), +-- credited so an overwrite only reserves the net growth +CREATE TABLE workspace_multipart_inflight ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + upload_id VARCHAR(512) NOT NULL, + part_id VARCHAR(256) NOT NULL, + storage VARCHAR(255) NOT NULL, + part_bytes BIGINT NOT NULL DEFAULT 0, + target_existing_size BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, upload_id, part_id) +); + +CREATE INDEX idx_workspace_multipart_inflight_created_at + ON workspace_multipart_inflight (created_at); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role that +-- set them (same gap as workspace_storage_usage, notify_event, script_trigger). +GRANT ALL ON workspace_multipart_inflight TO windmill_user; +GRANT ALL ON workspace_multipart_inflight TO windmill_admin; diff --git a/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql b/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql new file mode 100644 index 0000000000..aed2f9d32a --- /dev/null +++ b/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql @@ -0,0 +1,5 @@ +-- Postgres has no ALTER TYPE ... DROP VALUE for enums. The 'freshness' value +-- stays even on rollback, consistent with prior job_trigger_kind additions +-- (see 20260510174213_asset_trigger_dispatch). +DROP INDEX IF EXISTS idx_script_pipeline_freshness_scan; +DROP TABLE IF EXISTS pipeline_freshness_state; diff --git a/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql b/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql new file mode 100644 index 0000000000..acf9545c96 --- /dev/null +++ b/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql @@ -0,0 +1,35 @@ +-- Attribution for runs pushed by the pipeline freshness watchdog (the EE +-- background loop that re-runs a `// freshness`-annotated producer whose +-- output aged past its window). +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'freshness'; + +-- Per-(workspace, script) watchdog state: exponential-backoff bookkeeping so +-- a persistently failing producer isn't re-pushed on every scan tick, and an +-- atomic claim so concurrent servers can't double-push in the same tick +-- (claim = the UPDATE/INSERT that advances next_attempt_at; only the winner +-- pushes). Rows exist only while a script is stale — observing it fresh (or +-- its annotation gone) deletes the row, resetting the backoff. +CREATE TABLE pipeline_freshness_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + script_path VARCHAR(510) NOT NULL, + attempts INTEGER NOT NULL DEFAULT 1, + last_push_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path) +); + +-- Written only by the server monitor loop on the raw (non-RLS) pool, but +-- granted like every other app table so a future user-transaction reader +-- doesn't hit the recurring missing-GRANT class of bug. +GRANT ALL ON pipeline_freshness_state TO windmill_user; +GRANT ALL ON pipeline_freshness_state TO windmill_admin; + +-- The watchdog's ~60s candidate scan (latest deployed pipeline members) +-- filters on this exact predicate and orders by (workspace_id, path, +-- created_at DESC); without a matching partial index it seq-scans the whole +-- script-version heap on every tick, on instances that mostly have zero +-- pipeline scripts. (idx_script_pipeline_path is text_pattern_ops for +-- prefix LIKE — it can't serve this ordering.) +CREATE INDEX idx_script_pipeline_freshness_scan + ON script (workspace_id, path, created_at DESC) + WHERE auto_kind = 'pipeline' AND archived = false AND deleted = false; diff --git a/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql b/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql new file mode 100644 index 0000000000..fd722bf414 --- /dev/null +++ b/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS fork_ducklake_namespace; diff --git a/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql b/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql new file mode 100644 index 0000000000..7762304199 --- /dev/null +++ b/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql @@ -0,0 +1,51 @@ +-- Registry of ducklake namespaces provisioned for fork/dev workspaces. One row per +-- (fork workspace, lake name): records the exact catalog metadata schema and data +-- sub-path the fork's jobs attach to, so fork deletion can drop the pg schema and +-- delete the S3 prefix deterministically (the row is written on first resolution, +-- before any physical state exists). +-- +-- Deliberately NO foreign key to workspace(id): rows are the durable cleanup ledger and +-- must OUTLIVE the workspace row when physical cleanup fails after the delete commits +-- (unreachable catalog, storage outage) — a CASCADE would erase the only record of the +-- orphaned namespace, letting a recreated same-id fork silently reattach stale tables. +-- Rows are deleted explicitly after each successful cleanup; leftover rows for a reused +-- id are retried at fork creation, which refuses to proceed while they cannot be cleaned. +CREATE TABLE fork_ducklake_namespace ( + workspace_id VARCHAR(50) NOT NULL, + ducklake_name VARCHAR(255) NOT NULL, + metadata_schema VARCHAR(63) NOT NULL, + -- Canonical identity of the catalog database the metadata schema lives in + -- (`:`, e.g. `instance:wm_ducklake` or + -- `postgres:u/admin/pg`). Cleanup connects to THIS catalog, not whatever the fork's + -- settings point at by then — a drifted catalog resource must not make cleanup drop a + -- schema in the wrong database and orphan the real one. + catalog TEXT NOT NULL, + -- Named workspace storage holding the fork's data files; '' = the default storage + -- (part of the PK, which cannot hold NULL). + storage TEXT NOT NULL DEFAULT '', + -- The storage's RESOLVED identity at registration time (`:`, e.g. `s3:u/admin/minio` or `filesystem:/data/lfs`; '' = unknown). Cleanup + -- deletes the fork prefix from THIS storage, not whatever the logical name points at by + -- then — repointing a storage after attach must not orphan the original fork data (or + -- delete a colliding prefix from the new one). + storage_ref TEXT NOT NULL DEFAULT '', + -- The fork namespace's data path within that storage (a bucket-root + -- `__wm_forks//…` prefix). + data_path TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Cleanup phase state: true once the metadata schema has been dropped but data files (or + -- the row delete) still failed. Later retries then skip the schema phase entirely — they + -- need NO catalog credentials, which may be gone for good with the deleted fork's + -- resources. Reset to false whenever a live fork re-registers the row (re-attaching + -- recreates the schema). + schema_dropped BOOLEAN NOT NULL DEFAULT false, + -- One row per physical location EVER attached: if the fork's lake settings drift + -- (catalog/storage/path change), later attaches add rows rather than replace them, so + -- cleanup covers every location the fork wrote, not just the first. + PRIMARY KEY (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path) +); + +-- Resolution runs under user_db transactions (SET LOCAL ROLE) in API contexts, so the +-- windmill roles need explicit grants (default privileges don't apply to app-created tables). +GRANT ALL ON fork_ducklake_namespace TO windmill_user; +GRANT ALL ON fork_ducklake_namespace TO windmill_admin; diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index e8a0a1cc40..8d2e87a53e 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -262,6 +262,14 @@ impl AssetsFinder { }; match arg_val { + // S3 helpers take an `S3Object` (constructor or dict literal) or an + // `s3://bucket/key` string. Other helpers take a bare resource-path + // string literal. + Some(arg) if matches!(kind, AssetKind::S3Object) => { + let path = s3_object_arg_path(arg).ok_or(())?; + self.assets + .push(ParseAssetsResult { kind, path, access_type, columns: None }); + } Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { let path = parse_asset_syntax(&value, false) .map(|(_, p)| p) @@ -282,6 +290,81 @@ impl AssetsFinder { // Positional arguments in python can also be used by their name struct Arg(usize, &'static str); +/// Extract a string-literal keyword argument, e.g. `s3="value"` in a call. +fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option { + keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .and_then(|kw| match &kw.value { + Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()), + _ => None, + }) +} + +/// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`. +fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option { + dict.keys + .iter() + .zip(dict.values.iter()) + .find_map(|(key, value)| match (key.as_ref()?, value) { + ( + Expr::Constant(ExprConstant { value: Constant::Str(k), .. }), + Expr::Constant(ExprConstant { value: Constant::Str(v), .. }), + ) if k.as_str() == name => Some(v.clone()), + _ => None, + }) +} + +/// Resolve the SDK `S3Object` argument of `load_s3_file`/`load_s3_file_reader`/ +/// `write_s3_file` to a canonical asset path, mirroring `windmill-parser-ts-asset`: +/// `S3Object(s3="", storage=""?)` — or the equivalent dict literal — +/// maps to the URI `s3:///` (empty bucket for default storage, i.e. +/// `s3:///`), and a `"s3://bucket/key"` URI string is passed through. +/// String args mirror the runtime `parse_s3_object` contract exactly: only a +/// `s3:///` URI with a non-empty key is valid — any other +/// string (bare key, `s3://x`, empty key) raises at run time, so recording an +/// edge for it would be a phantom node for a call that can only error. +/// The resulting URI is fed through `parse_asset_syntax` so the stored path +/// matches the TS object form and the `# on s3:///…` trigger form exactly. +fn s3_object_arg_path(expr: &Expr) -> Option { + let uri = match expr { + Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + match s + .strip_prefix("s3://") + .and_then(|rest| rest.split_once('/')) + { + Some((_, key)) if !key.is_empty() => s.clone(), + _ => return None, + } + } + Expr::Call(call) => { + // `S3Object(...)` imported directly or as `wmill.S3Object(...)` + let func_name = call + .func + .as_name_expr() + .map(|n| n.id.as_str()) + .or_else(|| call.func.as_attribute_expr().map(|a| a.attr.as_str()))?; + if func_name != "S3Object" { + return None; + } + let key = keyword_str_value(&call.keywords, "s3")?; + let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default(); + format!("s3://{storage}/{key}") + } + Expr::Dict(dict) => { + let key = dict_str_value(dict, "s3")?; + let storage = dict_str_value(dict, "storage").unwrap_or_default(); + format!("s3://{storage}/{key}") + } + _ => return None, + }; + Some( + parse_asset_syntax(&uri, false) + .map(|(_, p)| p.to_string()) + .unwrap_or(uri), + ) +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -300,13 +383,227 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "/test.csv".to_string(), + path: "test.csv".to_string(), access_type: Some(R), columns: None, },]) ); } + #[test] + fn test_py_asset_parser_bare_string_no_asset() { + // A plain (non-`s3://`) string is rejected by the runtime + // `parse_s3_object`, so the parser must not record a phantom asset + // for a call that can only error. + let input = r#" +import wmill +def main(): + wmill.write_s3_file("pipelines/etl/out.jsonl", b"") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![])); + } + + #[test] + fn test_py_asset_parser_invalid_uri_no_write_edge() { + // `s3://x` (no key part) and `s3://bucket/` (empty key) are rejected + // by the runtime `parse_s3_object` — same rule for the SDK-arg path: + // no R/W edge. The generic URI-literal scan may still record them as + // ambiguous (`access_type: None`) assets, like any `s3://…` string + // constant anywhere in a script. + let input = r#" +import wmill +def main(): + wmill.write_s3_file("s3://broken", b"") + wmill.write_s3_file("s3://bucket/", b"") +"#; + let assets = parse_assets(input).expect("parse").assets; + assert!( + assets.iter().all(|a| a.access_type.is_none()), + "invalid URIs must not produce R/W edges: {assets:?}" + ); + } + + #[test] + fn test_py_asset_parser_write_s3_object_constructor() { + // The SDK signature is `write_s3_file(s3object: S3Object | str, ...)` and + // its docstring recommends the constructor form with a bare key. It must + // resolve to the same canonical path as the TS object form and a + // `# on s3:///` trigger. + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.write_s3_file(S3Object(s3="analytics/x.csv"), b"content") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "analytics/x.csv".to_string(), + access_type: Some(W), + columns: None, + },]) + ); + } + + #[test] + fn test_py_write_key_matches_duckdb_read_key() { + // Cross-language lineage: this write records `exports/x`, the same path a + // DuckDB `read_csv('s3://exports/x')` resolves to (see + // windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`), + // so the producer and consumer connect in the pipeline graph. + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.write_s3_file(S3Object(s3="exports/x"), b"content") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "exports/x".to_string(), + access_type: Some(W), + columns: None, + },]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_with_storage() { + // `S3Object(s3=..., storage=...)` maps to `s3:///`, + // matching the `s3://bucket/key` string form and the TS object form. + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.load_s3_file(S3Object(s3="dir/in.csv", storage="mybucket")) +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "mybucket/dir/in.csv".to_string(), + access_type: Some(R), + columns: None, + },]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_reader_and_keyword_arg() { + // load_s3_file_reader + passing the S3Object via the `s3object` keyword + // and via the `wmill.S3Object` attribute form. + let input = r#" +import wmill +def main(): + wmill.load_s3_file_reader(s3object=wmill.S3Object(s3="dir/in.csv")) +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "dir/in.csv".to_string(), + access_type: Some(R), + columns: None, + },]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_dict_literal() { + // `S3Object` subclasses dict, so the SDK also accepts a plain dict + // literal — same resolution as the constructor form. + let input = r#" +import wmill +def main(): + wmill.write_s3_file({"s3": "out.json"}, b"{}") + wmill.load_s3_file({"s3": "dir/in.csv", "storage": "mybucket"}) +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "mybucket/dir/in.csv".to_string(), + access_type: Some(R), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "out.json".to_string(), + access_type: Some(W), + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_multiple_s3_object_writes() { + // Several direct constructor-form writes in main() — all outputs must + // be detected (merge_assets returns a deterministic path-sorted order). + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.write_s3_file(S3Object(s3="pipelines/km_real/raw_events.json"), b"[]") + wmill.write_s3_file(S3Object(s3="pipelines/km_real/enriched.json"), b"[]") + wmill.write_s3_file(S3Object(s3="pipelines/km_real/summary.json"), b"[]") + wmill.write_s3_file(S3Object(s3="pipelines/km_real/report.json"), b"{}") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "pipelines/km_real/enriched.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "pipelines/km_real/raw_events.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "pipelines/km_real/report.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "pipelines/km_real/summary.json".to_string(), + access_type: Some(W), + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_dynamic_key_no_false_positive() { + // A computed key can't be resolved statically — must yield nothing + // rather than a bogus path. + let input = r#" +import wmill +from wmill import S3Object +def main(name: str): + wmill.write_s3_file(S3Object(s3=f"pipelines/{name}.json"), b"{}") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![])); + } + #[test] fn test_py_asset_parser_unused_sql() { let input = r#" diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index f7c578a274..d2ab5b51a7 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -83,18 +83,56 @@ pub fn parse_assets(input: &str) -> anyhow::Result { // Body-inferred column lineage, with `// column` annotations taking // precedence per output column (explicit declaration overrides inference). pipeline.column_lineage = merge_column_lineage(inferred, pipeline.column_lineage); - Ok(ParseAssetsOutput::new( - merge_assets(collector.assets), - Vec::new(), - pipeline, - )) + // A bare string literal in query position is only weak read evidence: a + // summary `SELECT 's3:///out.csv' AS target` after `COPY … TO + // 's3:///out.csv'` must not turn the write into rw (which draws a + // spurious read edge and an asset⇄script cycle in the pipeline graph). + // Surface weak reads only for assets with no other recorded usage, so a + // path that is *merely* mentioned still shows up linked to the script. + let mut assets = merge_assets(collector.assets); + for weak in merge_assets(collector.weak_string_reads) { + if !assets + .iter() + .any(|a| a.kind == weak.kind && a.path == weak.path) + { + assets.push(weak); + } + } + assets.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(ParseAssetsOutput::new(assets, Vec::new(), pipeline)) +} + +/// Provenance of the innermost access context. The access type alone can't +/// tell a definitive read apart from a mere mention: a bare string literal in +/// generic query position (`QueryRead`) is only *weak* read evidence — e.g. a +/// summary `SELECT 's3:///out.csv' AS target` echoing a path — while the same +/// literal as a read-function argument or a `COPY` target is definitive. +#[derive(Clone, Copy, PartialEq, Eq)] +enum AccessCtx { + QueryRead, + ReadFn, + CopyWrite, +} + +impl AccessCtx { + fn access_type(self) -> AssetUsageAccessType { + match self { + AccessCtx::QueryRead | AccessCtx::ReadFn => R, + AccessCtx::CopyWrite => W, + } + } } /// Visitor that collects S3 asset literals from SQL statements struct AssetCollector { assets: Vec, - // e.g set to Read when we are inside a SELECT ... FROM ... statement - current_access_type_stack: Vec, + // Bare string literals seen in generic query position — weak read + // evidence, surfaced by `parse_assets` only when the script has no other + // recorded usage of the same asset (a real write must not gain a spurious + // read edge from a mention). + weak_string_reads: Vec, + // e.g set to QueryRead when we are inside a SELECT ... FROM ... statement + current_access_type_stack: Vec, // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") } var_identifiers: BTreeMap, // e.g USE dl; @@ -119,6 +157,7 @@ impl AssetCollector { fn new() -> Self { Self { assets: Vec::new(), + weak_string_reads: Vec::new(), current_access_type_stack: Vec::with_capacity(8), var_identifiers: BTreeMap::new(), currently_used_asset: None, @@ -162,7 +201,11 @@ impl AssetCollector { name: &ObjectName, access_type: Option, ) -> Option { - let access_type = access_type.or_else(|| self.current_access_type_stack.last().copied()); + let access_type = access_type.or_else(|| { + self.current_access_type_stack + .last() + .map(|c| c.access_type()) + }); if let Some((kind, path)) = &self.currently_used_asset { // We don't want to infer that any simple identifier refers to an asset if // we are not in a known R/W context @@ -301,12 +344,18 @@ impl AssetCollector { // Check if the string matches our asset syntax patterns if let Some((kind, path)) = parse_asset_syntax(s, false) { if kind == AssetKind::S3Object { - self.assets.push(ParseAssetsResult { + let ctx = self.current_access_type_stack.last().copied(); + let result = ParseAssetsResult { kind, path: path.to_string(), - access_type: self.current_access_type_stack.last().copied(), + access_type: ctx.map(AccessCtx::access_type), columns: None, - }); + }; + if ctx == Some(AccessCtx::QueryRead) { + self.weak_string_reads.push(result); + } else { + self.assets.push(result); + } } } } @@ -314,7 +363,7 @@ impl AssetCollector { fn handle_obj_name_pre(&mut self, name: &ObjectName) { if let Some(fname) = get_trivial_obj_name(name) { if is_read_fn(fname) { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::ReadFn); } } if let Some(str_lit) = get_str_lit_from_obj_name(name) { @@ -691,9 +740,20 @@ impl Visitor for AssetCollector { match table_factor { TableFactor::Table { name, args, .. } => { if args.is_none() { - // Avoid Table Functions - self.handle_obj_name_pre(name); + // FROM 's3:///…' is a definitive read — record it directly + // so it isn't demoted to a weak in-query mention. + if let Some(asset) = self.get_s3_asset_from_str_literal_table(table_factor) { + self.assets.push(asset); + } } + // For a read-function table factor this pushes ReadFn, making + // every literal inside its arguments a definitive read — the + // direct form (read_csv('s3:///…')) but also list and named + // arguments (read_parquet(['s3:///…'])). Must run for BOTH the + // plain-table and table-function branches: post_visit_table_factor + // pops via handle_obj_name_post unconditionally, so skipping the + // push here would unbalance the stack. + self.handle_obj_name_pre(name); } _ => {} } @@ -719,6 +779,13 @@ impl Visitor for AssetCollector { Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => { self.handle_string_literal(s); } + // Read-function call in expression position: its argument literals + // are definitive reads. Balances the pop in `post_visit_expr`. + Expr::Function(func) => { + if get_trivial_obj_name(&func.name).is_some_and(is_read_fn) { + self.current_access_type_stack.push(AccessCtx::ReadFn); + } + } _ => {} } std::ops::ControlFlow::Continue(()) @@ -946,7 +1013,7 @@ impl Visitor for AssetCollector { } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { - self.current_access_type_stack.push(W); + self.current_access_type_stack.push(AccessCtx::CopyWrite); self.handle_string_literal(filename); self.current_access_type_stack.pop(); } @@ -1013,7 +1080,7 @@ impl Visitor for AssetCollector { &mut self, query: &sqlparser::ast::Query, ) -> std::ops::ControlFlow { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::QueryRead); self.cte_name_stack.push(collect_cte_names(query)); std::ops::ControlFlow::Continue(()) } @@ -1078,13 +1145,13 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "/a.parquet".to_string(), + path: "a.parquet".to_string(), access_type: Some(R), columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "/c.parquet".to_string(), + path: "c.parquet".to_string(), access_type: Some(W), columns: None }, @@ -1098,6 +1165,151 @@ mod tests { ); } + #[test] + fn test_duckdb_read_key_matches_sdk_write_key() { + // Cross-language lineage: a TS `writeS3File({ s3: "exports/x" })` or + // Python `write_s3_file(S3Object(s3="exports/x"))` records the asset path + // `exports/x` (default storage). A DuckDB reader of the same object must + // resolve to the identical path so the graph connects the producer and + // consumer — both the bare `s3://exports/x` and the triple-slash + // `s3:///exports/x` default-storage form must yield `exports/x`. + for uri in ["s3://exports/x", "s3:///exports/x"] { + let input = format!("SELECT * FROM read_csv('{uri}');"); + let assets = parse_assets(&input).expect("parse").assets; + assert_eq!( + assets, + vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "exports/x".to_string(), + access_type: Some(R), + columns: None + }], + "DuckDB read of {uri} must resolve to the SDK write key" + ); + } + } + + #[test] + fn test_copy_target_echoed_in_select_stays_write_only() { + // The trailing summary SELECT merely mentions the COPY target — it + // must not add a read (rw would draw an asset⇄script cycle). + let input = r#" + COPY (SELECT 1 AS x) TO 's3:///out.csv'; + SELECT 's3:///out.csv' AS target, 42 AS rows_written; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "out.csv".to_string(), + access_type: Some(W), + columns: None + }]) + ); + } + + #[test] + fn test_bare_string_mention_without_other_usage_is_a_read() { + let input = r#" + SELECT 's3:///referenced.csv' AS path; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "referenced.csv".to_string(), + access_type: Some(R), + columns: None + }]) + ); + } + + #[test] + fn test_self_refresh_read_fn_plus_copy_stays_rw() { + // A definitive read (read_csv) of the same file the script rewrites + // is a real rw — only *bare-literal* mentions are demoted. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_csv('s3:///data.csv'); + COPY (SELECT * FROM tmp) TO 's3:///data.csv'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "data.csv".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_plus_copy_stays_rw() { + // read_parquet's list form is as definitive as the direct literal — + // it must not be demoted to a weak mention when the file is rewritten. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_parquet(['s3:///data.parquet']); + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_multiple_files_are_reads() { + let input = r#" + SELECT * FROM read_parquet(['s3:///a.parquet', 's3:///b.parquet']); + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "a.parquet".to_string(), + access_type: Some(R), + columns: None + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "b.parquet".to_string(), + access_type: Some(R), + columns: None + } + ]) + ); + } + + #[test] + fn test_from_string_literal_of_written_file_stays_rw() { + // FROM-position string literal is likewise a definitive read. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM 's3:///data.parquet'; + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + #[test] fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() { let input = r#" @@ -1773,7 +1985,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "/example_file.parquet"); + assert_eq!(result[0].path, "example_file.parquet"); assert_eq!(result[0].access_type, Some(R)); let columns = result[0].columns.as_ref().expect("Should have columns"); @@ -1804,7 +2016,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "/example_file.parquet"); + assert_eq!(result[0].path, "example_file.parquet"); assert!(result[0].columns.is_none()); } @@ -1817,7 +2029,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "/example_file.parquet"); + assert_eq!(result[0].path, "example_file.parquet"); let columns = result[0].columns.as_ref().expect("Should have columns"); assert_eq!(columns.get("col1"), Some(&R)); @@ -1836,7 +2048,7 @@ mod tests { assert_eq!(result.len(), 2); assert!(result.iter().any(|a| { - a.path == "/file1.parquet" + a.path == "file1.parquet" && a.columns.as_ref().map_or(false, |c| c.contains_key("col1")) })); assert!(result.iter().any(|a| { @@ -1869,7 +2081,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "/test.parquet"); + assert_eq!(result[0].path, "test.parquet"); assert_eq!(result[0].access_type, Some(R)); let columns = result[0].columns.as_ref().expect("Should have columns"); diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 931f470d3c..dc094a0a7b 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -826,6 +826,29 @@ fn parse_duckdb_file(code: &str) -> anyhow::Result>> { } args.append(&mut parse_sql_sanitized_interpolation(code)); + + // A `// partitioned` script receives its resolved partition as a job arg + // named `partition` (windmill_common::partition::PARTITION_ARG), and duckdb + // binds named parameters only when they appear in the parsed signature — + // so auto-declare it (as `-- $partition (text)` would) to make `$partition` + // usable without a manual declaration. An explicit declaration wins. + // `has_default` keeps the field optional: the platform resolves the value + // at run start when it is not passed explicitly. + if !args.iter().any(|arg| arg.name == "partition") + && windmill_parser::asset_parser::parse_pipeline_annotations(code) + .partition + .is_some() + { + args.push(Arg { + name: "partition".to_string(), + typ: Typ::Str(None), + default: None, + otyp: Some("text".to_string()), + has_default: true, + oidx: None, + otyp_inferred: false, + }); + } Ok(Some(args)) } @@ -1985,4 +2008,63 @@ SELECT x Ok(()) } + + #[test] + fn test_parse_duckdb_partitioned_auto_declares_partition() -> anyhow::Result<()> { + let code = r#"// partitioned daily +// materialize ducklake://main/sales_daily +SELECT $partition AS day, count(*) AS n FROM sales WHERE day = $partition +"#; + let args = parse_duckdb_sig(code)?.args; + assert_eq!( + args, + vec![Arg { + otyp: Some("text".to_string()), + name: "partition".to_string(), + typ: Typ::Str(None), + default: None, + has_default: true, + oidx: None, + otyp_inferred: false, + }] + ); + + // `--`-style annotation headers auto-declare too. + let dash_code = "-- partitioned hourly\nSELECT $partition AS h\n"; + assert_eq!(parse_duckdb_sig(dash_code)?.args, args); + + Ok(()) + } + + #[test] + fn test_parse_duckdb_partitioned_explicit_declaration_wins() -> anyhow::Result<()> { + let code = r#"// partitioned daily +-- $partition (text) +-- $limit (int) = 10 +SELECT * FROM sales WHERE day = $partition LIMIT $limit +"#; + let args = parse_duckdb_sig(code)?.args; + // No duplicate: the explicit (required) declaration is kept as-is. + assert_eq!(args.iter().filter(|a| a.name == "partition").count(), 1); + assert_eq!( + args[0], + Arg { + otyp: Some("text".to_string()), + name: "partition".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + } + ); + Ok(()) + } + + #[test] + fn test_parse_duckdb_unpartitioned_does_not_declare_partition() -> anyhow::Result<()> { + let code = "SELECT 1 AS partition_count\n"; + assert_eq!(parse_duckdb_sig(code)?.args, vec![]); + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-ts-asset/src/lib.rs b/backend/parsers/windmill-parser-ts-asset/src/lib.rs index b7fe0fea5c..f273db5cbe 100644 --- a/backend/parsers/windmill-parser-ts-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-ts-asset/src/lib.rs @@ -341,12 +341,25 @@ fn object_str_prop(obj: &ObjectLit, name: &str) -> Option { /// `writeS3File` to a canonical asset path, mirroring the runtime /// `parseS3Object`: an object `{ s3: "", storage?: "" }` maps to /// the URI `s3:///` (empty bucket for default storage, i.e. -/// `s3:///`), and a bare `"s3://bucket/key"` string is passed through. +/// `s3:///`), and a `"s3://bucket/key"` URI string is passed through. +/// String args mirror the runtime `parseS3Object` contract exactly: only a +/// `s3:///` URI with a non-empty key is valid — any other +/// string (bare key, `s3://x`, empty key) throws at run time, so recording an +/// edge for it would be a phantom node for a call that can only error. /// The resulting URI is fed through `parse_asset_syntax` so the stored path /// matches the `// on s3:///…` trigger form exactly. fn s3_object_arg_path(arg: &Expr) -> Option { let uri = match arg { - Expr::Lit(Lit::Str(s)) => s.value.to_string(), + Expr::Lit(Lit::Str(s)) => { + let v = s.value.to_string(); + match v + .strip_prefix("s3://") + .and_then(|rest| rest.split_once('/')) + { + Some((_, key)) if !key.is_empty() => v, + _ => return None, + } + } Expr::Object(obj) => { let key = object_str_prop(obj, "s3")?; let storage = object_str_prop(obj, "storage").unwrap_or_default(); @@ -420,7 +433,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "/test.csv".to_string(), + path: "test.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -448,7 +461,31 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "/pipelines/km_real/raw_events.json".to_string(), + path: "pipelines/km_real/raw_events.json".to_string(), + access_type: Some(W), + columns: None, + },]) + ); + } + + #[test] + fn test_ts_write_key_matches_duckdb_read_key() { + // Cross-language lineage: this write records `exports/x`, the same path a + // DuckDB `read_csv('s3://exports/x')` resolves to (see + // windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`), + // so the producer and consumer connect in the pipeline graph. + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + await wmill.writeS3File({ s3: "exports/x" }, "[]") + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map(|r| r.assets).map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "exports/x".to_string(), access_type: Some(W), columns: None, },]) @@ -477,6 +514,42 @@ mod tests { ); } + #[test] + fn test_ts_asset_parser_bare_string_no_asset() { + // A plain (non-`s3://`) string is rejected by the runtime + // `parseS3Object`, so the parser must not record a phantom asset for + // a call that can only error. + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + await wmill.writeS3File("pipelines/etl/out.jsonl", "[]") + } + "#; + let s = parse_assets(input); + assert_eq!(s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![])); + } + + #[test] + fn test_ts_asset_parser_invalid_uri_no_write_edge() { + // `s3://x` (no key part) and `s3://bucket/` (empty key) are rejected + // by the runtime `parseS3Object` — same rule for the SDK-arg path: + // no R/W edge. The generic URI-literal scan may still record them as + // ambiguous (`access_type: None`) assets, like any `s3://…` string + // constant anywhere in a script. + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + await wmill.writeS3File("s3://broken", "[]") + await wmill.writeS3File("s3://bucket/", "[]") + } + "#; + let assets = parse_assets(input).expect("parse").assets; + assert!( + assets.iter().all(|a| a.access_type.is_none()), + "invalid URIs must not produce R/W edges: {assets:?}" + ); + } + #[test] fn test_ts_asset_parser_multiple_s3_object_writes() { // Mirrors the f/km/r_seed shape: several direct object-form writes in @@ -497,25 +570,25 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "/pipelines/km_real/enriched.json".to_string(), + path: "pipelines/km_real/enriched.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "/pipelines/km_real/raw_events.json".to_string(), + path: "pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "/pipelines/km_real/report.json".to_string(), + path: "pipelines/km_real/report.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "/pipelines/km_real/summary.json".to_string(), + path: "pipelines/km_real/summary.json".to_string(), access_type: Some(W), columns: None, }, @@ -536,7 +609,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "/out.json".to_string(), + path: "out.json".to_string(), access_type: Some(W), columns: None, },]) diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 096d93163b..96a1072017 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.750.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.750.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.750.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.750.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index e2afd2483b..c718763f55 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.742.0" +version = "1.750.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 5d9945cadd..f2a1a2b824 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -269,6 +269,12 @@ pub struct DelegateToGitRepoDetails { pub commit: Option, pub inventories_location: Option, pub vars_location: Option, + /// Path (relative to the cloned repo root) of an `ansible.cfg` to use as the + /// effective config for the run. When set, Windmill points `ANSIBLE_CONFIG` at + /// it so the repo's own settings (roles paths, inventory plugins, callbacks…) + /// apply, and only injects the settings that depend on runtime state it alone + /// controls (temp/home dirs, vault password) on top. + pub ansible_cfg: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub install_requirements: bool, } @@ -629,6 +635,10 @@ fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option Option, // `// freshness ` — SLA stating outputs must be at most - // `duration` old. Active backstop: when no other trigger has fired the - // script within the window, a watchdog re-runs it. Distinct from - // schedule (which is producer cadence); freshness is consumer SLA and - // applies regardless of which trigger last fired. + // `duration` old. Drives passive monitoring in CE (the asset graph + // colors the node's badge fresh/stale against its last successful run) + // and the enterprise watchdog (windmill-queue `freshness_watchdog`), + // which re-runs a stale unpartitioned producer. Distinct from schedule + // (which is producer cadence); freshness is consumer SLA and applies + // regardless of which trigger last fired. #[serde(skip_serializing_if = "Option::is_none", default)] pub freshness: Option, // `// trigger all` → AND join barrier; default (`any`) = OR (current @@ -122,6 +124,17 @@ pub struct ParseAssetsOutput { // column-lineage graph view, executes nothing. #[serde(skip_serializing_if = "Vec::is_empty", default)] pub column_lineage: Vec, + // Bare `// macros` (must be alone on the line, like `// pipeline`) — + // marks this DuckDB script as a workspace *macro library*: its body is + // CREATE [OR REPLACE] MACRO statements (plus plain setup) registered at + // deploy and injected as TEMP macros into consumer jobs at run time. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub macros: bool, + // `// use ` — force-inject the whole named macro + // library (definitions + its setup statements) into this script's jobs, + // for dynamic SQL that call-detection can't see. Accumulating. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub use_libs: Vec, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -205,6 +218,40 @@ pub struct PartitionSpec { pub start: Option, } +impl PartitionKind { + /// The `strftime`/chrono format that renders a time grain's identity + /// string. SINGLE SOURCE OF TRUTH: the partition resolver stamps the stored + /// identity with this, and the `wm_partition` materialize macro filters + /// against it — the two must never drift, so both read it from here. + /// `Dynamic` has no wall-clock identity; it falls back to a plain date only + /// where some format is unconditionally required (never for bucketing). + pub fn default_time_format(&self) -> &'static str { + match self { + PartitionKind::Hourly => "%Y-%m-%dT%H", + PartitionKind::Weekly => "%G-W%V", + PartitionKind::Monthly => "%Y-%m", + _ => "%Y-%m-%d", + } + } +} + +impl PartitionSpec { + /// Effective identity format for a TIME partition: the explicit `format=` + /// override, else the per-grain default. `None` for `dynamic` — its + /// identity is a caller-supplied key, not a formatted instant, so there is + /// no `strftime` bucketing expression (and hence no `wm_partition` macro). + pub fn time_strftime_format(&self) -> Option<&str> { + match self.kind { + PartitionKind::Dynamic { .. } => None, + _ => Some( + self.format + .as_deref() + .unwrap_or_else(|| self.kind.default_time_format()), + ), + } + } +} + // Freshness SLA. The duration is kept as a raw string ("1h", "30m", "2d") // and validated downstream — the parser deliberately doesn't bind to a // specific duration crate so the annotation grammar stays parser-light. @@ -224,15 +271,23 @@ pub struct RetrySpec { pub delay: Option, } -// `// materialize [manual] [append] [key=]` — declares that this -// script produces a *managed* materialization of `` (a `ducklake://` -// table). By default the runtime generates the write DDL around the script's -// single trailing `SELECT` and owns idempotency, partition-state and snapshot -// capture. `manual` is the escape hatch: the script writes its own DDL and the -// runtime only records state (track-only). The reconciliation strategy options -// (`append`, `key=`) apply to managed mode: none → DELETE-by-partition + -// INSERT (replace); `key=` → MERGE (dedup within slice); `append` → +// `// materialize [manual] [append] [key=] [history] [track=]` +// — declares that this script produces a *managed* materialization of `` +// (a `ducklake://` table). By default the runtime generates the write DDL around +// the script's single trailing `SELECT` and owns idempotency, partition-state +// and snapshot capture. `manual` is the escape hatch: the script writes its own +// DDL and the runtime only records state (track-only). The reconciliation +// strategy options apply to managed mode: none → DELETE-by-partition + INSERT +// (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). +// `key= history` upgrades the merge to SCD type 2: the SELECT is the current +// snapshot (one row per key), and a change to any tracked column (`track=`, +// default all non-key) closes the prior version and opens a new one, keeping full +// history (`valid_from`/`valid_to`/`is_current`). The leading keyword `scd2` is a +// recognized alias for `history`. `deletes=close` (scd2 only) also closes a key +// that disappears from the snapshot; default leaves absent keys current. +// `on_schema_change=ignore` suppresses downstream schema-contract warnings for +// the produced asset (save-time metadata only; default `warn`). #[derive(Serialize, Debug, PartialEq, Clone)] pub struct MaterializeSpec { pub target_kind: AssetKind, @@ -243,6 +298,131 @@ pub struct MaterializeSpec { pub append: bool, #[serde(skip_serializing_if = "Option::is_none", default)] pub unique_key: Option, + // `scd2` managed history mode: the SELECT is the current snapshot (one row + // per `unique_key`), and the runtime maintains a Slowly-Changing-Dimension + // type-2 history (`valid_from`/`valid_to`/`is_current`). `unique_key` (the + // `key=` opt) is the natural key; `track` lists the columns whose change + // opens a new version (empty ⇒ all non-key columns). Managed mode only. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub scd2: bool, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub track: Vec, + // scd2 only: `deletes=close` opts into hard-delete-close — a key that + // disappears from the snapshot has its current version closed (dbt's + // `hard_deletes=close`). Default (false) leaves absent keys current. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub close_deleted: bool, + // `on_schema_change=warn|ignore|fail|sync` governs two orthogonal things: + // • Save-time contract warnings (gap #2b): consumers referencing columns + // the captured schema no longer has warn by default; only `ignore` + // suppresses those warnings (`warn`/`fail`/`sync` all keep them). + // • Run-time write guardrails for the persist-and-mutate strategies + // (partitioned replace, merge, append), where the table schema is fixed + // at first CREATE and the write is positional — a renamed/added/removed + // SELECT column silently lands in the wrong column. `warn` logs the + // drift and proceeds positionally; `fail` aborts before mutating; `sync` + // ALTERs the table to match and writes by name. Whole-table replace and + // scd2 are unaffected. See `sql_materialize.rs`. + #[serde(skip_serializing_if = "OnSchemaChange::is_warn", default)] + pub on_schema_change: OnSchemaChange, +} + +impl MaterializeSpec { + /// The `_current` SCD2 companion view this managed materialize also + /// produces, or `None` when it isn't a managed scd2 target. Managed scd2 + /// creates the base table *and* a `_current` "latest row per key" view + /// each run (see `sql_materialize.rs`); `manual` mode owns its own DDL and + /// short-circuits before that codegen, so it produces no companion. + pub fn scd2_current_target(&self) -> Option<(AssetKind, String)> { + (self.scd2 && !self.manual) + .then(|| (self.target_kind, format!("{}_current", self.target_path))) + } + + /// Every asset this managed materialize produces: the base table, plus — for + /// managed scd2 — the `_current` companion view. The producer's + /// trailing `SELECT` doesn't express these writes (the runtime generates the + /// DDL), so this is the single source of truth every graph surface (deploy + /// asset rows, the CLI `--local` graph, and the frontend live graph) uses to + /// link reads of the base *and* the `_current` view back to this producer. + pub fn write_targets(&self) -> Vec<(AssetKind, String)> { + let mut targets = vec![(self.target_kind, self.target_path.clone())]; + targets.extend(self.scd2_current_target()); + targets + } +} + +// dbt's `on_schema_change`, covering both the save-time contract check and the +// run-time write guardrail for the positional persist-and-mutate strategies: +// • `warn` (default): surface consumer contract warnings; at write time, log +// the drift loudly and proceed with the positional write against the fixed +// table schema. +// • `ignore`: suppress consumer contract warnings; at write time, no guard +// (the pre-guardrail behaviour). +// • `fail`: keep contract warnings; at write time, abort the run before +// mutating when the SELECT's column *set* diverges from the table's. +// • `sync`: keep contract warnings; at write time, ALTER the table to match +// the SELECT (add/drop columns) and INSERT BY NAME. +// `warn`/`fail` drift detection is name-set based (added/removed columns), which +// is what the positional persist-and-mutate INSERT can misalign on. It does NOT +// flag a pure *reorder* of same-named columns: `SELECT b, a` into a `(a, b)` +// table has an identical column set, so `fail` does not abort and the positional +// INSERT swaps the values. Reorder-safety is exactly what `sync` provides +// (INSERT BY NAME maps by name), so a SELECT whose column order is not pinned to +// the table's should use `sync`, not `fail`. (An ordered-list comparison would +// close this, but a false positive there would abort a correctly-aligned write, +// so the guard stays on the set difference.) +// `fail`/`sync` only affect partitioned replace, merge and append; whole-table +// replace already rebuilds each run, and scd2 has no positional write — for an +// scd2 target `sync` degrades to `warn` (no write-time effect; deploy-time +// rejection is out of scope here). +#[derive(Serialize, Debug, PartialEq, Eq, Clone, Copy, Default)] +#[serde(rename_all = "lowercase")] +pub enum OnSchemaChange { + #[default] + Warn, + Ignore, + Fail, + Sync, +} + +impl OnSchemaChange { + pub fn is_warn(&self) -> bool { + matches!(self, OnSchemaChange::Warn) + } +} + +impl MaterializeSpec { + /// Deploy-time validation of the option combination against the script's + /// partitioning, returning a human-facing error for combinations the + /// runtime cannot honor. Called at save (`create_script_internal`) so a + /// misconfigured script is rejected up front, and again in the DuckDB + /// executor as a safety net for preview/test runs that never deploy. Both + /// checks are SCD2-specific and inert for `manual` mode (which owns its DDL + /// and ignores the reconciliation strategy). `partitioned` is whether the + /// script declares `// partitioned`. + pub fn validate(&self, partitioned: bool) -> Result<(), String> { + if self.manual || !self.scd2 { + return Ok(()); + } + // SCD2 needs a natural key to identify an entity across versions. + if self.unique_key.as_deref().map_or(true, str::is_empty) { + return Err( + "materialize scd2: requires a natural key — add `key=` (e.g. \ + `// materialize ducklake:/// key=id history`)" + .to_string(), + ); + } + // SCD2's diff/close/open shape has no partition-scoped form in v1. + if partitioned { + return Err( + "materialize scd2: `// partitioned` is not supported with scd2 in v1 — remove \ + `// partitioned`, or drop `history`/`scd2` to materialize the partition without \ + history" + .to_string(), + ); + } + Ok(()) + } } // `// data_test …` — a data-quality assertion run against the @@ -346,6 +526,8 @@ pub struct PipelineAnnotations { pub materialize: Option, pub data_tests: Vec, pub column_lineage: Vec, + pub macros: bool, + pub use_libs: Vec, } impl ParseAssetsOutput { @@ -372,6 +554,8 @@ impl ParseAssetsOutput { materialize: pipeline.materialize, data_tests: pipeline.data_tests, column_lineage: pipeline.column_lineage, + macros: pipeline.macros, + use_libs: pipeline.use_libs, } } } @@ -487,6 +671,24 @@ pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(Asset for (prefix, kind) in ASSET_KINDS.iter() { if s.starts_with(prefix) { let path = &s[prefix.len()..]; + // Canonicalize S3 keys to a single asset identity. The SDK object + // form (`{ s3: "key" }` / `S3Object(s3="key")`, default storage) + // resolves to `s3:///key`, whose path is `/key`, while DuckDB + // `s3://key` and `// on s3://key` yield the bare `key`. Strip every + // leading slash so the triple-slash default-storage form and the + // `s3://storage/key` form share one path — otherwise a TS/Python + // writer and a DuckDB reader of the same object become disconnected + // nodes in the pipeline graph. Stripping ALL leading slashes (not + // just one) keeps the identity stable through URI reconstruction: + // `trigger_spec_to_row` rebuilds `s3://`, so a canonical path + // must never itself start with `/` or the rebuilt ref would parse + // back to a different key. Only leading slashes are touched, so + // Hive-partition keys (`s3://b/y=2024/f.parquet`) are untouched. + let path = if matches!(kind, AssetKind::S3Object) { + path.trim_start_matches('/') + } else { + path + }; return Some((*kind, path)); } } @@ -602,7 +804,7 @@ fn parse_kv_opts(s: &str) -> BTreeMap { // - `on ` → asset / native trigger edge (including // the marker-only `on schedule` form) // - `partitioned [opts]` → partition declaration -// - `freshness ` → SLA / active backstop +// - `freshness ` → SLA window (badge + EE watchdog) // - `tag ` → worker-tag override (annotation wins // over UI-set value at deploy) // - `retry []` → cascade-only retry policy @@ -662,6 +864,30 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { continue; } + if let Some(after_kw) = consume_keyword(rest, "macros") { + // Strict like `pipeline`: keyword alone on the line, so prose + // such as `// macros are defined below` never false-positives. + if after_kw.trim().is_empty() { + out.macros = true; + } + continue; + } + + // `// use ` — accumulating. The argument must be a + // single whitespace-free token containing `/` (all script paths do), + // so prose like `// use this script to …` is dropped fail-safe. + if let Some(after_kw) = consume_keyword(rest, "use") { + let path = after_kw.trim(); + if !path.is_empty() + && !path.contains(char::is_whitespace) + && path.contains('/') + && !out.use_libs.iter().any(|p| p == path) + { + out.use_libs.push(path.to_string()); + } + continue; + } + if let Some(after_kw) = consume_keyword(rest, "partitioned") { if out.partition.is_none() { if let Some(spec) = parse_partitioned_spec(after_kw.trim()) { @@ -780,6 +1006,39 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { out } +// Count `// data_test <…>` lines in the leading comment header whose right-hand +// side fails to parse into a check. `parse_pipeline_annotations` drops these +// fail-safe (a malformed line just yields no check), which is the wrong default +// for a data-quality assertion: a typo silently disables the test. The deploy +// path uses this count to warn. Same leading-block boundary as the parser (stop +// at the first non-comment line) so a body comment can't be miscounted, and the +// same `parse_data_test_spec` grammar so "malformed" means exactly what the +// parser rejects — no second grammar to drift. +pub fn count_malformed_data_tests(code: &str) -> usize { + let mut malformed = 0; + for raw_line in code.lines() { + let line = raw_line.trim_start(); + if line.is_empty() { + continue; + } + let rest = if let Some(r) = line.strip_prefix("//") { + r + } else if let Some(r) = line.strip_prefix("--") { + r + } else if let Some(r) = line.strip_prefix('#') { + r + } else { + break; + }; + if let Some(after_kw) = consume_keyword(rest.trim_start(), "data_test") { + if parse_data_test_spec(after_kw.trim()).is_none() { + malformed += 1; + } + } + } + malformed +} + // Parse a `// retry []` right-hand side. `` is a // non-negative decimal; `` is an optional raw duration string left // for `parse_duration_secs` to validate at deploy. A bare zero count (or @@ -800,19 +1059,31 @@ fn parse_retry_spec(s: &str) -> Option { Some(RetrySpec { count, delay }) } -// Parse a `// materialize [manual] [append] [key=]` right-hand -// side. An optional leading `manual` token (whitespace-delimited) opts out of -// managed mode (track-only). The next whitespace token is the target asset URI +// Parse a `// materialize [manual] [append] [key=] [history] +// [track=]` right-hand side. An optional leading `manual` token opts out +// of managed mode (track-only); a leading `scd2` token is an alias for the +// `history` flag. The next whitespace token is the target asset URI // (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the -// remainder are strategy options — bare `append` and `key=` (merge key), -// which apply to managed mode only. A missing/empty target yields `None` (the -// annotation is dropped, fail-safe). +// remainder are strategy options — bare `append`, bare `history` (SCD type-2 on +// a keyed merge), `key=` (merge/scd2 key), `track=` (scd2 tracked +// columns), and `deletes=close` (scd2 hard-delete-close) — which apply to managed +// mode only. A missing/empty target yields `None` (the annotation is dropped, +// fail-safe). fn parse_materialize_spec(s: &str) -> Option { - let (manual, rest) = match s.strip_prefix("manual") { - Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => { - (true, after.trim_start()) - } - _ => (false, s), + // One optional leading mode keyword: `manual` (escape hatch, track-only) or + // `scd2` (alias for the `history` flag below). A missing keyword is the + // default managed mode. + fn strip_mode<'a>(s: &'a str, kw: &str) -> Option<&'a str> { + s.strip_prefix(kw) + .filter(|after| after.is_empty() || after.starts_with(char::is_whitespace)) + .map(|after| after.trim_start()) + } + let (manual, scd2_kw, rest) = if let Some(after) = strip_mode(s, "manual") { + (true, false, after) + } else if let Some(after) = strip_mode(s, "scd2") { + (false, true, after) + } else { + (false, false, s) }; let mut it = rest.trim().splitn(2, char::is_whitespace); let asset_tok = it.next()?; @@ -822,11 +1093,48 @@ fn parse_materialize_spec(s: &str) -> Option { return None; } let append = opts_str.split_whitespace().any(|t| t == "append"); - let unique_key = parse_kv_opts(opts_str) - .get("key") - .filter(|k| !k.is_empty()) - .cloned(); - Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key }) + // SCD type-2 history mode. The primary spelling is the bare `history` flag on + // a keyed merge (`key=history`) — it reads as "a keyed upsert that keeps + // history"; the leading `scd2` keyword is a recognized alias for the same. + let scd2 = scd2_kw || opts_str.split_whitespace().any(|t| t == "history"); + let opts = parse_kv_opts(opts_str); + let unique_key = opts.get("key").filter(|k| !k.is_empty()).cloned(); + // `track=` (scd2 only): comma-separated columns whose change opens a + // new version. Empty entries dropped; an empty list ⇒ track all non-key cols. + // Like every `=`-option here the value is whitespace-terminated, so the list + // must contain no spaces (`track=a,b`, not `track=a, b` — the rest is dropped). + let track = opts + .get("track") + .map(|v| { + v.split(',') + .map(|c| c.trim().to_string()) + .filter(|c| !c.is_empty()) + .collect::>() + }) + .unwrap_or_default(); + // `deletes=close` (scd2 only) opts into hard-delete-close; any other value + // (or absence) keeps the soft-delete default. + let close_deleted = opts.get("deletes").map(|v| v == "close").unwrap_or(false); + // `on_schema_change=ignore|fail|sync`; any other value (or absence) keeps + // the `warn` default, fail-safe like `deletes=` above (a typo must never + // silently disable the guardrail). + let on_schema_change = match opts.get("on_schema_change").map(String::as_str) { + Some("ignore") => OnSchemaChange::Ignore, + Some("fail") => OnSchemaChange::Fail, + Some("sync") => OnSchemaChange::Sync, + _ => OnSchemaChange::Warn, + }; + Some(MaterializeSpec { + target_kind, + target_path: path.to_string(), + manual, + append, + unique_key, + scd2, + track, + close_deleted, + on_schema_change, + }) } // Parse a `// data_test …` right-hand side into one `DataTest`. The @@ -1025,6 +1333,83 @@ fn parse_trigger_spec(s: &str) -> Option { mod pipeline_annotation_tests { use super::*; + #[test] + fn s3_key_normalization_unifies_uri_forms() { + // A TS/Python SDK write of `{ s3: "exports/x" }` (default storage) + // resolves to the URI `s3:///exports/x`, while a DuckDB read of + // `s3://exports/x` and the `// on s3://exports/x` trigger form yield the + // bare `exports/x`. All three must canonicalize to one asset key so + // the writer and reader connect in the pipeline graph. + let sdk_write = parse_asset_syntax("s3:///exports/x", false); + let duckdb_read = parse_asset_syntax("s3://exports/x", false); + assert_eq!(sdk_write, Some((AssetKind::S3Object, "exports/x"))); + assert_eq!(duckdb_read, Some((AssetKind::S3Object, "exports/x"))); + assert_eq!(sdk_write, duckdb_read); + + // The `// on` trigger annotation goes through the same function. + assert_eq!( + parse_asset_syntax("s3:///exports/x", true), + parse_asset_syntax("s3://exports/x", true) + ); + + // Explicit-storage form is unaffected (no leading slash to strip). + assert_eq!( + parse_asset_syntax("s3://mybucket/exports/x", false), + Some((AssetKind::S3Object, "mybucket/exports/x")) + ); + + // Hive-partition keys and nested paths under default storage are + // preserved verbatim (only leading slashes are stripped). + assert_eq!( + parse_asset_syntax("s3:///t/year=2024/month=01/f.parquet", false), + Some((AssetKind::S3Object, "t/year=2024/month=01/f.parquet")) + ); + + // Every leading slash is stripped so a canonical S3 path never starts + // with `/`. `S3Object(s3="/x")` resolves to the quad-slash URI + // `s3:////x`; the identity must be the bare `x` (not `/x`) so the ref + // that `trigger_spec_to_row` rebuilds round-trips back to it. + assert_eq!( + parse_asset_syntax("s3:////x", false), + Some((AssetKind::S3Object, "x")) + ); + assert_eq!( + parse_asset_syntax("s3://///deep///", false), + Some((AssetKind::S3Object, "deep///")) + ); + + // Non-S3 kinds keep their leading slash (their paths are workspace- + // relative and the slash is significant). + assert_eq!( + parse_asset_syntax("res://f/foo", false), + Some((AssetKind::Resource, "f/foo")) + ); + assert_eq!( + parse_asset_syntax("ducklake://analytics/orders", false), + Some((AssetKind::Ducklake, "analytics/orders")) + ); + } + + #[test] + fn s3_explicit_storage_aliases_default_storage_nested_key() { + // Accepted tradeoff of one canonical key: the explicit-storage form + // `s3://storage/key` and the default-storage nested-key form + // `s3:///storage/key` collapse to the same node `storage/key`, even + // though they name different objects. This is a best-effort lineage + // graph that does not split the first segment as a storage name; the + // collision only happens when a storage config is named to match a + // default-storage prefix. Pinned so the aliasing is intentional, not a + // latent surprise. + assert_eq!( + parse_asset_syntax("s3://mybucket/x", false), + parse_asset_syntax("s3:///mybucket/x", false) + ); + assert_eq!( + parse_asset_syntax("s3://mybucket/x", false), + Some((AssetKind::S3Object, "mybucket/x")) + ); + } + #[test] fn bare_pipeline_marker() { let out = parse_pipeline_annotations("// pipeline\nconsole.log('hi')"); @@ -1051,6 +1436,33 @@ mod pipeline_annotation_tests { assert!(!out.in_pipeline); } + #[test] + fn macros_marker_strict_like_pipeline() { + assert!(parse_pipeline_annotations("// macros\nCREATE MACRO m(a) AS a;").macros); + assert!(parse_pipeline_annotations("-- macros \nSELECT 1;").macros); + // Trailing prose / keyword variants disqualify the line. + assert!(!parse_pipeline_annotations("// macros are defined below\n").macros); + assert!(!parse_pipeline_annotations("// macros_v2\n").macros); + } + + #[test] + fn use_accumulates_dedups_and_rejects_prose() { + let out = parse_pipeline_annotations( + "// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\nSELECT 1;", + ); + assert_eq!(out.use_libs, vec!["f/lib/stats", "f/lib/dates"]); + + // Prose, slashless tokens, and multi-token lines are dropped fail-safe. + let out = parse_pipeline_annotations( + "// use this script to compute\n// use standalone\n// use f/lib/ok extra\n", + ); + assert!(out.use_libs.is_empty()); + + // Only the leading comment header is scanned. + let out = parse_pipeline_annotations("SELECT 1;\n-- use f/lib/late\n"); + assert!(out.use_libs.is_empty()); + } + #[test] fn on_schedule_marker() { // `// on schedule` is marker-only — the binding is the schedule row's @@ -1457,6 +1869,196 @@ mod pipeline_annotation_tests { assert_eq!(m.unique_key, None); } + #[test] + fn materialize_scd2_history_flag_with_key_and_track() { + // Primary spelling: `key=history` on a merge. + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history track=name,tier", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert!(!m.manual); + assert_eq!(m.unique_key.as_deref(), Some("id")); + assert_eq!(m.track, vec!["name".to_string(), "tier".to_string()]); + } + + #[test] + fn materialize_scd2_keyword_is_alias_for_history() { + let out = parse_pipeline_annotations("// materialize scd2 ducklake://a/dim key=id"); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert_eq!(m.unique_key.as_deref(), Some("id")); + assert!(m.track.is_empty()); + // soft-delete default + assert!(!m.close_deleted); + } + + #[test] + fn materialize_scd2_write_targets_include_current_view() { + // A managed scd2 materialize produces the base table AND the + // `_current` companion view, so the producer must be recorded as + // writing both (reads of the view otherwise resolve to an orphan asset). + let out = parse_pipeline_annotations( + "// materialize ducklake://main/dim_customers key=id history", + ); + let m = out.materialize.expect("materialize"); + assert_eq!( + m.write_targets(), + vec![ + (AssetKind::Ducklake, "main/dim_customers".to_string()), + ( + AssetKind::Ducklake, + "main/dim_customers_current".to_string() + ), + ] + ); + assert_eq!( + m.scd2_current_target(), + Some(( + AssetKind::Ducklake, + "main/dim_customers_current".to_string() + )) + ); + } + + #[test] + fn materialize_non_scd2_write_targets_are_base_only() { + // A plain merge (no `history`) creates no companion view — only the base. + let out = parse_pipeline_annotations("// materialize ducklake://main/dim_customers key=id"); + let m = out.materialize.expect("materialize"); + assert_eq!( + m.write_targets(), + vec![(AssetKind::Ducklake, "main/dim_customers".to_string())] + ); + assert_eq!(m.scd2_current_target(), None); + } + + #[test] + fn materialize_manual_scd2_has_no_companion_view() { + // `manual` mode owns its own DDL and never creates the `_current` view, + // so registering it would be a false producer edge. + let out = parse_pipeline_annotations( + "// materialize manual ducklake://main/dim_customers key=id history", + ); + let m = out.materialize.expect("materialize"); + assert!(m.manual && m.scd2); + assert_eq!(m.scd2_current_target(), None); + assert_eq!( + m.write_targets(), + vec![(AssetKind::Ducklake, "main/dim_customers".to_string())] + ); + } + + #[test] + fn materialize_scd2_deletes_close_opt() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history deletes=close", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert!(m.close_deleted); + // any other value keeps the soft-delete default + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history deletes=ignore", + ); + assert!(!out.materialize.expect("materialize").close_deleted); + } + + #[test] + fn materialize_validate_scd2_requires_key() { + // scd2 without `key=` is rejected at deploy (was a run-time error). + let m = parse_pipeline_annotations("// materialize ducklake://a/dim history") + .materialize + .expect("materialize"); + assert!(m.scd2 && m.unique_key.is_none()); + let err = m.validate(false).expect_err("scd2 without key must fail"); + assert!(err.contains("requires a natural key")); + // with a key it validates + let m = parse_pipeline_annotations("// materialize ducklake://a/dim key=id history") + .materialize + .expect("materialize"); + assert!(m.validate(false).is_ok()); + } + + #[test] + fn materialize_validate_scd2_rejects_partitioned() { + // scd2 + `// partitioned` has no v1 form — rejected at deploy. + let m = parse_pipeline_annotations("// materialize ducklake://a/dim key=id history") + .materialize + .expect("materialize"); + let err = m.validate(true).expect_err("scd2 + partitioned must fail"); + assert!(err.contains("`// partitioned` is not supported with scd2")); + // unpartitioned scd2 is fine + assert!(m.validate(false).is_ok()); + } + + #[test] + fn materialize_validate_non_scd2_and_manual_are_inert() { + // Non-scd2 strategies are unconstrained by these checks, partitioned or not. + let m = parse_pipeline_annotations("// materialize ducklake://a/orders key=id") + .materialize + .expect("materialize"); + assert!(m.validate(true).is_ok()); + assert!(m.validate(false).is_ok()); + // `manual` owns its DDL and ignores the strategy — never rejected here, + // even with a partitioned scd2-looking combo. + let m = parse_pipeline_annotations("// materialize manual ducklake://a/dim history") + .materialize + .expect("materialize"); + assert!(m.manual && m.scd2); + assert!(m.validate(true).is_ok()); + } + + #[test] + fn materialize_on_schema_change_opt() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/orders on_schema_change=ignore", + ); + let m = out.materialize.expect("materialize"); + assert_eq!(m.on_schema_change, OnSchemaChange::Ignore); + // default is warn + let out = parse_pipeline_annotations("// materialize ducklake://a/orders"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Warn + ); + // fail + sync parse to their own variants + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders on_schema_change=fail"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Fail + ); + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders on_schema_change=sync"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Sync + ); + // unknown/junk value keeps the warn default (fail-safe, like `deletes=`) + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders on_schema_change=bogus"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Warn + ); + // composes with other opts + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history on_schema_change=ignore", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert_eq!(m.on_schema_change, OnSchemaChange::Ignore); + } + + #[test] + fn materialize_key_without_history_is_plain_merge() { + let out = parse_pipeline_annotations("// materialize ducklake://a/dim key=id"); + let m = out.materialize.expect("materialize"); + assert!(!m.scd2, "no history flag ⇒ SCD1 merge, not scd2"); + assert_eq!(m.unique_key.as_deref(), Some("id")); + } + #[test] fn materialize_default_syntax_shorthand() { let out = parse_pipeline_annotations("// materialize ducklake"); @@ -1778,4 +2380,23 @@ mod pipeline_annotation_tests { )); assert!(out.column_lineage.is_empty()); } + + #[test] + fn count_malformed_data_tests_counts_only_broken_header_lines() { + let n = count_malformed_data_tests(concat!( + "-- data_test not_null id\n", // valid + "-- data_test unique id\n", // valid + "-- data_test accepted_values status paid\n", // malformed: missing `=` + "-- data_test relationships cust ducklake\n", // malformed: no `->` + "-- data_test\n", // malformed: bare keyword + "-- data_test f/tests/custom\n", // valid: custom script path + "SELECT 1 -- data_test not_a_test\n", // body line: not counted + )); + assert_eq!(n, 3); + // A clean header has zero. + assert_eq!( + count_malformed_data_tests("-- data_test unique id\nSELECT 1"), + 0 + ); + } } diff --git a/backend/parsers/windmill-parser/src/duckdb_builtins.rs b/backend/parsers/windmill-parser/src/duckdb_builtins.rs new file mode 100644 index 0000000000..d324fa4c30 --- /dev/null +++ b/backend/parsers/windmill-parser/src/duckdb_builtins.rs @@ -0,0 +1,936 @@ +// DuckDB built-in function names (scalar, aggregate, table, macro), used to +// reject workspace macros that would silently shadow a built-in (DuckDB +// allows the shadowing without error — verified on 1.5.4). Only +// identifier-shaped names are listed: workspace macro names are validated +// to `[a-z_][a-z0-9_]*` before this check, so operator names can't collide. +// +// Regenerate on DuckDB upgrades (sort in codepoint order — binary search): +// python3 -c "import duckdb,re; print('\\n'.join(sorted(set(r[0] for r in +// duckdb.connect().execute(\"SELECT DISTINCT lower(function_name) FROM +// duckdb_functions()\").fetchall() if re.fullmatch(r'[a-z_][a-z0-9_]*', r[0])))))" + +pub fn is_duckdb_builtin(name: &str) -> bool { + DUCKDB_BUILTIN_FUNCTIONS + .binary_search(&name.to_ascii_lowercase().as_str()) + .is_ok() +} + +const DUCKDB_BUILTIN_FUNCTIONS: &[&str] = &[ + "__internal_compress_integral_ubigint", + "__internal_compress_integral_uinteger", + "__internal_compress_integral_usmallint", + "__internal_compress_integral_utinyint", + "__internal_compress_string_hugeint", + "__internal_compress_string_ubigint", + "__internal_compress_string_uhugeint", + "__internal_compress_string_uinteger", + "__internal_compress_string_usmallint", + "__internal_compress_string_utinyint", + "__internal_decompress_integral_bigint", + "__internal_decompress_integral_hugeint", + "__internal_decompress_integral_integer", + "__internal_decompress_integral_smallint", + "__internal_decompress_integral_ubigint", + "__internal_decompress_integral_uhugeint", + "__internal_decompress_integral_uinteger", + "__internal_decompress_integral_usmallint", + "__internal_decompress_string", + "abs", + "acos", + "acosh", + "add", + "add_parquet_key", + "age", + "aggregate", + "ago", + "alias", + "all_profiling_output", + "any_value", + "apply", + "approx_count_distinct", + "approx_quantile", + "approx_top_k", + "arbitrary", + "arg_max", + "arg_max_null", + "arg_max_nulls_last", + "arg_min", + "arg_min_null", + "arg_min_nulls_last", + "argmax", + "argmin", + "array_agg", + "array_aggr", + "array_aggregate", + "array_append", + "array_apply", + "array_cat", + "array_concat", + "array_contains", + "array_cosine_distance", + "array_cosine_similarity", + "array_cross_product", + "array_distance", + "array_distinct", + "array_dot_product", + "array_extract", + "array_filter", + "array_grade_up", + "array_has", + "array_has_all", + "array_has_any", + "array_indexof", + "array_inner_product", + "array_intersect", + "array_length", + "array_negative_dot_product", + "array_negative_inner_product", + "array_pop_back", + "array_pop_front", + "array_position", + "array_prepend", + "array_push_back", + "array_push_front", + "array_reduce", + "array_resize", + "array_reverse", + "array_reverse_sort", + "array_select", + "array_slice", + "array_sort", + "array_to_json", + "array_to_string", + "array_to_string_comma_default", + "array_transform", + "array_unique", + "array_value", + "array_where", + "array_zip", + "arrow_scan", + "arrow_scan_dumb", + "ascii", + "asin", + "asinh", + "atan", + "atan2", + "atanh", + "avg", + "bar", + "base64", + "bin", + "bit_and", + "bit_count", + "bit_length", + "bit_or", + "bit_position", + "bit_xor", + "bitstring", + "bitstring_agg", + "bool_and", + "bool_or", + "can_cast_implicitly", + "cardinality", + "cast_to_type", + "cbrt", + "ceil", + "ceiling", + "century", + "char_length", + "character_length", + "checkpoint", + "chr", + "col_description", + "collations", + "combine", + "concat", + "concat_ws", + "constant_or_null", + "contains", + "copy_database", + "corr", + "cos", + "cosh", + "cot", + "count", + "count_if", + "count_star", + "countif", + "covar_pop", + "covar_samp", + "create_sort_key", + "cume_dist", + "current_catalog", + "current_connection_id", + "current_database", + "current_date", + "current_localtime", + "current_localtimestamp", + "current_query", + "current_query_id", + "current_role", + "current_schema", + "current_schemas", + "current_setting", + "current_transaction_id", + "current_user", + "currval", + "damerau_levenshtein", + "database_list", + "database_size", + "date_add", + "date_diff", + "date_part", + "date_sub", + "date_trunc", + "datediff", + "datepart", + "datesub", + "datetrunc", + "day", + "dayname", + "dayofmonth", + "dayofweek", + "dayofyear", + "days_in_month", + "decade", + "decode", + "degrees", + "dense_rank", + "disable_checkpoint_on_shutdown", + "disable_logging", + "disable_object_cache", + "disable_optimizer", + "disable_print_progress_bar", + "disable_profile", + "disable_profiling", + "disable_progress_bar", + "disable_verification", + "disable_verify_external", + "disable_verify_fetch_row", + "disable_verify_parallelism", + "disable_verify_serializer", + "divide", + "duckdb_approx_database_count", + "duckdb_columns", + "duckdb_connection_count", + "duckdb_constraints", + "duckdb_coordinate_systems", + "duckdb_databases", + "duckdb_dependencies", + "duckdb_extensions", + "duckdb_external_file_cache", + "duckdb_functions", + "duckdb_indexes", + "duckdb_keywords", + "duckdb_log_contexts", + "duckdb_logs", + "duckdb_logs_parsed", + "duckdb_memory", + "duckdb_optimizers", + "duckdb_prepared_statements", + "duckdb_profiling_settings", + "duckdb_schemas", + "duckdb_secret_types", + "duckdb_secrets", + "duckdb_sequences", + "duckdb_settings", + "duckdb_table_sample", + "duckdb_tables", + "duckdb_temporary_files", + "duckdb_types", + "duckdb_variables", + "duckdb_views", + "editdist3", + "element_at", + "enable_checkpoint_on_shutdown", + "enable_logging", + "enable_object_cache", + "enable_optimizer", + "enable_print_progress_bar", + "enable_profile", + "enable_profiling", + "enable_progress_bar", + "enable_verification", + "encode", + "ends_with", + "entropy", + "enum_code", + "enum_first", + "enum_last", + "enum_range", + "enum_range_boundary", + "epoch", + "epoch_ms", + "epoch_ns", + "epoch_us", + "equi_width_bins", + "era", + "error", + "even", + "exp", + "extension_versions", + "factorial", + "favg", + "fdiv", + "fill", + "filter", + "finalize", + "first", + "first_value", + "flatten", + "floor", + "fmod", + "force_checkpoint", + "format", + "format_bytes", + "format_pg_type", + "format_type", + "formatreadabledecimalsize", + "formatreadablesize", + "from_base64", + "from_binary", + "from_hex", + "from_json", + "from_json_strict", + "fsum", + "functions", + "gamma", + "gcd", + "gen_random_uuid", + "generate_series", + "generate_subscripts", + "geomean", + "geometric_mean", + "get_bit", + "get_block_size", + "get_current_time", + "get_current_timestamp", + "get_type", + "getvariable", + "glob", + "grade_up", + "greatest", + "greatest_common_divisor", + "group_concat", + "hamming", + "has_any_column_privilege", + "has_column_privilege", + "has_database_privilege", + "has_foreign_data_wrapper_privilege", + "has_function_privilege", + "has_language_privilege", + "has_schema_privilege", + "has_sequence_privilege", + "has_server_privilege", + "has_table_privilege", + "has_tablespace_privilege", + "hash", + "hex", + "histogram", + "histogram_exact", + "histogram_values", + "hour", + "icu_calendar_names", + "icu_collate_af", + "icu_collate_am", + "icu_collate_ar", + "icu_collate_ar_sa", + "icu_collate_as", + "icu_collate_az", + "icu_collate_be", + "icu_collate_bg", + "icu_collate_bn", + "icu_collate_bo", + "icu_collate_br", + "icu_collate_bs", + "icu_collate_ca", + "icu_collate_ceb", + "icu_collate_chr", + "icu_collate_cs", + "icu_collate_cy", + "icu_collate_da", + "icu_collate_de", + "icu_collate_de_at", + "icu_collate_dsb", + "icu_collate_dz", + "icu_collate_ee", + "icu_collate_el", + "icu_collate_en", + "icu_collate_en_us", + "icu_collate_eo", + "icu_collate_es", + "icu_collate_et", + "icu_collate_fa", + "icu_collate_fa_af", + "icu_collate_ff", + "icu_collate_fi", + "icu_collate_fil", + "icu_collate_fo", + "icu_collate_fr", + "icu_collate_fr_ca", + "icu_collate_fy", + "icu_collate_ga", + "icu_collate_gl", + "icu_collate_gu", + "icu_collate_ha", + "icu_collate_haw", + "icu_collate_he", + "icu_collate_he_il", + "icu_collate_hi", + "icu_collate_hr", + "icu_collate_hsb", + "icu_collate_hu", + "icu_collate_hy", + "icu_collate_id", + "icu_collate_id_id", + "icu_collate_ig", + "icu_collate_is", + "icu_collate_it", + "icu_collate_ja", + "icu_collate_ka", + "icu_collate_kk", + "icu_collate_kl", + "icu_collate_km", + "icu_collate_kn", + "icu_collate_ko", + "icu_collate_kok", + "icu_collate_ku", + "icu_collate_ky", + "icu_collate_lb", + "icu_collate_lij", + "icu_collate_lkt", + "icu_collate_ln", + "icu_collate_lo", + "icu_collate_lt", + "icu_collate_lv", + "icu_collate_mk", + "icu_collate_ml", + "icu_collate_mn", + "icu_collate_mr", + "icu_collate_ms", + "icu_collate_mt", + "icu_collate_my", + "icu_collate_nb", + "icu_collate_nb_no", + "icu_collate_ne", + "icu_collate_nl", + "icu_collate_nn", + "icu_collate_noaccent", + "icu_collate_nso", + "icu_collate_om", + "icu_collate_or", + "icu_collate_pa", + "icu_collate_pa_in", + "icu_collate_pl", + "icu_collate_ps", + "icu_collate_pt", + "icu_collate_ro", + "icu_collate_ru", + "icu_collate_sa", + "icu_collate_se", + "icu_collate_si", + "icu_collate_sk", + "icu_collate_sl", + "icu_collate_smn", + "icu_collate_sq", + "icu_collate_sr", + "icu_collate_sr_ba", + "icu_collate_sr_me", + "icu_collate_sr_rs", + "icu_collate_st", + "icu_collate_sv", + "icu_collate_sw", + "icu_collate_ta", + "icu_collate_te", + "icu_collate_th", + "icu_collate_tk", + "icu_collate_tn", + "icu_collate_to", + "icu_collate_tr", + "icu_collate_ug", + "icu_collate_uk", + "icu_collate_ur", + "icu_collate_uz", + "icu_collate_vi", + "icu_collate_wae", + "icu_collate_wo", + "icu_collate_xh", + "icu_collate_yi", + "icu_collate_yo", + "icu_collate_yue", + "icu_collate_yue_cn", + "icu_collate_zh", + "icu_collate_zh_cn", + "icu_collate_zh_hk", + "icu_collate_zh_mo", + "icu_collate_zh_sg", + "icu_collate_zh_tw", + "icu_collate_zu", + "icu_sort_key", + "ilike_escape", + "import_database", + "in_search_path", + "inet_client_addr", + "inet_client_port", + "inet_server_addr", + "inet_server_port", + "instr", + "is_histogram_other_bin", + "isfinite", + "isinf", + "isnan", + "isodow", + "isoyear", + "jaccard", + "jaro_similarity", + "jaro_winkler_similarity", + "json", + "json_array", + "json_array_length", + "json_contains", + "json_deserialize_sql", + "json_each", + "json_execute_serialized_sql", + "json_exists", + "json_extract", + "json_extract_path", + "json_extract_path_text", + "json_extract_string", + "json_group_array", + "json_group_object", + "json_group_structure", + "json_keys", + "json_merge_patch", + "json_object", + "json_pretty", + "json_quote", + "json_serialize_plan", + "json_serialize_sql", + "json_structure", + "json_transform", + "json_transform_strict", + "json_tree", + "json_type", + "json_valid", + "json_value", + "julian", + "kahan_sum", + "kurtosis", + "kurtosis_pop", + "lag", + "last", + "last_day", + "last_value", + "lcase", + "lcm", + "lead", + "least", + "least_common_multiple", + "left", + "left_grapheme", + "len", + "length", + "length_grapheme", + "levenshtein", + "lgamma", + "like_escape", + "list", + "list_aggr", + "list_aggregate", + "list_any_value", + "list_append", + "list_apply", + "list_approx_count_distinct", + "list_avg", + "list_bit_and", + "list_bit_or", + "list_bit_xor", + "list_bool_and", + "list_bool_or", + "list_cat", + "list_concat", + "list_contains", + "list_cosine_distance", + "list_cosine_similarity", + "list_count", + "list_distance", + "list_distinct", + "list_dot_product", + "list_element", + "list_entropy", + "list_extract", + "list_filter", + "list_first", + "list_grade_up", + "list_has", + "list_has_all", + "list_has_any", + "list_histogram", + "list_indexof", + "list_inner_product", + "list_intersect", + "list_kurtosis", + "list_kurtosis_pop", + "list_last", + "list_mad", + "list_max", + "list_median", + "list_min", + "list_mode", + "list_negative_dot_product", + "list_negative_inner_product", + "list_pack", + "list_position", + "list_prepend", + "list_product", + "list_reduce", + "list_resize", + "list_reverse", + "list_reverse_sort", + "list_select", + "list_sem", + "list_skewness", + "list_slice", + "list_sort", + "list_stddev_pop", + "list_stddev_samp", + "list_string_agg", + "list_sum", + "list_transform", + "list_unique", + "list_value", + "list_var_pop", + "list_var_samp", + "list_where", + "list_zip", + "listagg", + "ln", + "log", + "log10", + "log2", + "lower", + "lpad", + "ltrim", + "mad", + "make_date", + "make_time", + "make_timestamp", + "make_timestamp_ms", + "make_timestamp_ns", + "make_timestamptz", + "make_type", + "map", + "map_concat", + "map_contains", + "map_contains_entry", + "map_contains_value", + "map_entries", + "map_extract", + "map_extract_value", + "map_from_entries", + "map_keys", + "map_to_pg_oid", + "map_values", + "max", + "max_by", + "md5", + "md5_number", + "md5_number_lower", + "md5_number_upper", + "mean", + "median", + "metadata_info", + "microsecond", + "millennium", + "millisecond", + "min", + "min_by", + "minute", + "mismatches", + "mod", + "mode", + "month", + "monthname", + "multiply", + "nanosecond", + "nextafter", + "nextval", + "nfc_normalize", + "normalized_interval", + "not_ilike_escape", + "not_like_escape", + "now", + "nth_value", + "ntile", + "nullif", + "obj_description", + "octet_length", + "ord", + "pandas_scan", + "parquet_bloom_probe", + "parquet_file_metadata", + "parquet_full_metadata", + "parquet_kv_metadata", + "parquet_metadata", + "parquet_scan", + "parquet_schema", + "parse_dirname", + "parse_dirpath", + "parse_duckdb_log_message", + "parse_filename", + "parse_formatted_bytes", + "parse_path", + "percent_rank", + "pg_collation_is_visible", + "pg_conf_load_time", + "pg_conversion_is_visible", + "pg_function_is_visible", + "pg_get_constraintdef", + "pg_get_expr", + "pg_get_viewdef", + "pg_has_role", + "pg_is_other_temp_schema", + "pg_my_temp_schema", + "pg_opclass_is_visible", + "pg_operator_is_visible", + "pg_opfamily_is_visible", + "pg_postmaster_start_time", + "pg_size_pretty", + "pg_sleep", + "pg_table_is_visible", + "pg_timezone_names", + "pg_ts_config_is_visible", + "pg_ts_dict_is_visible", + "pg_ts_parser_is_visible", + "pg_ts_template_is_visible", + "pg_type_is_visible", + "pg_typeof", + "pi", + "platform", + "position", + "pow", + "power", + "pragma_collations", + "pragma_database_size", + "pragma_metadata_info", + "pragma_platform", + "pragma_show", + "pragma_storage_info", + "pragma_table_info", + "pragma_user_agent", + "pragma_version", + "prefix", + "printf", + "product", + "python_map_function", + "quantile", + "quantile_cont", + "quantile_disc", + "quarter", + "query", + "query_table", + "radians", + "random", + "range", + "rank", + "rank_dense", + "read_blob", + "read_csv", + "read_csv_auto", + "read_duckdb", + "read_json", + "read_json_auto", + "read_json_objects", + "read_json_objects_auto", + "read_ndjson", + "read_ndjson_auto", + "read_ndjson_objects", + "read_parquet", + "read_text", + "reduce", + "regexp_escape", + "regexp_extract", + "regexp_extract_all", + "regexp_full_match", + "regexp_matches", + "regexp_replace", + "regexp_split_to_array", + "regexp_split_to_table", + "regr_avgx", + "regr_avgy", + "regr_count", + "regr_intercept", + "regr_r2", + "regr_slope", + "regr_sxx", + "regr_sxy", + "regr_syy", + "remap_struct", + "repeat", + "repeat_row", + "replace", + "replace_type", + "reservoir_quantile", + "reverse", + "right", + "right_grapheme", + "round", + "round_even", + "roundbankers", + "row", + "row_number", + "row_to_json", + "rpad", + "rtrim", + "second", + "sem", + "seq_scan", + "session_user", + "set_bit", + "setseed", + "sha1", + "sha256", + "shobj_description", + "show", + "show_databases", + "show_tables", + "show_tables_expanded", + "sign", + "signbit", + "sin", + "sinh", + "skewness", + "sleep_ms", + "sniff_csv", + "split", + "split_part", + "sqrt", + "st_asbinary", + "st_astext", + "st_aswkb", + "st_aswkt", + "st_crs", + "st_geomfromwkb", + "st_intersects_extent", + "st_setcrs", + "starts_with", + "stats", + "stddev", + "stddev_pop", + "stddev_samp", + "storage_info", + "str_split", + "str_split_regex", + "strftime", + "string_agg", + "string_split", + "string_split_regex", + "string_to_array", + "strip_accents", + "strlen", + "strpos", + "strptime", + "struct_concat", + "struct_contains", + "struct_extract", + "struct_extract_at", + "struct_has", + "struct_indexof", + "struct_insert", + "struct_keys", + "struct_pack", + "struct_position", + "struct_update", + "struct_values", + "substr", + "substring", + "substring_grapheme", + "subtract", + "suffix", + "sum", + "sum_no_overflow", + "sumkahan", + "summary", + "switch", + "table_info", + "tan", + "tanh", + "test_all_types", + "test_vector_types", + "time_bucket", + "timetz_byte_comparable", + "timezone", + "timezone_hour", + "timezone_minute", + "to_base", + "to_base64", + "to_binary", + "to_centuries", + "to_days", + "to_decades", + "to_hex", + "to_hours", + "to_json", + "to_microseconds", + "to_millennia", + "to_milliseconds", + "to_minutes", + "to_months", + "to_quarters", + "to_seconds", + "to_timestamp", + "to_weeks", + "to_years", + "today", + "transaction_timestamp", + "translate", + "trim", + "trunc", + "truncate_duckdb_logs", + "try_strptime", + "txid_current", + "typeof", + "ucase", + "unbin", + "unhex", + "unicode", + "union_extract", + "union_tag", + "union_value", + "unnest", + "unpivot_list", + "upper", + "url_decode", + "url_encode", + "user", + "user_agent", + "uuid", + "uuid_extract_timestamp", + "uuid_extract_version", + "uuidv4", + "uuidv7", + "var_pop", + "var_samp", + "variance", + "variant_bytes_to_variant", + "variant_extract", + "variant_normalize", + "variant_to_parquet_variant", + "variant_typeof", + "vector_type", + "verify_external", + "verify_fetch_row", + "verify_parallelism", + "verify_serializer", + "version", + "wavg", + "week", + "weekday", + "weekofyear", + "weighted_avg", + "which_secret", + "write_log", + "xor", + "year", + "yearweek", +]; diff --git a/backend/parsers/windmill-parser/src/duckdb_macros.rs b/backend/parsers/windmill-parser/src/duckdb_macros.rs new file mode 100644 index 0000000000..20438d23d1 --- /dev/null +++ b/backend/parsers/windmill-parser/src/duckdb_macros.rs @@ -0,0 +1,591 @@ +//! Workspace DuckDB macro libraries (`// macros` annotation). +//! +//! A macro-library script's body is `CREATE [OR REPLACE] [TEMP] MACRO` +//! statements plus plain setup (ATTACH/INSTALL/LOAD/SET/PRAGMA). At deploy the +//! macros are parsed into the `macro_definition` registry; at job time the +//! worker injects the (transitively) called ones into consumer scripts as +//! `CREATE OR REPLACE TEMP MACRO` blocks. Everything is stored and re-emitted +//! **verbatim** (params, body) — no AST round-trip — so any expression DuckDB +//! accepts survives unchanged. +//! +//! DuckDB bind-checks macro bodies at CREATE time (macro→macro and table +//! references alike), so injected definitions must be emitted in dependency +//! order — hence `topo_order_macros` — and after the consumer's ATTACHes. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use crate::sql_materialize::{classify_block, split_statements, BlockClass}; + +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedMacro { + /// Lowercased bare identifier (DuckDB identifiers are case-insensitive + /// unquoted; qualified / quoted names are rejected at parse). + pub name: String, + /// Verbatim text inside the parameter parens (may be empty). + pub params: String, + /// Verbatim text after `AS [TABLE]`, without the trailing `;`. + pub body: String, + pub is_table: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum LibStatement { + Macro(ParsedMacro), + /// A non-macro statement allowed in a library: setup-class only + /// (ATTACH/INSTALL/LOAD/SET/PRAGMA/USE/CREATE TEMP …). Re-emitted verbatim + /// ahead of the macro definitions when the lib is injected via `// use`. + Setup(String), +} + +/// Statement text for injecting one macro into a consumer job. Always +/// TEMP (session-scoped — no catalog writes) and OR REPLACE (idempotent). +pub fn macro_create_statement(name: &str, params: &str, is_table: bool, body: &str) -> String { + format!( + "CREATE OR REPLACE TEMP MACRO {}({}) AS {}{};", + name, + params, + if is_table { "TABLE " } else { "" }, + body + ) +} + +fn is_ident(s: &str) -> bool { + let mut chars = s.chars(); + matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +// Case-insensitive whole-word prefix strip (whitespace-bounded), returning the +// remainder with leading whitespace trimmed. `get` (not slicing) so a +// multi-byte char straddling the boundary yields None instead of panicking. +fn strip_kw<'a>(s: &'a str, kw: &str) -> Option<&'a str> { + let prefix = s.get(..kw.len())?; + if prefix.eq_ignore_ascii_case(kw) { + let after = &s[kw.len()..]; + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + return Some(after.trim_start()); + } + } + None +} + +/// Parse one (comment-free, `;`-less) statement as a CREATE MACRO. Returns +/// `Ok(None)` when the statement is not macro-shaped at all (caller decides +/// whether it is acceptable setup), `Err` when it is macro-shaped but invalid. +fn parse_create_macro(stmt: &str) -> Result, String> { + let Some(mut rest) = strip_kw(stmt.trim(), "create") else { + return Ok(None); + }; + if let Some(r) = strip_kw(rest, "or") { + rest = strip_kw(r, "replace").ok_or("expected REPLACE after CREATE OR")?; + } + if let Some(r) = strip_kw(rest, "temp").or_else(|| strip_kw(rest, "temporary")) { + rest = r; + } + // `FUNCTION` is DuckDB's alias for `MACRO`. + let Some(rest) = strip_kw(rest, "macro").or_else(|| strip_kw(rest, "function")) else { + return Ok(None); + }; + + let name_end = rest + .find(|c: char| c.is_whitespace() || c == '(') + .unwrap_or(rest.len()); + let raw_name = &rest[..name_end]; + if raw_name.is_empty() { + return Err("CREATE MACRO: missing macro name".to_string()); + } + if raw_name.contains('.') || raw_name.contains('"') || !is_ident(raw_name) { + return Err(format!( + "macro name `{}` must be a plain unqualified identifier ([A-Za-z_][A-Za-z0-9_]*) in v1", + raw_name + )); + } + let name = raw_name.to_ascii_lowercase(); + + let rest = rest[name_end..].trim_start(); + if !rest.starts_with('(') { + return Err(format!( + "macro `{}`: expected a parenthesized parameter list after the name", + name + )); + } + // Balanced-paren scan for the verbatim param list. The statement comes + // from `split_statements` so comments are gone, but strings may contain + // parens (e.g. a default value) — skip quoted spans. + let bytes = rest.as_bytes(); + let mut depth = 0usize; + let mut i = 0usize; + let mut close = None; + while i < bytes.len() { + match bytes[i] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + close = Some(i); + break; + } + } + q @ (b'\'' | b'"') => { + i += 1; + while i < bytes.len() && bytes[i] != q { + i += 1; + } + } + _ => {} + } + i += 1; + } + let Some(close) = close else { + return Err(format!( + "macro `{}`: unbalanced parameter parentheses", + name + )); + }; + let params = rest[1..close].trim().to_string(); + + let after_params = rest[close + 1..].trim_start(); + let Some(mut body) = strip_kw(after_params, "as") else { + return Err(format!( + "macro `{}`: expected AS after the parameter list", + name + )); + }; + let is_table = match strip_kw(body, "table") { + Some(r) => { + body = r; + true + } + None => false, + }; + let body = body.trim().trim_end_matches(';').trim_end().to_string(); + if body.is_empty() { + return Err(format!("macro `{}`: empty body", name)); + } + Ok(Some(ParsedMacro { name, params, body, is_table })) +} + +fn stmt_head(stmt: &str) -> String { + stmt.split_whitespace() + .take(4) + .collect::>() + .join(" ") +} + +/// Managed ATTACH forms (`ducklake://…`, `datatable://…`, resource URIs) are +/// rewritten by the worker's transform pass — which `// use`-injected lib +/// setup bypasses — so a library may only use plain, self-contained ATTACHes. +pub fn is_managed_attach(stmt: &str) -> bool { + let s = stmt.trim(); + if strip_kw(s, "attach").is_none() { + return false; + } + let lower = s.to_ascii_lowercase(); + [ + "'ducklake:", + "'datatable:", + "'windmill:", + "'$res:", + "\"ducklake:", + "\"datatable:", + "\"windmill:", + "\"$res:", + ] + .iter() + .any(|p| lower.contains(p)) +} + +/// Parse a `// macros` library body. Statements are either macro definitions +/// or setup; anything else is an error (user-facing message). +pub fn parse_macro_library(sql: &str) -> Result, String> { + let mut out = Vec::new(); + for stmt in split_statements(sql) { + match parse_create_macro(&stmt)? { + Some(m) => out.push(LibStatement::Macro(m)), + None => match classify_block(&stmt) { + BlockClass::Setup => { + if is_managed_attach(&stmt) { + return Err(format!( + "a `// macros` library cannot use managed ATTACH forms \ + (ducklake://, datatable://, resource URIs) in v1 — its setup is \ + injected verbatim into consumers, bypassing the ATTACH rewrite: `{}`", + stmt_head(&stmt) + )); + } + out.push(LibStatement::Setup(stmt)) + } + _ => { + return Err(format!( + "a `// macros` library may only contain CREATE [OR REPLACE] MACRO \ + statements plus setup (ATTACH/INSTALL/LOAD/SET/PRAGMA); found: `{}`", + stmt_head(&stmt) + )) + } + }, + } + } + Ok(out) +} + +/// Whether the statement is macro-definition-shaped (`CREATE [OR REPLACE] +/// [TEMP] MACRO|FUNCTION …`), regardless of whether the rest of the header +/// parses. Used by the worker's injection splice to keep injected blocks +/// *after* a script's own leading definitions (an injected body may only call +/// a local macro once the local CREATE has run — DuckDB binds at CREATE). +pub fn is_macro_definition(stmt: &str) -> bool { + let Some(mut rest) = strip_kw(stmt.trim(), "create") else { + return false; + }; + if let Some(r) = strip_kw(rest, "or") { + let Some(r2) = strip_kw(r, "replace") else { + return false; + }; + rest = r2; + } + if let Some(r) = strip_kw(rest, "temp").or_else(|| strip_kw(rest, "temporary")) { + rest = r; + } + strip_kw(rest, "macro") + .or_else(|| strip_kw(rest, "function")) + .is_some() +} + +/// Parse a single macro-definition statement (`CREATE [OR REPLACE] [TEMP] +/// MACRO …`). `None` when the statement isn't macro-shaped or its header is +/// malformed — callers using definitions as placement anchors skip those +/// (they fail at execution regardless). +pub fn parse_macro_definition(stmt: &str) -> Option { + parse_create_macro(stmt).ok().flatten() +} + +/// Names of macros the given statements define themselves (`CREATE [OR +/// REPLACE] [TEMP] MACRO …`). A consumer's own definition is authoritative +/// over a same-named workspace macro — the worker subtracts these before +/// planning injection, so deploying a library can never silently replace a +/// script's local macro. Malformed macro-shaped statements are skipped (they +/// fail at execution regardless). +pub fn locally_defined_macro_names(statements: &[String]) -> HashSet { + statements + .iter() + .filter_map(|s| parse_create_macro(s).ok().flatten().map(|m| m.name)) + .collect() +} + +/// Names from `names` that `sql` calls: an identifier token immediately +/// followed by `(` (after optional whitespace), not `.`-qualified. Scans the +/// comment-stripped statements; string-literal contents are skipped. Matching +/// is deliberately lexical — over-matching only injects an unused TEMP macro. +pub fn detect_macro_calls(sql: &str, names: &HashSet) -> HashSet { + let mut found = HashSet::new(); + if names.is_empty() { + return found; + } + for stmt in split_statements(sql) { + let bytes = stmt.as_bytes(); + let n = bytes.len(); + let mut i = 0usize; + let mut prev: Option = None; + while i < n { + let c = bytes[i]; + // skip quoted spans ('' escape for single quotes is handled by + // the fact that the reopened quote just starts another skip) + if c == b'\'' || c == b'"' { + let q = c; + i += 1; + while i < n && bytes[i] != q { + i += 1; + } + i += 1; + prev = Some(q); + continue; + } + let is_ident_start = c.is_ascii_alphabetic() || c == b'_'; + let prev_blocks = + matches!(prev, Some(p) if p == b'.' || p.is_ascii_alphanumeric() || p == b'_'); + if is_ident_start && !prev_blocks { + let start = i; + while i < n && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') { + i += 1; + } + let word = stmt[start..i].to_ascii_lowercase(); + let mut j = i; + while j < n && (bytes[j] as char).is_whitespace() { + j += 1; + } + if j < n && bytes[j] == b'(' && names.contains(&word) { + found.insert(word); + } + prev = Some(bytes[i - 1]); + continue; + } + prev = Some(c); + i += 1; + } + } + found +} + +/// Order `selected` macro names so every macro comes after the macros its +/// body calls (Kahn's algorithm, name-sorted ties for determinism). `defs` +/// maps each selected name to its body. Errors on a dependency cycle (only +/// reachable via cross-library deploy interleaving — DuckDB itself could +/// never have bound a cycle). +pub fn topo_order_macros( + selected: &HashSet, + defs: &BTreeMap, +) -> Result, String> { + let all: HashSet = selected.clone(); + // deps[m] = selected macros m's body calls; rev[d] = macros depending on d + let mut deps: BTreeMap> = BTreeMap::new(); + let mut rev: BTreeMap> = BTreeMap::new(); + for name in selected { + let body = defs + .get(name) + .ok_or_else(|| format!("macro `{}` has no definition", name))?; + let mut called = detect_macro_calls(body, &all); + called.remove(name); // ignore self-recursion (DuckDB rejects it at CREATE anyway) + for d in &called { + rev.entry(d.clone()).or_default().insert(name.clone()); + } + deps.insert(name.clone(), called.into_iter().collect()); + } + let mut ready: BTreeSet = deps + .iter() + .filter(|(_, d)| d.is_empty()) + .map(|(n, _)| n.clone()) + .collect(); + let mut out = Vec::with_capacity(selected.len()); + while let Some(name) = ready.iter().next().cloned() { + ready.remove(&name); + out.push(name.clone()); + if let Some(dependents) = rev.get(&name) { + for dep in dependents.clone() { + let d = deps.get_mut(&dep).unwrap(); + d.remove(&name); + if d.is_empty() { + ready.insert(dep); + } + } + } + deps.remove(&name); + } + if !deps.is_empty() { + let cycle: Vec = deps.keys().cloned().collect(); + return Err(format!( + "macro dependency cycle involving: {}", + cycle.join(", ") + )); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names(list: &[&str]) -> HashSet { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn parses_scalar_macro() { + let lib = + parse_macro_library("CREATE MACRO surrogate_key(a, b) AS md5(concat_ws('||', a, b));") + .unwrap(); + assert_eq!( + lib, + vec![LibStatement::Macro(ParsedMacro { + name: "surrogate_key".into(), + params: "a, b".into(), + body: "md5(concat_ws('||', a, b))".into(), + is_table: false, + })] + ); + } + + #[test] + fn parses_table_macro_or_replace_temp_and_function_alias() { + let lib = parse_macro_library( + "CREATE OR REPLACE TEMP MACRO top_n(t_max) AS TABLE SELECT * FROM t WHERE x <= t_max;\n\ + create function dbl(a) as a * 2;", + ) + .unwrap(); + match &lib[0] { + LibStatement::Macro(m) => { + assert_eq!(m.name, "top_n"); + assert!(m.is_table); + assert_eq!(m.body, "SELECT * FROM t WHERE x <= t_max"); + } + other => panic!("expected macro, got {:?}", other), + } + match &lib[1] { + LibStatement::Macro(m) => { + assert_eq!(m.name, "dbl"); + assert!(!m.is_table); + } + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn parses_default_params_and_nested_parens() { + let lib = parse_macro_library( + "CREATE MACRO safe_div(a, b, fallback := (0)) AS CASE WHEN b = 0 THEN fallback ELSE a / b END;", + ) + .unwrap(); + match &lib[0] { + LibStatement::Macro(m) => { + assert_eq!(m.params, "a, b, fallback := (0)"); + assert!(m.body.starts_with("CASE WHEN")); + } + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn params_with_string_containing_paren() { + let lib = parse_macro_library("CREATE MACRO f(sep := '(') AS concat(sep, 'x');").unwrap(); + match &lib[0] { + LibStatement::Macro(m) => assert_eq!(m.params, "sep := '('"), + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn setup_statements_allowed_and_kept_in_order() { + let lib = + parse_macro_library("-- a comment\nATTACH 'x.duckdb' AS ext;\nCREATE MACRO m() AS 1;") + .unwrap(); + assert_eq!(lib.len(), 2); + assert!(matches!(&lib[0], LibStatement::Setup(s) if s.starts_with("ATTACH"))); + assert!(matches!(&lib[1], LibStatement::Macro(_))); + } + + #[test] + fn rejects_non_setup_statements() { + let err = parse_macro_library("CREATE MACRO m() AS 1; SELECT 1;").unwrap_err(); + assert!(err.contains("may only contain"), "{err}"); + let err = parse_macro_library("CREATE TABLE t(x int);").unwrap_err(); + assert!(err.contains("may only contain"), "{err}"); + } + + #[test] + fn rejects_managed_attach_setup() { + let err = + parse_macro_library("ATTACH 'ducklake://analytics' AS lake;\nCREATE MACRO m() AS 1;") + .unwrap_err(); + assert!(err.contains("managed ATTACH"), "{err}"); + } + + #[test] + fn rejects_qualified_and_quoted_names() { + assert!(parse_macro_library("CREATE MACRO lake.m(a) AS a;") + .unwrap_err() + .contains("unqualified")); + assert!(parse_macro_library("CREATE MACRO \"weird name\"(a) AS a;").is_err()); + } + + #[test] + fn rejects_missing_params_or_body() { + assert!(parse_macro_library("CREATE MACRO m AS 1;").is_err()); + assert!(parse_macro_library("CREATE MACRO m(a);").is_err()); + assert!(parse_macro_library("CREATE MACRO m(a) AS ;").is_err()); + } + + #[test] + fn detect_basic_and_word_boundaries() { + let ns = names(&["dbl", "avg_x"]); + let found = detect_macro_calls("SELECT dbl(1), my_dbl(2), avg_x (3) FROM t", &ns); + assert!(found.contains("dbl")); + assert!(found.contains("avg_x")); // whitespace before paren ok + assert_eq!(found.len(), 2); + } + + #[test] + fn detect_skips_qualified_strings_and_comments() { + let ns = names(&["dbl"]); + assert!(detect_macro_calls("SELECT lake.dbl(1)", &ns).is_empty()); + assert!(detect_macro_calls("SELECT 'dbl(1)'", &ns).is_empty()); + assert!(detect_macro_calls("-- dbl(1)\nSELECT 1", &ns).is_empty()); + assert!(detect_macro_calls("SELECT dbl FROM t", &ns).is_empty()); // no call parens + } + + #[test] + fn detect_case_insensitive_and_table_macro_position() { + let ns = names(&["top_n"]); + assert!(!detect_macro_calls("SELECT * FROM TOP_N(3)", &ns).is_empty()); + } + + #[test] + fn topo_chain_and_diamond() { + let mut defs = BTreeMap::new(); + defs.insert("a".to_string(), "b(1) + c(2)".to_string()); + defs.insert("b".to_string(), "d(1)".to_string()); + defs.insert("c".to_string(), "d(2)".to_string()); + defs.insert("d".to_string(), "1".to_string()); + let sel = names(&["a", "b", "c", "d"]); + let order = topo_order_macros(&sel, &defs).unwrap(); + let pos = |n: &str| order.iter().position(|x| x == n).unwrap(); + assert!(pos("d") < pos("b")); + assert!(pos("d") < pos("c")); + assert!(pos("b") < pos("a")); + assert!(pos("c") < pos("a")); + } + + #[test] + fn topo_cycle_errors() { + let mut defs = BTreeMap::new(); + defs.insert("a".to_string(), "b(1)".to_string()); + defs.insert("b".to_string(), "a(1)".to_string()); + let err = topo_order_macros(&names(&["a", "b"]), &defs).unwrap_err(); + assert!(err.contains("cycle"), "{err}"); + } + + #[test] + fn non_ascii_body_survives_verbatim() { + // Regression: the statement splitter must not Latin-1-mojibake + // multi-byte text — macro bodies are persisted and re-executed. + let lib = parse_macro_library("CREATE MACRO greet(a) AS a || ' café ☕';").unwrap(); + match &lib[0] { + LibStatement::Macro(m) => assert_eq!(m.body, "a || ' café ☕'"), + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn non_ascii_garbage_errors_instead_of_panicking() { + // Regression: keyword matching must not byte-slice across a char + // boundary (panicked on inputs like this before). + assert!(parse_macro_library("abcé foo;").is_err()); + assert!(parse_macro_library("créate macro m(a) AS a;").is_err()); + } + + #[test] + fn locally_defined_names_extracted() { + let blocks = vec![ + "ATTACH 'x' AS a;".to_string(), + "CREATE TEMP MACRO dbl(a) AS a * 2;".to_string(), + "SELECT dbl(2);".to_string(), + ]; + let local = locally_defined_macro_names(&blocks); + assert!(local.contains("dbl")); + assert_eq!(local.len(), 1); + } + + #[test] + fn create_statement_roundtrip() { + assert_eq!( + macro_create_statement("m", "a, b := 1", true, "SELECT a + b"), + "CREATE OR REPLACE TEMP MACRO m(a, b := 1) AS TABLE SELECT a + b;" + ); + } + + #[test] + fn builtin_lookup() { + use crate::duckdb_builtins::is_duckdb_builtin; + assert!(is_duckdb_builtin("concat")); + assert!(is_duckdb_builtin("CONCAT")); + assert!(is_duckdb_builtin("read_csv")); + assert!(!is_duckdb_builtin("surrogate_key")); + } +} diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 19bc5602cd..4270edcc79 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -13,6 +13,8 @@ use serde::Serialize; use serde_json::Value; pub mod asset_parser; +pub mod duckdb_builtins; +pub mod duckdb_macros; pub mod sql_materialize; /// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index 7a3c72bdce..0e6e6656eb 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -87,28 +87,32 @@ impl WrapError { pub fn split_statements(sql: &str) -> Vec { let mut out = Vec::new(); let mut cur = String::new(); - let bytes = sql.as_bytes(); + // Char-wise, not byte-wise: the emitted strings are re-executed + // (materialize codegen) and persisted (macro registry), so a Latin-1 + // `bytes[i] as char` decode would corrupt any multi-byte text inside + // statements. All delimiters are ASCII — split positions are unaffected. + let chars: Vec = sql.chars().collect(); let mut i = 0; - let n = bytes.len(); + let n = chars.len(); while i < n { - let c = bytes[i] as char; + let c = chars[i]; // line comment — `--` (SQL) or `//`. The `//` form is not SQL, but it // is how Windmill pipeline annotations (`// materialize`, `// pipeline`, // …) are written, and they sit above the SQL in the same script; strip // them so they don't pollute the first statement block's classification // or the generated setup SQL. - if (c == '-' && i + 1 < n && bytes[i + 1] == b'-') - || (c == '/' && i + 1 < n && bytes[i + 1] == b'/') + if (c == '-' && i + 1 < n && chars[i + 1] == '-') + || (c == '/' && i + 1 < n && chars[i + 1] == '/') { - while i < n && bytes[i] != b'\n' { + while i < n && chars[i] != '\n' { i += 1; } continue; } // block comment - if c == '/' && i + 1 < n && bytes[i + 1] == b'*' { + if c == '/' && i + 1 < n && chars[i + 1] == '*' { i += 2; - while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { + while i + 1 < n && !(chars[i] == '*' && chars[i + 1] == '/') { i += 1; } i += 2; @@ -119,10 +123,10 @@ pub fn split_statements(sql: &str) -> Vec { cur.push(c); i += 1; while i < n { - cur.push(bytes[i] as char); - if bytes[i] == b'\'' { + cur.push(chars[i]); + if chars[i] == '\'' { // doubled '' is an escaped quote, stay in string - if i + 1 < n && bytes[i + 1] == b'\'' { + if i + 1 < n && chars[i + 1] == '\'' { cur.push('\''); i += 2; continue; @@ -139,8 +143,8 @@ pub fn split_statements(sql: &str) -> Vec { cur.push(c); i += 1; while i < n { - cur.push(bytes[i] as char); - if bytes[i] == b'"' { + cur.push(chars[i]); + if chars[i] == '"' { i += 1; break; } @@ -327,20 +331,45 @@ fn snippet(stmt: &str) -> String { // --------------------------------------------------------------------------- /// How a (partition of a) materialized table is reconciled on each run. -/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else -/// `unique_key` → `Merge`, else `Replace`. +/// Derived at deploy from the annotation: `key=history` (or the `scd2` +/// alias) → `Scd2`, else `append` → `Append`, else `unique_key` → `Merge`, else +/// `Replace`. #[derive(Debug, Clone, PartialEq, Eq)] pub enum MaterializeStrategy { /// DELETE the current partition, then INSERT — partition becomes exactly /// what the SELECT returned. Full-refresh of the slice. Replace, /// Upsert within the slice on `unique_key` (delete-by-key + insert); rows - /// absent from the SELECT are left in place. + /// absent from the SELECT are left in place. This is SCD type 1: a changed + /// row overwrites the prior value, keeping no history. Merge { unique_key: String }, /// INSERT only — immutable event-log semantics. Append, + /// Slowly Changing Dimension type 2: the SELECT is the *current* snapshot + /// (one row per `key`); a change to any tracked column closes the prior + /// version (`valid_to`/`is_current=false`) and opens a new one, so the full + /// history is preserved. `track` empty ⇒ every non-key column is tracked. + /// `close_deleted` (opt-in `deletes=close`) also closes the current version + /// of a key that disappears from the snapshot (dbt's `hard_deletes=close`); + /// default (false) leaves absent keys current (soft delete). + /// Unpartitioned only (the worker rejects `// partitioned` + scd2). + Scd2 { key: String, track: Vec, close_deleted: bool }, } +/// SCD2 metadata columns appended to the managed history table. Fixed names so +/// the generated diff/close/open SQL and any `// data_test` on them agree. +const SCD2_VALID_FROM: &str = "valid_from"; +const SCD2_VALID_TO: &str = "valid_to"; +const SCD2_IS_CURRENT: &str = "is_current"; +/// Connection-local temp table holding the keys whose version must be rotated +/// this run (changed + new). Captured before the write so the close and the +/// open see the same set. `_wm_` prefix so it never collides with user tables. +const SCD2_CHANGED_KEYS: &str = "_wm_scd2_changed"; +/// Connection-local temp table holding the keys that disappeared from the +/// snapshot this run (present-and-current in the table, absent from the SELECT). +/// Only used when `close_deleted` (`deletes=close`) is set. +const SCD2_DELETED_KEYS: &str = "_wm_scd2_deleted"; + /// Inputs to materialization codegen, all resolved at run time by the worker. /// Pure: produces SQL text; executes nothing. #[derive(Debug, Clone)] @@ -361,9 +390,39 @@ pub struct MaterializeCodegen<'a> { /// and the partition column / `SET PARTITIONED BY` are omitted. pub partitioned: bool, pub strategy: MaterializeStrategy, + /// Write-time guardrail for a drifted SELECT vs the fixed table schema. + /// Only the persist-and-mutate strategies (partitioned replace, merge, + /// append) act on it: `Fail` emits an in-txn guard that raises on drift, + /// `Sync` writes BY NAME and expects the executor to inject `ALTER TABLE` + /// DDL at the [`SYNC_ALTER_SENTINEL`] slot, `Warn`/`Ignore` write + /// positionally (drift surfaced by the summary in `Warn`, silent in + /// `Ignore`). See [`MaterializeCodegen::is_persist_and_mutate`]. + pub on_schema_change: OnSchemaChange, } +/// The exact statement the `sync` codegen emits right after `BEGIN +/// TRANSACTION;` as the injection slot for `ALTER TABLE … ADD/DROP COLUMN` +/// DDL. The executor computes the drift with a pre-pass probe and replaces this +/// literal in the assembled query text (with the DDL, or removes it when there +/// is no drift). Classified `Write` so the EE write-audit-publish reassembly +/// keeps it inside the transaction with the mutations; a plain no-op SELECT so +/// that if it is somehow left un-replaced the run still succeeds unchanged. +pub const SYNC_ALTER_SENTINEL: &str = "SELECT '__wm_sync_alter_sentinel__' AS _wm_sync;"; + impl<'a> MaterializeCodegen<'a> { + /// Whether this (strategy, partitioned) uses the positional persist-and- + /// mutate write whose table schema is fixed at first CREATE — the only case + /// the `on_schema_change` write-time guardrail applies to. Whole-table + /// replace (`CREATE OR REPLACE`) and scd2 self-heal / are name-mapped, so + /// they are excluded. + pub fn is_persist_and_mutate(&self) -> bool { + match self.strategy { + MaterializeStrategy::Scd2 { .. } => false, + MaterializeStrategy::Replace => self.partitioned, + MaterializeStrategy::Append | MaterializeStrategy::Merge { .. } => true, + } + } + /// The ordered statements that perform the materialization, to be run after /// the setup blocks and inside the caller's execution. The first-run /// bootstrap is idempotent (`IF NOT EXISTS`), so this is safe to run every @@ -377,6 +436,13 @@ impl<'a> MaterializeCodegen<'a> { let pval = self.partition_value_sql; let mut out = Vec::new(); + // SCD2 has a shape unlike the DELETE/INSERT strategies (diff → close old + // → open new) and does not support partitioning (rejected at the worker), + // so it is generated up front by its own helper. + if let MaterializeStrategy::Scd2 { key, track, close_deleted } = &self.strategy { + return self.scd2_statements(key, track, *close_deleted); + } + // Whole-table replace: rebuild the table to match the SELECT's *current* // schema each run with one atomic `CREATE OR REPLACE` (which DuckLake // still snapshots). This is the only path that survives a changed SELECT @@ -404,22 +470,39 @@ impl<'a> MaterializeCodegen<'a> { "CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;" )); } - out.push("BEGIN TRANSACTION;".to_string()); + // Write-time schema guardrail, emitted right after BEGIN so it runs + // before any mutation (a failing guard aborts before touching data; the + // sync ALTERs run before the INSERT so the sets match). Reached only on + // the persist-and-mutate path here (whole-table replace and scd2 return + // above), so no extra strategy gate is needed. + match self.on_schema_change { + OnSchemaChange::Fail => out.push(schema_drift_guard_sql(sel, t, pcol)), + OnSchemaChange::Sync => out.push(SYNC_ALTER_SENTINEL.to_string()), + OnSchemaChange::Warn | OnSchemaChange::Ignore => {} + } // The rows to write, with the partition column appended when partitioned. let source = if self.partitioned { format!("SELECT *, {pval} AS {pcol} FROM ({sel})") } else { format!("SELECT * FROM ({sel})") }; + // `sync` maps columns by name (positional would cross-wire: an ALTERed + // ADD COLUMN appends at the end, so a positional INSERT of the SELECT + // would fill it from the wrong source column). + let by_name = if self.on_schema_change == OnSchemaChange::Sync { + " BY NAME" + } else { + "" + }; match &self.strategy { MaterializeStrategy::Replace => { // Only reached when partitioned (whole-table replace returned above). out.push(format!("DELETE FROM {t} WHERE {pcol} = {pval};")); - out.push(format!("INSERT INTO {t} {source};")); + out.push(format!("INSERT INTO {t}{by_name} {source};")); } MaterializeStrategy::Append => { - out.push(format!("INSERT INTO {t} {source};")); + out.push(format!("INSERT INTO {t}{by_name} {source};")); } MaterializeStrategy::Merge { unique_key } => { // Upsert within the slice via delete-by-key + insert (dbt's @@ -431,6 +514,11 @@ impl<'a> MaterializeCodegen<'a> { // same write shape as `replace`, which is reliable. The DELETE is // scoped to the current partition when partitioned so it stays // slice-local (a key present in another partition is untouched). + // + // Guard first: the DELETE+INSERT does not dedup the source, so + // two incoming rows sharing a key would both persist under it. + // Raise instead of silently double-writing (see the helper). + out.push(duplicate_source_key_guard_sql(sel, unique_key)); let scope = if self.partitioned { format!("{pcol} = {pval} AND ") } else { @@ -439,12 +527,268 @@ impl<'a> MaterializeCodegen<'a> { out.push(format!( "DELETE FROM {t} WHERE {scope}{unique_key} IN (SELECT {unique_key} FROM ({sel}));" )); - out.push(format!("INSERT INTO {t} {source};")); + out.push(format!("INSERT INTO {t}{by_name} {source};")); } + // Handled by the early return above (scd2 has no partitioned form). + MaterializeStrategy::Scd2 { .. } => unreachable!("scd2 handled before this match"), } out.push("COMMIT;".to_string()); out } + + /// SCD2 codegen: the incoming SELECT is the *current desired snapshot* (one + /// row per `key`); we diff it against the live current rows, close the prior + /// version of every changed/new key, and open a fresh one — so history is + /// kept. `track` empty ⇒ every non-key column is tracked for change + /// detection. + /// + /// Shape (all one transaction for the mutation, mirroring the other + /// strategies so a partial failure leaves the prior snapshot intact): + /// 1. bootstrap the table (business columns + `valid_from/valid_to/ + /// is_current`), idempotent; + /// 2. capture changed+new keys into a connection-local temp table *before* + /// the write — the close below flips `is_current`, so recomputing the + /// diff after it would see a different set; + /// 3. close the prior open version of those keys (`UPDATE` — not `MERGE`: + /// DuckLake's MERGE is the unreliable path, plain UPDATE works); + /// 3b. when `close_deleted`, also capture the keys that vanished from the + /// snapshot and close their current version (no reopen) — dbt's + /// `hard_deletes=close`; + /// 4. open a new current version from the snapshot; + /// 5. create the `_current` convenience view once (`IF NOT EXISTS`), + /// inside the same transaction so it doesn't advance the DuckLake snapshot + /// past the data write the summary records (and so an unchanged rerun, + /// whose UPDATE/INSERT touch no rows, stays a true no-op). + /// + /// Close/open match keys with `IS NOT DISTINCT FROM` (via a correlated + /// `EXISTS`), not `key IN (…)`: SQL `IN` never matches `NULL`, so a `NULL` + /// natural key would be flagged as changed yet silently skipped by both the + /// close and the open, dropping the row. Null-safe matching materializes it + /// instead (a `NULL` key is still ill-formed for a dimension — guard it with + /// `// data_test not_null ` — but it must not vanish). + /// + /// Without `close_deleted`, keys present in the table but absent from the + /// SELECT are left current (soft delete — dbt's `hard_deletes=ignore` default; + /// with `close_deleted` they are closed instead — see step 3b). The effective + /// timestamp is `now()`, which DuckDB fixes to + /// the transaction start, so `valid_from`/`valid_to` are consistent within a + /// run without a nondeterministic per-statement clock. + /// + /// Reserved columns: `valid_from`/`valid_to`/`is_current` are appended to the + /// user's SELECT with these fixed names (kept clean so consumers write + /// `WHERE is_current` / `ASOF JOIN … >= valid_from`). A SELECT that already + /// projects one of them is a v1 constraint violation — the bootstrap then + /// produces a duplicate output column and the run fails at execution + /// (documented; not statically checkable here since the SELECT's columns + /// aren't known at codegen time). + fn scd2_statements(&self, key: &str, track: &[String], close_deleted: bool) -> Vec { + let t = self.target_qualified; + let sel = self.select_sql; + let k = quote_ident(key); + let vf = SCD2_VALID_FROM; + let vt = SCD2_VALID_TO; + let ic = SCD2_IS_CURRENT; + let changed = SCD2_CHANGED_KEYS; + let deleted = SCD2_DELETED_KEYS; + // Transaction-stable effective timestamp (see doc above). Cast to plain + // TIMESTAMP so it matches the bootstrapped column type (now() is TZ-aware). + let ts = "CAST(now() AS TIMESTAMP)"; + + // Projection compared to detect change. Empty `track` ⇒ all business + // columns via `* EXCLUDE ()` on the table side (which carries + // the extra metadata columns) and `*` on the snapshot side. An explicit + // `track` ⇒ key + those columns on both sides. `EXCEPT` treats NULLs as + // equal, so an unchanged NULL is not read as a change. + let (src_proj, tgt_proj) = if track.is_empty() { + ( + format!("SELECT * FROM ({sel})"), + format!("SELECT * EXCLUDE ({vf}, {vt}, {ic}) FROM {t} WHERE {ic}"), + ) + } else { + let cols = std::iter::once(key) + .chain(track.iter().map(String::as_str)) + .map(quote_ident) + .collect::>() + .join(", "); + ( + format!("SELECT {cols} FROM ({sel})"), + format!("SELECT {cols} FROM {t} WHERE {ic}"), + ) + }; + + let mut out = vec![ + format!( + "CREATE TABLE IF NOT EXISTS {t} AS SELECT *, \ + CAST(NULL AS TIMESTAMP) AS {vf}, \ + CAST(NULL AS TIMESTAMP) AS {vt}, \ + CAST(NULL AS BOOLEAN) AS {ic} FROM ({sel}) WHERE false;" + ), + format!( + "CREATE OR REPLACE TEMP TABLE {changed} AS \ + SELECT {k} FROM ({src_proj} EXCEPT {tgt_proj});" + ), + ]; + // Hard-delete-close (`deletes=close`): the keys that vanished from the + // snapshot — present-and-current in the table, absent from the SELECT. + // Captured before the close (like `changed`) and disjoint from it (a + // key is either in the snapshot or not), so the two closes never overlap. + if close_deleted { + out.push(format!( + "CREATE OR REPLACE TEMP TABLE {deleted} AS \ + SELECT {k} FROM (SELECT {k} FROM {t} WHERE {ic} EXCEPT SELECT {k} FROM ({sel}));" + )); + } + out.push("BEGIN TRANSACTION;".to_string()); + out.push(format!( + "UPDATE {t} SET {vt} = {ts}, {ic} = false \ + WHERE {ic} AND EXISTS (SELECT 1 FROM {changed} \ + WHERE {changed}.{k} IS NOT DISTINCT FROM {t}.{k});" + )); + // Close vanished keys — no matching INSERT below, so they close without + // reopening. A key that later reappears isn't in `WHERE is_current`, so the + // `changed` diff treats it as new and opens a fresh version (a validity gap + // between the delete and the reactivation — correct SCD2). + if close_deleted { + out.push(format!( + "UPDATE {t} SET {vt} = {ts}, {ic} = false \ + WHERE {ic} AND EXISTS (SELECT 1 FROM {deleted} \ + WHERE {deleted}.{k} IS NOT DISTINCT FROM {t}.{k});" + )); + } + out.push(format!( + "INSERT INTO {t} SELECT s.*, {ts} AS {vf}, CAST(NULL AS TIMESTAMP) AS {vt}, \ + true AS {ic} FROM ({sel}) s WHERE EXISTS (SELECT 1 FROM {changed} c \ + WHERE c.{k} IS NOT DISTINCT FROM s.{k});" + )); + out.push( + // Consumer convenience: a `_current` view (the live slice) so the + // common "just the latest version" read needs no `WHERE is_current`, + // and downstream scripts can `// on` / read it directly. For the + // effective-dated payoff, consumers `ASOF JOIN ON fact.key = + // dim. AND fact.ts >= dim.valid_from`. + // + // `CREATE VIEW IF NOT EXISTS` (not `OR REPLACE`), created inside the + // write transaction, on purpose: the view definition never changes + // (`SELECT * WHERE is_current` always reflects live data), and a + // catalog write advances the DuckLake snapshot — so `OR REPLACE` on + // every run would (a) advance the snapshot on an otherwise no-op + // unchanged run and (b) make the summary's `max(snapshot_id)` record + // the view DDL instead of the data write. `IF NOT EXISTS` creates it + // once (folded into the first data-write snapshot) and is a true no-op + // afterwards. The `_current` name is reserved: if a real table by + // that name already exists, `IF NOT EXISTS` skips silently (no view, + // no error) — documented as a reserved suffix. + format!("CREATE VIEW IF NOT EXISTS {t}_current AS SELECT * FROM {t} WHERE {ic};"), + ); + out.push("COMMIT;".to_string()); + out + } +} + +// --------------------------------------------------------------------------- +// on_schema_change drift detection (write-time guardrail) +// --------------------------------------------------------------------------- +// +// Drift is computed entirely in SQL against the live DuckDB session: the +// SELECT's output columns come from `DESCRIBE`, the table's from `DESCRIBE` of +// the target (the managed `_wm_partition` column is excluded so it is compared +// as the producer's logical output, matching schema capture). `added` = SELECT +// columns absent from the table, `removed` = table columns absent from the +// SELECT. When the table was just created this run (first materialize) the two +// DESCRIBEs agree, so both lists are empty and no guard fires. + +/// A `DESCRIBE`-derived set of column names of `rel_sql` (any SELECT-able +/// relation, already parenthesized/qualified by the caller), optionally +/// dropping the managed partition column. +fn describe_col_names(rel_sql: &str, exclude_col: Option<&str>) -> String { + let filter = match exclude_col { + Some(c) => format!(" WHERE column_name <> {}", quote_lit(c)), + None => String::new(), + }; + format!("SELECT column_name FROM (DESCRIBE SELECT * FROM {rel_sql}){filter}") +} + +/// Scalar subqueries `(added, removed)` — the list of column names in the +/// SELECT but not the table, and vice versa. Each is a `list(...)` over an +/// `EXCEPT`; empty ⇒ `list()` yields an empty list (`len` 0). The partition +/// column is excluded on the table side only. +/// +/// Set difference, not ordered: this catches the add/remove/rename that a +/// positional INSERT misaligns on, but by design NOT a pure reorder of +/// same-named columns (identical sets ⇒ empty added/removed). See the +/// `OnSchemaChange` doc in asset_parser.rs — reorder-safety is `sync`'s job +/// (INSERT BY NAME); the set difference is deliberately kept over an ordered +/// comparison so the `fail` guard cannot false-positive on a correctly-aligned +/// write. +fn drift_lists(sel_sql: &str, target_qualified: &str, partition_col: &str) -> (String, String) { + let sel = describe_col_names(&format!("({sel_sql})"), None); + let tbl = describe_col_names(target_qualified, Some(partition_col)); + let added = format!("(SELECT list(column_name) FROM (({sel}) EXCEPT ({tbl})))"); + let removed = format!("(SELECT list(column_name) FROM (({tbl}) EXCEPT ({sel})))"); + (added, removed) +} + +/// The `fail`-mode guard: a single statement that raises via DuckDB `error(...)` +/// when the SELECT's columns diverge from the table's, naming the added/removed +/// columns and the target. Both `CASE` branches are cast to VARCHAR so the +/// planner cannot constant-fold the `error(...)` away, and the condition depends +/// on the runtime drift subqueries so it is never folded to a constant. +fn schema_drift_guard_sql(sel_sql: &str, target_qualified: &str, partition_col: &str) -> String { + let (added, removed) = drift_lists(sel_sql, target_qualified, partition_col); + // `target_qualified` is safe in table-reference position (quoted identifiers) + // but here it lands inside a SQL string literal, so single-quotes must be + // doubled (same as `quote_lit`). + let tq = target_qualified.replace('\'', "''"); + format!( + "SELECT CASE WHEN coalesce(len(_wm_added), 0) + coalesce(len(_wm_removed), 0) > 0 \ + THEN CAST(error('managed materialize: on_schema_change=fail blocked a schema-drifted \ + write to {tq} — added column(s): [' || coalesce(array_to_string(_wm_added, ', '), '') || \ + '], removed column(s): [' || coalesce(array_to_string(_wm_removed, ', '), '') || \ + ']. The table schema is fixed at first create; set on_schema_change=sync to auto-migrate, \ + or align the SELECT with the table.') AS VARCHAR) ELSE 'ok' END \ + FROM (SELECT {added} AS _wm_added, {removed} AS _wm_removed);", + tq = tq, + ) +} + +/// In-transaction guard for the keyed `merge` strategy: raises via DuckDB +/// `error(...)` when the source SELECT holds more than one row for the same +/// non-NULL `unique_key`. A keyed merge is delete-by-key + insert-all (it does +/// NOT deduplicate the source), so duplicate source keys would land every +/// duplicate row under one key — the exact silent double-write this guards +/// against. Erroring keeps the semantics explicit: the author must deduplicate +/// in the SELECT (or use `append`). NULL keys are excluded to match the delete's +/// `key IN (...)` scope, which never matches NULL. `unique_key` is embedded raw +/// in identifier position (matching the merge's own DELETE/IN) and doubled-quote +/// escaped where it lands inside the error string literal. +fn duplicate_source_key_guard_sql(sel_sql: &str, unique_key: &str) -> String { + let key_lit = unique_key.replace('\'', "''"); + format!( + "SELECT CASE WHEN _wm_dup_keys > 0 THEN CAST(error('managed materialize: keyed merge on \ + `{key_lit}` blocked — the source has ' || _wm_dup_keys || ' key value(s) with more than \ + one row. A keyed merge keeps one row per key and does not deduplicate; deduplicate in the \ + SELECT (e.g. QUALIFY row_number() OVER (PARTITION BY {key_lit} ORDER BY …) = 1) or use \ + `append`.') AS VARCHAR) ELSE 'ok' END FROM (SELECT count(*) AS _wm_dup_keys FROM (SELECT \ + {unique_key} FROM ({sel_sql}) WHERE {unique_key} IS NOT NULL GROUP BY {unique_key} HAVING \ + count(*) > 1));" + ) +} + +/// The `warn`-mode summary column: a `schema_drift` struct `{added, removed}` +/// when the SELECT drifted from the table, else NULL. Appended to the +/// materialize summary row so the executor can log it and fold it into the job +/// result without an extra round-trip. +fn schema_drift_summary_field( + sel_sql: &str, + target_qualified: &str, + partition_col: &str, +) -> String { + let (added, removed) = drift_lists(sel_sql, target_qualified, partition_col); + format!( + "(SELECT CASE WHEN coalesce(len(_wm_added), 0) + coalesce(len(_wm_removed), 0) > 0 \ + THEN {{'added': _wm_added, 'removed': _wm_removed}} END \ + FROM (SELECT {added} AS _wm_added, {removed} AS _wm_removed)) AS schema_drift" + ) } /// The read that captures the DuckLake snapshot id produced by the write, for @@ -460,6 +804,44 @@ pub fn snapshot_capture_sql(alias: &str) -> String { /// from the target ducklake's config and passes it in as `target_attach`. pub const TARGET_ALIAS: &str = "_wm_target"; +/// Structural role of one statement in a [`MaterializePlan`]. The public build +/// executes the plan verbatim, so the kinds are pure metadata there; they exist +/// so a downstream assembler (`pipeline_advanced::finalize_materialize_query`) +/// can reason about the plan without parsing SQL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MaterializeStmtKind { + /// Pre-write statement: user setup, the target ATTACH, referenced-asset + /// ATTACHes. + Setup, + /// Write work against the target: bootstrap DDL, SCD2 temp-table captures, + /// the mutation itself, the `_current` view. + Write, + /// The `BEGIN TRANSACTION;` marker emitted by the strategy codegen. + TxnBegin, + /// The `COMMIT;` marker emitted by the strategy codegen. + TxnCommit, + /// The trailing one-row summary read (asset / rows / snapshot_id / + /// data_tests breakdown). + Summary, +} + +/// One planned statement: its structural role and the SQL text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializeStmt { + pub kind: MaterializeStmtKind, + pub sql: String, +} + +/// The full ordered materialization plan [`build_wrap_blocks`] produces: +/// statements in execution order plus the compiled data-test checks (also +/// embedded in the summary statement's breakdown). Assembled into the final +/// statement list by `pipeline_advanced::finalize_materialize_query`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializePlan { + pub stmts: Vec, + pub checks: Vec, +} + /// Assemble the full ordered statement list the DuckDB executor runs for a /// managed `// materialize` script. This is the single entry point the worker /// calls; it composes the already-tested pieces (classifier split → target @@ -473,6 +855,14 @@ pub const TARGET_ALIAS: &str = "_wm_target"; /// the full `/
` for the result summary. The trailing statement is /// a one-row summary read (asset / rows / snapshot_id) that is both the job's /// result (a useful preview) and what the worker records. +/// +/// Returns a [`MaterializePlan`] — the statements plus their structural role +/// and the compiled data-test checks — rather than raw SQL: the executor hands +/// the plan to `windmill_common::pipeline_advanced::finalize_materialize_query` +/// (which this crate cannot depend on), whose public-build implementation +/// assembles the statements verbatim. Everything this function produces runs +/// as-is on the public build; the plan's structure is metadata about it, not a +/// second mode. pub fn build_wrap_blocks( plan: &WrapPlan, target_attach: &str, @@ -482,9 +872,20 @@ pub fn build_wrap_blocks( partition_value_sql: &str, partitioned: bool, strategy: MaterializeStrategy, + on_schema_change: OnSchemaChange, tests: &[DataTestResolved], -) -> Result, String> { +) -> Result { let target_qualified = format!("{TARGET_ALIAS}.{target_table}"); + let scd2 = matches!(strategy, MaterializeStrategy::Scd2 { .. }); + let ctx = DataTestCtx { + target_qualified: &target_qualified, + asset_path, + partition_col, + partition_value_sql, + partitioned, + scd2, + }; + let test_sql = build_data_test_checks(tests, &ctx)?; let cg = MaterializeCodegen { target_qualified: &target_qualified, select_sql: &plan.output, @@ -492,36 +893,53 @@ pub fn build_wrap_blocks( partition_value_sql, partitioned, strategy, + on_schema_change, }; - let ctx = DataTestCtx { - target_qualified: &target_qualified, - asset_path, - partition_col, - partition_value_sql, - partitioned, - }; - let test_sql = build_data_test_checks(tests, &ctx)?; - let mut blocks: Vec = Vec::new(); + // `warn` folds the post-write drift into the summary row (executor logs it + + // returns it) — only for the positional persist-and-mutate path, and only in + // `warn`: `fail`/`sync` guard the write itself, `ignore` is silent. + let drift_summary_select = + if on_schema_change == OnSchemaChange::Warn && cg.is_persist_and_mutate() { + Some(plan.output.as_str()) + } else { + None + }; + let mut stmts: Vec = Vec::new(); + let setup = |sql: String| MaterializeStmt { kind: MaterializeStmtKind::Setup, sql }; // Setup blocks come from the splitter with their `;` stripped — re-terminate // each so that when the executor re-joins and re-splits the assembled query, // adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH) // don't merge into one malformed statement. - blocks.extend(plan.setup.iter().map(|s| terminate(s))); - blocks.push(target_attach.to_string()); + stmts.extend(plan.setup.iter().map(|s| setup(terminate(s)))); + stmts.push(setup(target_attach.to_string())); // Referenced-asset ATTACHes (relationships tests) — read-only, before the // write and the summary that probes them. - blocks.extend(test_sql.attaches); - blocks.extend(cg.statements()); + stmts.extend(test_sql.attaches.into_iter().map(setup)); + // Classify the codegen statements by matching the exact transaction-marker + // literals this module emits (`BEGIN TRANSACTION;` / `COMMIT;`); everything + // else the codegen produces is write work. + stmts.extend(cg.statements().into_iter().map(|sql| { + let kind = match sql.as_str() { + "BEGIN TRANSACTION;" => MaterializeStmtKind::TxnBegin, + "COMMIT;" => MaterializeStmtKind::TxnCommit, + _ => MaterializeStmtKind::Write, + }; + MaterializeStmt { kind, sql } + })); // The summary read carries the per-test breakdown (when any tests apply). - blocks.push(materialize_result_sql( - &target_qualified, - asset_path, - partition_col, - partition_value_sql, - partitioned, - &test_sql.checks, - )); - Ok(blocks) + stmts.push(MaterializeStmt { + kind: MaterializeStmtKind::Summary, + sql: materialize_result_sql( + &target_qualified, + asset_path, + partition_col, + partition_value_sql, + partitioned, + &test_sql.checks, + drift_summary_select, + ), + }); + Ok(MaterializePlan { stmts, checks: test_sql.checks }) } /// The trailing one-row summary the materialize run returns: the asset it @@ -535,6 +953,10 @@ pub fn materialize_result_sql( partition_value_sql: &str, partitioned: bool, checks: &[DataTestCheck], + // `on_schema_change=warn` on a persist-and-mutate strategy: the SELECT to + // diff against the (post-write) table for the `schema_drift` summary column. + // `None` ⇒ no drift column (every other mode / strategy). + drift_summary_select: Option<&str>, ) -> String { let (count_expr, partition_sel) = if partitioned { // Row count is the slice this run wrote (the partition); `partition` @@ -581,25 +1003,47 @@ pub fn materialize_result_sql( FROM (SELECT column_name, column_type, row_number() OVER () AS _wm_ord \ FROM (DESCRIBE SELECT * FROM {target_qualified}){partition_filter})) AS output_schema" ); + // `on_schema_change=warn`: fold the drift `{added, removed}` (or NULL) into + // the same row so the executor logs it + returns it with no extra probe. + let drift_col = match drift_summary_select { + Some(sel) => format!( + ", {}", + schema_drift_summary_field(sel, target_qualified, partition_col) + ), + None => String::new(), + }; let base_cols = format!( "'ducklake://{asset_path}' AS materialized, \ {partition_sel}{count_expr} AS rows, \ (SELECT max(snapshot_id) FROM ducklake_snapshots('{TARGET_ALIAS}')) AS snapshot_id, \ - {schema_capture}" + {schema_capture}{drift_col}" ); if checks.is_empty() { return format!("SELECT {base_cols};"); } - // Per-test breakdown. Each check's violating-count is computed once as a CTE - // column (`c0`, `c1`, …); the `data_tests` list-of-struct then references - // those columns — DuckDB rejects scalar subqueries *inside* a struct/list - // literal, hence the CTE. Names are single-quote-escaped. The result row - // carries the whole breakdown so the worker runs every test (no - // abort-on-first) and decides pass/fail itself. - let cte_cols = checks + // Per-test breakdown. Each check's one-row probe `(v, s)` becomes a CTE + // (`_wm_t0`, `_wm_t1`, …); `_wm_tr` cross-joins them (all one-row, so the + // join stays one row) and the `data_tests` list-of-struct references the + // flattened columns — DuckDB rejects scalar subqueries *inside* a + // struct/list literal, hence the CTE lift. Names are single-quote-escaped. + // The result row carries the whole breakdown so the worker runs every + // test (no abort-on-first) and decides pass/fail itself. + let probe_ctes = checks .iter() .enumerate() - .map(|(i, c)| format!("{} AS c{i}", c.violating)) + .map(|(i, c)| format!("_wm_t{i} AS ({})", c.probe)) + .collect::>() + .join(", "); + let tr_cols = checks + .iter() + .enumerate() + .map(|(i, _)| format!("_wm_t{i}.v AS c{i}, _wm_t{i}.s AS s{i}")) + .collect::>() + .join(", "); + let tr_from = checks + .iter() + .enumerate() + .map(|(i, _)| format!("_wm_t{i}")) .collect::>() .join(", "); let list_items = checks @@ -607,17 +1051,40 @@ pub fn materialize_result_sql( .enumerate() .map(|(i, c)| { let name = c.name.replace('\'', "''"); - format!("{{'test': '{name}', 'violating': c{i}}}") + format!("{{'test': '{name}', 'violating': c{i}, 'sample': s{i}}}") }) .collect::>() .join(", "); format!( - "WITH _wm_tr AS (SELECT {cte_cols}) \ + "WITH {probe_ctes}, _wm_tr AS (SELECT {tr_cols} FROM {tr_from}) \ SELECT {base_cols}, [{list_items}] AS data_tests FROM _wm_tr;" ) } // Ensure a statement ends with a single `;`. +/// Convenience macro injected as the first setup statement of a partitioned +/// materialize: `wm_partition(ts)` renders a timestamp with the SAME identity +/// format the resolver used for `{partition}` (from +/// [`PartitionSpec::time_strftime_format`]). It lets a partitioned SELECT +/// filter to the active slice with a single grain-agnostic line — +/// `WHERE wm_partition() = {partition}` — so users never hand-write a +/// `strftime` format that can drift, nor reach for `= TIMESTAMP {partition}` +/// (which only parses for daily). `None` for `dynamic` (no wall-clock bucket). +/// +/// Timezone-agnostic by construction: it formats `ts` as given, matching the +/// prior documented `strftime(ts, fmt)` idiom. When a non-UTC `tz=` is set the +/// caller is responsible for expressing `ts` in that zone (same caveat the raw +/// idiom carried), so this doesn't silently reinterpret a column's instant. +pub fn wm_partition_macro(spec: &crate::asset_parser::PartitionSpec) -> Option { + let fmt = spec.time_strftime_format()?; + // fmt is a trusted per-grain constant or the author's `format=`; escape + // single quotes defensively so the emitted literal can't break out. + Some(format!( + "CREATE OR REPLACE TEMP MACRO wm_partition(ts) AS strftime(ts, '{}')", + fmt.replace('\'', "''") + )) +} + fn terminate(stmt: &str) -> String { let t = stmt.trim_end(); if t.ends_with(';') { @@ -633,20 +1100,21 @@ fn terminate(stmt: &str) -> String { // // A data test is the FIRST extensible annotation: the parser yields a // `DataTest` from a known vocabulary, and this module turns each into a -// *check* — a `(name, violating-row-count query)` pair — that runs against the -// freshly-materialized target after the write commits. The materialize summary -// query embeds every check's count in one `data_tests` column, so all tests -// run in a single pass (no abort-on-first) and the worker, not the SQL, -// decides pass/fail and reports the full per-test breakdown. +// *check* — a `(name, probe)` pair whose probe counts and samples the +// violating rows — that runs against the freshly-materialized target after +// the write commits. The materialize summary query embeds every check's +// outcome in one `data_tests` column, so all tests run in a single pass (no +// abort-on-first) and the worker, not the SQL, decides pass/fail and reports +// the full per-test breakdown. // -// The pattern is deliberately open: a verifier is just `(name, count query)`. -// Built-ins differ only in their count query; the `Custom` escape hatch -// supplies its own (a user SELECT returning the violating rows). A sibling -// annotation family (column-lineage) can emit its own checks through the same -// `push_check` shape rather than bolting on a parallel mechanism. See -// `docs/ducklake-materialization.md`. +// The pattern is deliberately open: a verifier is just `(name, violating-rows +// query)` handed to `push_check`. Built-ins differ only in their rows query; +// the `Custom` escape hatch supplies its own (a user SELECT returning the +// violating rows). A sibling annotation family (column-lineage) can emit its +// own checks through the same `push_check` shape rather than bolting on a +// parallel mechanism. See `docs/ducklake-materialization.md`. -use crate::asset_parser::{AssetKind, DataTest}; +use crate::asset_parser::{AssetKind, DataTest, OnSchemaChange}; /// Target context a data-test probe runs against — the materialized table and /// the partition slice (when partitioned, tests are scoped to the slice just @@ -663,6 +1131,11 @@ pub struct DataTestCtx<'a> { pub partition_value_sql: &'a str, /// Whether the target is partitioned (scopes probes to the slice). pub partitioned: bool, + /// Whether the target is an SCD2 history table. Built-in probes then assert + /// the *current snapshot* (`is_current` rows): the history legitimately + /// repeats the natural key across closed versions, so an unscoped + /// `unique()` would fail on the second change of any key. + pub scd2: bool, } /// A data test resolved enough to generate SQL. Built-ins carry only their @@ -674,19 +1147,31 @@ pub enum DataTestResolved { Custom { path: String, body: String }, } -/// One compiled data-test check: a human-readable `name` and a scalar SQL -/// expression (`violating`) yielding the number of rows that violate it (0 = -/// pass). The materialize summary query embeds every check's count so the -/// worker gets the whole breakdown in one result — all tests run (no -/// abort-on-first) and the worker, not the SQL, decides pass/fail. +/// One compiled data-test check: a human-readable `name` and a one-row probe +/// query yielding `(v, s)` — the violating-row count (0 = pass) and a bounded +/// `to_json` sample of the violating rows as a VARCHAR (NULL when there are +/// none, or when the serialized sample exceeds the size cap). The materialize +/// summary query embeds every check's outcome so the worker gets the whole +/// breakdown in one result — all tests run (no abort-on-first) and the +/// worker, not the SQL, decides pass/fail. The sample is decoration only: +/// enforcement reads `v`, never `s`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DataTestCheck { pub name: String, - /// Scalar subquery yielding the violating-row count, e.g. - /// `(SELECT count(*) AS v FROM (…))`. - pub violating: String, + /// One-row subquery `SELECT … AS v, … AS s FROM (…)` counting and + /// sampling the check's violating rows in a single scan. + pub probe: String, } +/// Row cap on a data-test sample. Bounded so the sample stays a debugging aid +/// (the full count is still reported); no ORDER BY on the violating rows, so +/// which rows land in the sample is nondeterministic. +const SAMPLE_MAX_ROWS: usize = 20; +/// Byte cap on one serialized sample. Oversized samples are dropped entirely +/// (NULL), never truncated — truncated JSON would fail parsing downstream +/// after paying the bytes anyway. +const SAMPLE_MAX_BYTES: usize = 51_200; + /// The SQL a set of data tests compiles to: referenced-asset `ATTACH` /// statements (resolved by the executor's ATTACH-transform pass) and the /// per-test checks, both in declaration order. @@ -723,24 +1208,89 @@ fn quote_lit(s: &str) -> String { format!("'{}'", s.replace('\'', "''")) } -// The `WHERE`/`AND` fragment scoping a probe to the current partition, or -// empty when unpartitioned. `prefix` is `WHERE ` or `AND ` per call site. +// The `WHERE`/`AND` fragment scoping a probe to the current partition and/or +// the SCD2 current snapshot, or empty when neither applies. `prefix` is +// `WHERE ` or `AND ` per call site; further conditions chain with `AND`. +// (`partitioned` and `scd2` are mutually exclusive today — the combo is +// rejected at codegen — but the chaining keeps this correct if that changes.) fn partition_scope(ctx: &DataTestCtx, prefix: &str, table_alias: Option<&str>) -> String { - if !ctx.partitioned { + let qualify = |col: String| match table_alias { + Some(a) => format!("{a}.{col}"), + None => col, + }; + let mut conds: Vec = Vec::new(); + if ctx.partitioned { + conds.push(format!( + "{} = {}", + qualify(quote_ident(ctx.partition_col)), + ctx.partition_value_sql + )); + } + if ctx.scd2 { + conds.push(qualify(SCD2_IS_CURRENT.to_string())); + } + if conds.is_empty() { return String::new(); } - let col = match table_alias { - Some(a) => format!("{a}.{}", quote_ident(ctx.partition_col)), - None => quote_ident(ctx.partition_col), - }; - format!("{prefix}{col} = {}", ctx.partition_value_sql) + format!("{prefix}{}", conds.join(" AND ")) } -// Record one check: its display `name` plus `count_query` (which yields a -// single-column violating-row count) wrapped as a scalar subquery. -fn push_check(out: &mut DataTestChecks, name: String, count_query: String) { - out.checks - .push(DataTestCheck { name, violating: format!("({count_query})") }); +// Record one check: its display `name` plus `rows_query`, the SELECT of its +// violating rows. The probe counts and samples those rows in one scan: +// `_wm_v` (the subquery alias) referenced as a column is the whole row as a +// struct; `to_json(...)::VARCHAR` keeps the sample a JSON *string* through +// the FFI — expanded rows would be visible to the executor's key-recursive +// `extract_i64(result, "rows"/"snapshot_id")` scans, which a user column of +// the same name could corrupt. `list()` over zero rows and an over-cap +// sample both degrade to NULL (`s` is optional by contract). +fn push_check(out: &mut DataTestChecks, name: String, rows_query: String) { + // `strlen` counts bytes (unlike `length`, characters) — the cap bounds + // payload size on the wire, so bytes are the right unit. + let probe = format!( + "SELECT v, CASE WHEN strlen(s_raw) <= {SAMPLE_MAX_BYTES} THEN s_raw END AS s \ + FROM (SELECT count(*) AS v, \ + to_json(list(_wm_v ORDER BY _wm_rn) FILTER (WHERE _wm_rn <= {SAMPLE_MAX_ROWS}))::VARCHAR AS s_raw \ + FROM (SELECT _wm_v, row_number() OVER () AS _wm_rn FROM ({rows_query}) _wm_v))" + ); + out.checks.push(DataTestCheck { name, probe }); +} + +// `SELECT *` for a sample rows-query, excluding the synthetic physical +// partition column on partitioned targets — it's Windmill's storage detail, +// not part of the producer's logical output (same rule as schema capture). +// `qualifier` scopes the star when the query aliases the target (`_wm_src`). +fn sample_star(ctx: &DataTestCtx, qualifier: Option<&str>) -> String { + let star = match qualifier { + Some(q) => format!("{q}.*"), + None => "*".to_string(), + }; + if ctx.partitioned { + format!("{star} EXCLUDE ({})", quote_ident(ctx.partition_col)) + } else { + star + } +} + +// Self-teaching tail appended to every malformed-custom-test error. It states +// the two rules that aren't documented or scaffolded anywhere else — the body +// is a single SELECT, and it reads the freshly-materialized target through the +// internal `_wm_target.
` alias — and doubles that alias into a copyable +// one-line example. `target_qualified` is already `_wm_target.
`. +fn custom_test_hint(target_qualified: &str) -> String { + format!( + "Write a single SELECT against `{target_qualified}` returning the offending rows, e.g. \ + `SELECT * FROM {target_qualified} WHERE ` — an empty result means the test \ + passes." + ) +} + +// Whether a custom-test statement reads the materialized target through the +// reserved `_wm_target` alias (the only handle the runtime attaches it under). +// SQL identifiers are case-insensitive, so match case-insensitively; +// `split_statements` has already stripped comments, so a match here is a real +// reference, not one buried in a comment. `TARGET_ALIAS` is lowercase. +fn references_target(stmt: &str) -> bool { + stmt.to_lowercase().contains(TARGET_ALIAS) } /// Compile resolved data tests into ATTACH statements + per-test checks for @@ -762,28 +1312,34 @@ pub fn build_data_test_checks( DataTestResolved::BuiltIn(DataTest::Unique { column }) => { let c = quote_ident(column); let scope = partition_scope(ctx, " AND ", None); + // The rows are the GROUP BY result — one `{value, count}` per + // duplicated key — so the count (number of duplicated values, + // not of rows) and the sample share one grain and can't + // contradict each other in the UI. let q = format!( - "SELECT count(*) AS v FROM (SELECT {c} FROM {t} WHERE {c} IS NOT NULL{scope} \ - GROUP BY {c} HAVING count(*) > 1)" + "SELECT {c} AS \"value\", count(*) AS \"count\" FROM {t} \ + WHERE {c} IS NOT NULL{scope} GROUP BY {c} HAVING count(*) > 1" ); push_check(&mut out, format!("unique({column})"), q); } DataTestResolved::BuiltIn(DataTest::NotNull { column }) => { let c = quote_ident(column); let scope = partition_scope(ctx, " AND ", None); - let q = format!("SELECT count(*) AS v FROM {t} WHERE {c} IS NULL{scope}"); + let star = sample_star(ctx, None); + let q = format!("SELECT {star} FROM {t} WHERE {c} IS NULL{scope}"); push_check(&mut out, format!("not_null({column})"), q); } DataTestResolved::BuiltIn(DataTest::AcceptedValues { column, values }) => { let c = quote_ident(column); let scope = partition_scope(ctx, " AND ", None); + let star = sample_star(ctx, None); let list = values .iter() .map(|v| quote_lit(v)) .collect::>() .join(", "); let q = format!( - "SELECT count(*) AS v FROM {t} WHERE {c} IS NOT NULL AND {c} NOT IN ({list}){scope}" + "SELECT {star} FROM {t} WHERE {c} IS NOT NULL AND {c} NOT IN ({list}){scope}" ); push_check(&mut out, format!("accepted_values({column})"), q); } @@ -847,8 +1403,9 @@ pub fn build_data_test_checks( // segment so the dot stays a schema separator, not a literal. let rt = quote_qualified(ref_table); let scope = partition_scope(ctx, " AND ", Some("_wm_src")); + let star = sample_star(ctx, Some("_wm_src")); let q = format!( - "SELECT count(*) AS v FROM {t} _wm_src \ + "SELECT {star} FROM {t} _wm_src \ WHERE _wm_src.{c} IS NOT NULL{scope} \ AND NOT EXISTS (SELECT 1 FROM {alias}.{rt} _wm_ref \ WHERE _wm_ref.{rc} = _wm_src.{c})" @@ -867,25 +1424,44 @@ pub fn build_data_test_checks( } DataTestResolved::Custom { path, body } => { // dbt singular-test convention: the body is a *single* SELECT - // (or CTE) returning the violating rows. It is embedded as a - // subquery (`FROM ()`), so a multi-statement body would - // produce invalid SQL — validate up front with an actionable - // error. It runs in the target's connection (can read - // `_wm_target` + the user's attaches); partition substitution is - // already applied by the worker. + // (or CTE) returning the violating rows, reading the + // freshly-materialized target through the internal `_wm_target` + // schema. It is embedded as a subquery (`FROM ()`), so a + // multi-statement or non-SELECT body would produce invalid SQL. + // Neither rule is documented or scaffolded elsewhere, so the + // errors are self-teaching: they name the exact violation and + // append a correct one-line example. It runs in the target's + // connection (can read `_wm_target` + the user's attaches); + // partition substitution is already applied by the worker. + let hint = custom_test_hint(t); let stmts = split_statements(body); if stmts.is_empty() { - return Err(format!("data_test custom `{path}`: empty test body")); + return Err(format!( + "data_test custom `{path}`: empty test body. {hint}" + )); } if stmts.len() > 1 { return Err(format!( - "data_test custom `{path}`: must be a single SELECT returning the \ - violating rows (found {} statements)", + "data_test custom `{path}`: a custom data test must be a single SELECT, \ + but found {} statements. {hint}", stmts.len() )); } - let q = format!("SELECT count(*) AS v FROM ({})", stmts[0]); - push_check(&mut out, format!("custom({path})"), q); + let stmt = &stmts[0]; + if classify_block(stmt) != BlockClass::Output { + return Err(format!( + "data_test custom `{path}`: a custom data test must be a single SELECT, \ + not a write or DDL statement. {hint}" + )); + } + if !references_target(stmt) { + return Err(format!( + "data_test custom `{path}`: the test never reads the freshly-materialized \ + target — reference it through the internal `{TARGET_ALIAS}` schema (as \ + `{t}`), not the table name on its own. {hint}" + )); + } + push_check(&mut out, format!("custom({path})"), stmt.to_string()); } } } @@ -903,6 +1479,55 @@ mod tests { classify_wrap(sql).expect_err("expected wrap-ineligible") } + use crate::asset_parser::{PartitionKind, PartitionSpec}; + + fn pspec(kind: PartitionKind, format: Option<&str>) -> PartitionSpec { + PartitionSpec { kind, tz: None, format: format.map(String::from), start: None } + } + + #[test] + fn wm_partition_macro_uses_grain_identity_format() { + // Every time grain's macro must strftime with the exact format the + // resolver stamps the identity with — otherwise the equality filter + // silently returns no rows. This is the whole point of the shared source. + let cases = [ + (PartitionKind::Daily, "%Y-%m-%d"), + (PartitionKind::Hourly, "%Y-%m-%dT%H"), + (PartitionKind::Weekly, "%G-W%V"), + (PartitionKind::Monthly, "%Y-%m"), + ]; + for (kind, fmt) in cases { + assert_eq!( + wm_partition_macro(&pspec(kind.clone(), None)).as_deref(), + Some( + format!( + "CREATE OR REPLACE TEMP MACRO wm_partition(ts) AS strftime(ts, '{fmt}')" + ) + .as_str() + ), + "wrong macro format for {kind:?}" + ); + } + } + + #[test] + fn wm_partition_macro_honors_format_override_and_skips_dynamic() { + // Explicit `format=` wins over the grain default. + assert!( + wm_partition_macro(&pspec(PartitionKind::Hourly, Some("%Y/%m/%d %H"))) + .unwrap() + .contains("strftime(ts, '%Y/%m/%d %H')") + ); + // Dynamic has no wall-clock bucket → no macro (user filters on their key). + assert_eq!( + wm_partition_macro(&pspec( + PartitionKind::Dynamic { key: "$.tenant".into() }, + None + )), + None + ); + } + #[test] fn split_respects_strings_comments_idents() { let sql = "SET x=1; -- a; comment\nSELECT ';' AS a, \"weird;col\" /* ; */ FROM t;"; @@ -1028,6 +1653,7 @@ mod tests { partition_value_sql: "'2026-06-19'", partitioned: true, strategy: MaterializeStrategy::Replace, + on_schema_change: OnSchemaChange::Warn, }; let st = cg.statements(); assert!(st[0].contains("CREATE TABLE IF NOT EXISTS _wm_target.orders_daily")); @@ -1053,6 +1679,7 @@ mod tests { partition_value_sql: "'2026-06-19'", partitioned: true, strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() }, + on_schema_change: OnSchemaChange::Warn, }; let st = cg.statements(); // upsert = delete-by-key (partition-scoped) + insert — NO `MERGE INTO` @@ -1070,6 +1697,42 @@ mod tests { .any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'"))); } + #[test] + fn codegen_merge_emits_duplicate_source_key_guard() { + // A keyed merge does not dedup its source, so codegen must emit a guard + // that fails the write when the SELECT has >1 row per key — otherwise + // duplicate source keys silently double-write. + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT order_id, amount FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: false, + strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() }, + on_schema_change: OnSchemaChange::Warn, + }; + let st = cg.statements(); + let guard = st + .iter() + .find(|s| s.contains("_wm_dup_keys")) + .expect("duplicate-key guard stmt"); + // raises via error(), counts non-NULL keys appearing more than once + assert!(guard.contains("error('managed materialize: keyed merge on `order_id` blocked")); + assert!(guard.contains("GROUP BY order_id HAVING count(*) > 1")); + assert!(guard.contains("WHERE order_id IS NOT NULL")); + // must run before the mutations so a violation aborts before any write + let guard_pos = st.iter().position(|s| s.contains("_wm_dup_keys")).unwrap(); + let del_pos = st + .iter() + .position(|s| s.starts_with("DELETE FROM")) + .unwrap(); + let ins_pos = st + .iter() + .position(|s| s.starts_with("INSERT INTO")) + .unwrap(); + assert!(guard_pos < del_pos && guard_pos < ins_pos); + } + #[test] fn codegen_append_inserts_only() { let cg = MaterializeCodegen { @@ -1079,6 +1742,7 @@ mod tests { partition_value_sql: "'2026-06-19'", partitioned: true, strategy: MaterializeStrategy::Append, + on_schema_change: OnSchemaChange::Warn, }; let st = cg.statements(); assert!(st @@ -1100,6 +1764,7 @@ mod tests { partition_value_sql: "''", partitioned: false, strategy: MaterializeStrategy::Replace, + on_schema_change: OnSchemaChange::Warn, }; let st = cg.statements(); assert_eq!( @@ -1111,6 +1776,297 @@ mod tests { ); } + #[test] + fn codegen_scd2_default_track_closes_old_opens_new() { + // Empty `track` ⇒ diff on all business columns via `* EXCLUDE (scd cols)`. + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim_scd2", + select_sql: "SELECT id, name FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec![], + close_deleted: false, + }, + on_schema_change: OnSchemaChange::Warn, + }; + let st = cg.statements(); + // bootstrap adds the three SCD metadata columns + assert!(st[0].starts_with("CREATE TABLE IF NOT EXISTS _wm_target.dim_scd2 AS SELECT *,")); + assert!(st[0].contains("AS valid_from")); + assert!(st[0].contains("AS valid_to")); + assert!(st[0].contains("AS is_current")); + // changed-key set captured before the transaction, all cols compared + assert!( + st[1].contains("CREATE OR REPLACE TEMP TABLE _wm_scd2_changed AS SELECT \"id\" FROM") + ); + assert!(st[1].contains("SELECT * FROM (SELECT id, name FROM dl.src) EXCEPT")); + assert!(st[1].contains("SELECT * EXCLUDE (valid_from, valid_to, is_current) FROM _wm_target.dim_scd2 WHERE is_current")); + assert_eq!(st[2], "BEGIN TRANSACTION;"); + // close: UPDATE (not MERGE) the prior open version of changed keys, with + // null-safe key matching (IS NOT DISTINCT FROM, not IN — IN drops NULLs) + assert!(st[3].starts_with("UPDATE _wm_target.dim_scd2 SET valid_to = CAST(now() AS TIMESTAMP), is_current = false")); + assert!(st[3].contains( + "WHERE is_current AND EXISTS (SELECT 1 FROM _wm_scd2_changed \ + WHERE _wm_scd2_changed.\"id\" IS NOT DISTINCT FROM _wm_target.dim_scd2.\"id\");" + )); + // open: INSERT the new current version, null-safe key matching + assert!(st[4].starts_with( + "INSERT INTO _wm_target.dim_scd2 SELECT s.*, CAST(now() AS TIMESTAMP) AS valid_from" + )); + assert!(st[4].contains( + "true AS is_current FROM (SELECT id, name FROM dl.src) s WHERE EXISTS \ + (SELECT 1 FROM _wm_scd2_changed c WHERE c.\"id\" IS NOT DISTINCT FROM s.\"id\");" + )); + // consumer-convenience `_current` view: `IF NOT EXISTS` (created once, + // no-op on unchanged reruns) and INSIDE the txn (folded into the write snapshot) + assert_eq!( + st[5], + "CREATE VIEW IF NOT EXISTS _wm_target.dim_scd2_current AS SELECT * FROM _wm_target.dim_scd2 WHERE is_current;" + ); + assert_eq!(st[6], "COMMIT;"); + // no fragile constructs: no MERGE INTO, and no NULL-dropping `IN (SELECT` + assert!(!st.iter().any(|s| s.contains("MERGE INTO"))); + assert!(!st.iter().any(|s| s.contains("IN (SELECT"))); + // soft-delete default: no deleted-key set, no second close + assert!(!st.iter().any(|s| s.contains("_wm_scd2_deleted"))); + } + + #[test] + fn codegen_scd2_explicit_track_projects_key_and_tracked_cols() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim", + select_sql: "SELECT id, name, addr FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec!["name".to_string()], + close_deleted: false, + }, + on_schema_change: OnSchemaChange::Warn, + }; + let st = cg.statements(); + // only key + tracked cols are compared (addr changes don't rotate a version) + assert!(st[1].contains("SELECT \"id\", \"name\" FROM (SELECT id, name, addr FROM dl.src) EXCEPT SELECT \"id\", \"name\" FROM _wm_target.dim WHERE is_current")); + } + + #[test] + fn codegen_scd2_close_deleted_adds_deleted_set_and_second_close() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim", + select_sql: "SELECT id, name FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec![], + close_deleted: true, + }, + on_schema_change: OnSchemaChange::Warn, + }; + let st = cg.statements(); + // the deleted-key set: current keys absent from the snapshot, captured + // before the transaction (like `changed`) + assert!(st.iter().any(|s| s.contains( + "CREATE OR REPLACE TEMP TABLE _wm_scd2_deleted AS SELECT \"id\" FROM \ + (SELECT \"id\" FROM _wm_target.dim WHERE is_current EXCEPT SELECT \"id\" FROM (SELECT id, name FROM dl.src));" + ))); + // a second close UPDATE against the deleted set (null-safe), and NO INSERT + // that reopens deleted keys (the only INSERT filters on `_wm_scd2_changed`) + assert!(st + .iter() + .any(|s| s.starts_with("UPDATE _wm_target.dim SET valid_to") + && s.contains( + "EXISTS (SELECT 1 FROM _wm_scd2_deleted \ + WHERE _wm_scd2_deleted.\"id\" IS NOT DISTINCT FROM _wm_target.dim.\"id\");" + ))); + assert_eq!( + st.iter().filter(|s| s.starts_with("INSERT INTO")).count(), + 1 + ); + assert!(st + .iter() + .find(|s| s.starts_with("INSERT INTO")) + .unwrap() + .contains("_wm_scd2_changed")); + // the deleted close is inside the transaction (between BEGIN and COMMIT) + let begin = st.iter().position(|s| s == "BEGIN TRANSACTION;").unwrap(); + let commit = st.iter().position(|s| s == "COMMIT;").unwrap(); + let del_close = st + .iter() + .position(|s| s.starts_with("UPDATE") && s.contains("_wm_scd2_deleted")) + .unwrap(); + assert!(begin < del_close && del_close < commit); + } + + fn persist_cg(strategy: MaterializeStrategy, osc: OnSchemaChange) -> Vec { + MaterializeCodegen { + target_qualified: "_wm_target.t", + select_sql: "SELECT a, c FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy, + on_schema_change: osc, + } + .statements() + } + + #[test] + fn codegen_fail_emits_drift_guard_inside_txn_before_write() { + let st = persist_cg( + MaterializeStrategy::Merge { unique_key: "a".into() }, + OnSchemaChange::Fail, + ); + let begin = st.iter().position(|s| s == "BEGIN TRANSACTION;").unwrap(); + let guard = st + .iter() + .position(|s| s.contains("error(") && s.contains("on_schema_change=fail")) + .expect("fail emits a guard"); + let del = st + .iter() + .position(|s| s.starts_with("DELETE FROM")) + .unwrap(); + let insert = st + .iter() + .position(|s| s.starts_with("INSERT INTO")) + .unwrap(); + let commit = st.iter().position(|s| s == "COMMIT;").unwrap(); + // guard runs right after BEGIN and before any mutation, inside the txn + assert!(begin < guard && guard < del && del < insert && insert < commit); + // both CASE branches are VARCHAR so the planner can't fold error() away + assert!(st[guard].contains("CAST(error(")); + assert!(st[guard].contains("ELSE 'ok' END")); + // fail is positional (no BY NAME) and emits no sync sentinel + assert!(st[insert].starts_with("INSERT INTO _wm_target.t SELECT")); + assert!(!st.iter().any(|s| s == SYNC_ALTER_SENTINEL)); + } + + #[test] + fn fail_guard_drift_is_name_set_based_not_ordered() { + // Documents a deliberate boundary: the fail guard fires on the column + // SET difference (EXCEPT over column_name), so a pure reorder of + // same-named columns is NOT caught here — that is `sync`'s job (BY NAME). + // Keeping this name-set (not an ordered comparison) is what prevents the + // guard from false-positive-aborting a correctly-aligned write. If this + // ever moves to an ordered comparison, update the OnSchemaChange doc. + let guard = + schema_drift_guard_sql("SELECT b, a FROM dl.src", "_wm_target.t", "_wm_partition"); + assert!(guard.contains("column_name")); + assert!(guard.contains("EXCEPT")); + // No positional/ordinal comparison in the guard condition. + assert!(!guard.to_lowercase().contains("ordinal")); + assert!(!guard.to_lowercase().contains("row_number")); + } + + #[test] + fn codegen_sync_uses_by_name_and_sentinel() { + let st = persist_cg(MaterializeStrategy::Append, OnSchemaChange::Sync); + let begin = st.iter().position(|s| s == "BEGIN TRANSACTION;").unwrap(); + let sentinel = st.iter().position(|s| s == SYNC_ALTER_SENTINEL).unwrap(); + let insert = st + .iter() + .position(|s| s.starts_with("INSERT INTO")) + .unwrap(); + // the ALTER-injection slot is right after BEGIN, before the write + assert!(begin < sentinel && sentinel < insert); + // name-mapped insert (a positional insert would cross-wire ALTERed cols) + assert!(st[insert].starts_with("INSERT INTO _wm_target.t BY NAME SELECT")); + assert!(!st.iter().any(|s| s.contains("error("))); + } + + #[test] + fn codegen_ignore_and_warn_write_positionally_no_guard() { + for osc in [OnSchemaChange::Ignore, OnSchemaChange::Warn] { + let st = persist_cg(MaterializeStrategy::Append, osc); + assert!(!st.iter().any(|s| s.contains("error(")), "{osc:?}"); + assert!(!st.iter().any(|s| s == SYNC_ALTER_SENTINEL), "{osc:?}"); + let insert = st.iter().find(|s| s.starts_with("INSERT INTO")).unwrap(); + assert!( + insert.starts_with("INSERT INTO _wm_target.t SELECT"), + "{osc:?}" + ); + } + } + + #[test] + fn codegen_whole_table_replace_and_scd2_ignore_guardrail() { + // Whole-table replace (unpartitioned) and scd2 are not persist-and-mutate: + // fail/sync must not add a guard/sentinel/BY NAME there. + let repl = MaterializeCodegen { + target_qualified: "_wm_target.t", + select_sql: "SELECT a FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Replace, + on_schema_change: OnSchemaChange::Fail, + }; + assert!(!repl.is_persist_and_mutate()); + let st = repl.statements(); + assert!(!st + .iter() + .any(|s| s.contains("error(") || s == SYNC_ALTER_SENTINEL)); + + let scd2 = MaterializeCodegen { + strategy: MaterializeStrategy::Scd2 { + key: "a".into(), + track: vec![], + close_deleted: false, + }, + on_schema_change: OnSchemaChange::Sync, + ..repl + }; + assert!(!scd2.is_persist_and_mutate()); + let st = scd2.statements(); + assert!(!st.iter().any(|s| s == SYNC_ALTER_SENTINEL)); + assert!(!st.iter().any(|s| s.contains(" BY NAME "))); + } + + #[test] + fn summary_schema_drift_field_only_for_warn_persist_and_mutate() { + // warn + persist-and-mutate ⇒ summary carries the drift column + let warn = plan_for_osc(MaterializeStrategy::Append, false, OnSchemaChange::Warn); + assert!(warn.stmts.last().unwrap().sql.contains("AS schema_drift")); + // ignore / fail / sync ⇒ no summary drift column (they guard the write) + for osc in [ + OnSchemaChange::Ignore, + OnSchemaChange::Fail, + OnSchemaChange::Sync, + ] { + let p = plan_for_osc(MaterializeStrategy::Append, false, osc); + assert!( + !p.stmts.last().unwrap().sql.contains("schema_drift"), + "{osc:?} must not emit the summary drift column" + ); + } + // whole-table replace + warn ⇒ not persist-and-mutate ⇒ no drift column + let repl = plan_for_osc(MaterializeStrategy::Replace, false, OnSchemaChange::Warn); + assert!(!repl.stmts.last().unwrap().sql.contains("schema_drift")); + } + + #[test] + fn fail_guard_is_write_kind_between_txn_markers() { + use MaterializeStmtKind::*; + let plan = plan_for_osc( + MaterializeStrategy::Merge { unique_key: "a".into() }, + false, + OnSchemaChange::Fail, + ); + let begin = kidx(&plan, |s| s.kind == TxnBegin); + let commit = kidx(&plan, |s| s.kind == TxnCommit); + let guard = kidx(&plan, |s| { + s.sql.contains("error(") && s.sql.contains("on_schema_change=fail") + }); + assert_eq!(plan.stmts[guard].kind, Write); + assert!(begin < guard && guard < commit); + } + #[test] fn snapshot_capture_targets_alias() { assert_eq!( @@ -1122,7 +2078,7 @@ mod tests { #[test] fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() { let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'"); - let blocks = build_wrap_blocks( + let blocks: Vec = build_wrap_blocks( &plan, "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');", "orders_daily", @@ -1131,9 +2087,14 @@ mod tests { "'2026-06-19'", true, MaterializeStrategy::Replace, + OnSchemaChange::Warn, &[], ) - .unwrap(); + .unwrap() + .stmts + .into_iter() + .map(|s| s.sql) + .collect(); // setup block first, then the target ATTACH, then codegen, then result. assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl")); // every setup block must be `;`-terminated so re-splitting can't merge it @@ -1156,6 +2117,112 @@ mod tests { assert!(last.contains("ducklake_snapshots('_wm_target')")); } + // -- materialize plan structure ------------------------------------------ + + fn plan_for(strategy: MaterializeStrategy, partitioned: bool) -> MaterializePlan { + plan_for_osc(strategy, partitioned, OnSchemaChange::Warn) + } + + fn plan_for_osc( + strategy: MaterializeStrategy, + partitioned: bool, + on_schema_change: OnSchemaChange, + ) -> MaterializePlan { + let plan = ok("SELECT a, b FROM src"); + build_wrap_blocks( + &plan, + "ATTACH 'ducklake:…' AS _wm_target;", + "orders", + "main/orders", + "_wm_partition", + "'2026-06-19'", + partitioned, + strategy, + on_schema_change, + &[ + DataTestResolved::BuiltIn(DataTest::NotNull { column: "a".into() }), + DataTestResolved::BuiltIn(DataTest::Unique { column: "b".into() }), + ], + ) + .unwrap() + } + + fn kidx(plan: &MaterializePlan, pred: impl Fn(&MaterializeStmt) -> bool) -> usize { + plan.stmts + .iter() + .position(|s| pred(s)) + .expect("stmt present") + } + + #[test] + fn plan_tags_structure_and_carries_checks() { + use MaterializeStmtKind::*; + let plan = plan_for(MaterializeStrategy::Replace, true); + // leading statements are Setup, ending with the target ATTACH + assert!(plan.stmts[0].kind == Setup); + assert!(plan + .stmts + .iter() + .take_while(|s| s.kind == Setup) + .any(|s| s.sql.contains("_wm_target"))); + // txn markers are tagged, everything between them is Write + let begin = kidx(&plan, |s| s.kind == TxnBegin); + let commit = kidx(&plan, |s| s.kind == TxnCommit); + assert!(begin < commit); + assert!(plan.stmts[begin + 1..commit] + .iter() + .all(|s| s.kind == Write)); + // bootstrap DDL is Write work (it targets the table, not the session) + let bootstrap = kidx(&plan, |s| s.sql.starts_with("CREATE TABLE IF NOT EXISTS")); + assert_eq!(plan.stmts[bootstrap].kind, Write); + // summary is last and carries the breakdown; checks ride along + let last = plan.stmts.last().unwrap(); + assert_eq!(last.kind, Summary); + assert!(last.sql.contains("AS data_tests")); + assert_eq!(plan.checks.len(), 2); + assert!(plan.checks[0].name.contains("not_null(a)")); + } + + #[test] + fn plan_whole_table_replace_has_no_txn_markers() { + use MaterializeStmtKind::*; + let plan = plan_for(MaterializeStrategy::Replace, false); + assert!(!plan.stmts.iter().any(|s| s.kind == TxnBegin)); + assert!(!plan.stmts.iter().any(|s| s.kind == TxnCommit)); + assert_eq!( + plan.stmts.iter().filter(|s| s.kind == Write).count(), + 1, + "single atomic CREATE OR REPLACE" + ); + } + + #[test] + fn plan_scd2_captures_are_write_kind() { + use MaterializeStmtKind::*; + let plan = plan_for( + MaterializeStrategy::Scd2 { key: "a".into(), track: vec![], close_deleted: false }, + false, + ); + let capture = kidx(&plan, |s| s.sql.contains("TEMP TABLE _wm_scd2_changed")); + assert_eq!(plan.stmts[capture].kind, Write); + // no test declared ⇒ empty checks + let plain = ok("SELECT a FROM src"); + let no_tests = build_wrap_blocks( + &plain, + "ATTACH 'ducklake:…' AS _wm_target;", + "orders", + "main/orders", + "_wm_partition", + "''", + false, + MaterializeStrategy::Append, + OnSchemaChange::Warn, + &[], + ) + .unwrap(); + assert!(no_tests.checks.is_empty()); + } + // -- data tests --------------------------------------------------------- fn ctx_partitioned() -> DataTestCtx<'static> { @@ -1165,11 +2232,15 @@ mod tests { partition_col: "_wm_partition", partition_value_sql: "'2026-06-19'", partitioned: true, + scd2: false, } } fn ctx_unpartitioned() -> DataTestCtx<'static> { DataTestCtx { partitioned: false, ..ctx_partitioned() } } + fn ctx_scd2() -> DataTestCtx<'static> { + DataTestCtx { partitioned: false, scd2: true, ..ctx_partitioned() } + } #[test] fn data_test_unique_and_not_null_partition_scoped() { @@ -1183,21 +2254,34 @@ mod tests { // short, asset-free names (the asset is shown once by the breakdown). assert_eq!(sql.checks[0].name, "unique(order_id)"); assert_eq!(sql.checks[1].name, "not_null(user_id)"); - // each `violating` is a scalar count subquery. + // each probe counts and samples the violating rows in one scan, with + // the size guard on the serialized sample. + for c in &sql.checks { + assert!(c + .probe + .starts_with("SELECT v, CASE WHEN strlen(s_raw) <= 51200 THEN s_raw END AS s")); + assert!(c.probe.contains("count(*) AS v")); + assert!(c.probe.contains("FILTER (WHERE _wm_rn <= 20)")); + } + // unique: groups non-null keys within the slice, having count>1; the + // sample is `{value, count}` pairs at the same grain as the count. assert!(sql.checks[0] - .violating - .starts_with("(SELECT count(*) AS v FROM")); - // unique: groups non-null keys within the slice, having count>1 - assert!(sql.checks[0] - .violating + .probe .contains("GROUP BY \"order_id\" HAVING count(*) > 1")); assert!(sql.checks[0] - .violating + .probe + .contains("SELECT \"order_id\" AS \"value\", count(*) AS \"count\"")); + assert!(sql.checks[0] + .probe .contains("\"order_id\" IS NOT NULL AND \"_wm_partition\" = '2026-06-19'")); - // not_null: null rows in the slice + // not_null: null rows in the slice; the sample excludes the synthetic + // partition column (storage detail, not producer output). assert!(sql.checks[1] - .violating + .probe .contains("WHERE \"user_id\" IS NULL AND \"_wm_partition\" = '2026-06-19'")); + assert!(sql.checks[1] + .probe + .contains("SELECT * EXCLUDE (\"_wm_partition\") FROM")); } #[test] @@ -1206,8 +2290,33 @@ mod tests { column: "id".into(), })]; let sql = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap(); - assert!(sql.checks[0].violating.contains("WHERE \"id\" IS NULL")); - assert!(!sql.checks[0].violating.contains("_wm_partition")); + assert!(sql.checks[0].probe.contains("WHERE \"id\" IS NULL")); + assert!(!sql.checks[0].probe.contains("_wm_partition")); + assert!(!sql.checks[0].probe.contains("EXCLUDE")); + } + + #[test] + fn data_test_scd2_scopes_builtins_to_current_rows() { + // On an SCD2 history table the natural key repeats across closed + // versions, so built-in probes must assert the current snapshot only. + let tests = vec![ + DataTestResolved::BuiltIn(DataTest::Unique { column: "customer_id".into() }), + DataTestResolved::BuiltIn(DataTest::NotNull { column: "tier".into() }), + DataTestResolved::BuiltIn(DataTest::AcceptedValues { + column: "region".into(), + values: vec!["emea".into()], + }), + ]; + let sql = build_data_test_checks(&tests, &ctx_scd2()).unwrap(); + assert!(sql.checks[0] + .probe + .contains("WHERE \"customer_id\" IS NOT NULL AND is_current")); + assert!(sql.checks[1] + .probe + .contains("WHERE \"tier\" IS NULL AND is_current")); + assert!(sql.checks[2].probe.contains("AND is_current")); + // no partition scope leaks in (scd2 is unpartitioned in v1) + assert!(!sql.checks[0].probe.contains("_wm_partition")); } #[test] @@ -1217,10 +2326,8 @@ mod tests { values: vec!["paid".into(), "o'brien".into()], })]; let sql = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap(); - assert!(sql.checks[0] - .violating - .contains("NOT IN ('paid', 'o''brien')")); - assert!(sql.checks[0].violating.contains("\"status\" IS NOT NULL")); + assert!(sql.checks[0].probe.contains("NOT IN ('paid', 'o''brien')")); + assert!(sql.checks[0].probe.contains("\"status\" IS NOT NULL")); } #[test] @@ -1244,14 +2351,19 @@ mod tests { assert_eq!(sql.attaches.len(), 1, "same db attached once"); assert_eq!(sql.attaches[0], "ATTACH 'datatable://prod' AS _wm_ref_0;"); assert!(sql.checks[0] - .violating + .probe .contains("NOT EXISTS (SELECT 1 FROM _wm_ref_0.\"users\"")); assert!(sql.checks[1] - .violating + .probe .contains("NOT EXISTS (SELECT 1 FROM _wm_ref_0.\"buyers\"")); assert!(sql.checks[0] - .violating + .probe .contains("_wm_src.\"_wm_partition\" = '2026-06-19'")); + // sample rows come from the aliased target and drop the synthetic + // partition column. + assert!(sql.checks[0] + .probe + .contains("SELECT _wm_src.* EXCLUDE (\"_wm_partition\") FROM")); assert_eq!( sql.checks[0].name, "relationships(user_id -> prod/users.id)" @@ -1288,7 +2400,7 @@ mod tests { "same-lake ref must not ATTACH again" ); assert!(sql.checks[0] - .violating + .probe .contains("NOT EXISTS (SELECT 1 FROM _wm_target.\"users\"")); } @@ -1309,10 +2421,10 @@ mod tests { ); assert!( sql.checks[0] - .violating + .probe .contains("FROM _wm_ref_0.\"main\".\"dim_products\""), "schema-qualified target should be quoted per segment: {}", - sql.checks[0].violating + sql.checks[0].probe ); } @@ -1334,23 +2446,96 @@ mod tests { body: "SELECT * FROM _wm_target.orders WHERE amount < 0;".into(), }]; let sql = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap(); - // trailing ; stripped, wrapped as a count subquery - assert!(sql.checks[0].violating.contains( - "SELECT count(*) AS v FROM (SELECT * FROM _wm_target.orders WHERE amount < 0)" - )); + // trailing ; stripped, body embedded as the probe's rows query + assert!(sql.checks[0] + .probe + .contains("FROM (SELECT * FROM _wm_target.orders WHERE amount < 0) _wm_v")); + assert!(sql.checks[0].probe.contains("count(*) AS v")); assert_eq!(sql.checks[0].name, "custom(f/tests/amount)"); } #[test] fn data_test_custom_rejects_multi_statement_body() { // The body is embedded as a subquery, so a setup-then-SELECT body would - // produce invalid SQL — reject it up front with an actionable error. + // produce invalid SQL — reject it up front with a self-teaching error + // that names the violation and shows the correct single-SELECT shape. let tests = vec![DataTestResolved::Custom { path: "f/tests/amount".into(), body: "SET threads = 1; SELECT * FROM _wm_target.orders WHERE amount < 0".into(), }]; let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); assert!(err.contains("single SELECT"), "unexpected error: {err}"); + assert!( + err.contains("found 2 statements"), + "unexpected error: {err}" + ); + // the copyable example points at the internal target alias. + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_rejects_non_select_body() { + // A write/DDL body can't be embedded as `FROM ()`; the error must + // say so and teach the single-SELECT convention. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "DELETE FROM _wm_target.orders WHERE amount < 0".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("single SELECT"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_rejects_wrong_target_alias() { + // Referencing the target by its bare table name (not `_wm_target.
`) + // is the most common custom-test mistake — the runtime only attaches the + // freshly-materialized target under `_wm_target`, so the query would fail + // at runtime. Catch it at codegen with a self-teaching error. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "SELECT * FROM orders WHERE amount < 0".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("_wm_target"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_accepts_from_first_and_uppercased_alias() { + // DuckDB's FROM-first syntax is a valid Output, and the alias match is + // case-insensitive (SQL identifiers are), so this passes. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "FROM _WM_TARGET.orders WHERE amount < 0".into(), + }]; + let sql = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap(); + assert!(sql.checks[0] + .probe + .contains("FROM (FROM _WM_TARGET.orders WHERE amount < 0) _wm_v")); + } + + #[test] + fn data_test_custom_empty_body_teaches_shape() { + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: " \n-- just a comment\n".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("empty test body"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); } #[test] @@ -1364,14 +2549,8 @@ mod tests { #[test] fn materialize_result_sql_embeds_data_tests_breakdown() { let checks = vec![ - DataTestCheck { - name: "unique(order_id)".into(), - violating: "(SELECT count(*) AS v FROM q0)".into(), - }, - DataTestCheck { - name: "custom(f/t)".into(), - violating: "(SELECT count(*) AS v FROM q1)".into(), - }, + DataTestCheck { name: "unique(order_id)".into(), probe: "SELECT v, s FROM q0".into() }, + DataTestCheck { name: "custom(f/t)".into(), probe: "SELECT v, s FROM q1".into() }, ]; let sql = materialize_result_sql( "_wm_target.orders", @@ -1380,11 +2559,19 @@ mod tests { "'2026-06-19'", false, &checks, + None, + ); + // each probe runs once as a one-row CTE; _wm_tr cross-joins them and + // the list-of-struct references the flattened count/sample columns. + assert!(sql.starts_with( + "WITH _wm_t0 AS (SELECT v, s FROM q0), _wm_t1 AS (SELECT v, s FROM q1), \ + _wm_tr AS (SELECT _wm_t0.v AS c0, _wm_t0.s AS s0, _wm_t1.v AS c1, _wm_t1.s AS s1 \ + FROM _wm_t0, _wm_t1)" + )); + assert!(sql.contains("[{'test': 'unique(order_id)', 'violating': c0, 'sample': s0}, ")); + assert!( + sql.contains("{'test': 'custom(f/t)', 'violating': c1, 'sample': s1}] AS data_tests") ); - // counts computed once in a CTE, referenced by the list-of-struct. - assert!(sql.starts_with("WITH _wm_tr AS (SELECT (SELECT count(*) AS v FROM q0) AS c0,")); - assert!(sql.contains("[{'test': 'unique(order_id)', 'violating': c0}, ")); - assert!(sql.contains("{'test': 'custom(f/t)', 'violating': c1}] AS data_tests")); assert!(sql.contains("FROM _wm_tr;")); // no tests -> plain summary, no CTE / data_tests column. let plain = materialize_result_sql( @@ -1394,6 +2581,7 @@ mod tests { "'x'", false, &[], + None, ); assert!(plain.starts_with("SELECT 'ducklake://analytics/orders' AS materialized")); assert!(!plain.contains("data_tests")); @@ -1420,6 +2608,7 @@ mod tests { "'2026-06-19'", true, &[], + None, ); assert!(sql.contains( "FROM (DESCRIBE SELECT * FROM _wm_target.orders_daily) \ diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 4ecbc992bc..cabfe20ddf 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -48,7 +48,11 @@ "expected": { "in_pipeline": true, "asset_triggers": [], - "native_triggers": ["kafka", "schedule", "data_upload"], + "native_triggers": [ + "kafka", + "schedule", + "data_upload" + ], "partition": null, "freshness": null, "tag": null, @@ -154,7 +158,10 @@ "partition": null, "freshness": null, "tag": null, - "retry": { "count": 3, "delay": "5s" } + "retry": { + "count": 3, + "delay": "5s" + } } }, { @@ -167,7 +174,10 @@ "partition": null, "freshness": null, "tag": null, - "retry": { "count": 2, "delay": null } + "retry": { + "count": 2, + "delay": null + } } }, { @@ -201,7 +211,9 @@ "code": "// pipeline\n// on s3://bucket/daily/{partition}/data.parquet\nexport function main() {}", "expected": { "in_pipeline": true, - "asset_triggers": ["s3object:bucket/daily/{partition}/data.parquet"], + "asset_triggers": [ + "s3object:bucket/daily/{partition}/data.parquet" + ], "native_triggers": [], "partition": null, "freshness": null, @@ -214,7 +226,9 @@ "code": " -- pipeline\n\t-- on datatable://main/x\nSELECT 1;", "expected": { "in_pipeline": true, - "asset_triggers": ["datatable:main/x"], + "asset_triggers": [ + "datatable:main/x" + ], "native_triggers": [], "partition": null, "freshness": null, @@ -253,6 +267,104 @@ } } }, + { + "name": "materialize scd2 via history flag with key, track and deletes=close", + "code": "// pipeline\n// materialize ducklake://analytics/dim_customer key=id history track=name,tier deletes=close\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim_customer", + "unique_key": "id", + "scd2": true, + "track": [ + "name", + "tier" + ], + "close_deleted": true + } + } + }, + { + "name": "materialize scd2 keyword alias with key only (track all non-key cols)", + "code": "// materialize scd2 ducklake://analytics/dim key=id\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim", + "unique_key": "id", + "scd2": true + } + } + }, + { + "name": "materialize on_schema_change=ignore opt", + "code": "// pipeline\n// materialize ducklake://analytics/orders on_schema_change=ignore\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders", + "on_schema_change": "ignore" + } + } + }, + { + "name": "materialize on_schema_change unknown value keeps warn default (fail-safe)", + "code": "// materialize ducklake://analytics/orders key=id on_schema_change=bogus\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders", + "unique_key": "id", + "on_schema_change": "warn" + } + } + }, + { + "name": "materialize key without history is plain merge (SCD1, not scd2)", + "code": "// materialize ducklake://analytics/dim key=id\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim", + "unique_key": "id" + } + } + }, { "name": "materialize manual escape hatch, first value wins", "code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}", @@ -319,12 +431,22 @@ "unique_key": "order_id" }, "data_tests": [ - { "type": "unique", "column": "order_id" }, - { "type": "not_null", "column": "user_id" }, + { + "type": "unique", + "column": "order_id" + }, + { + "type": "not_null", + "column": "user_id" + }, { "type": "accepted_values", "column": "status", - "values": ["paid", "pending", "refunded"] + "values": [ + "paid", + "pending", + "refunded" + ] }, { "type": "relationships", @@ -348,7 +470,15 @@ "tag": null, "retry": null, "data_tests": [ - { "type": "accepted_values", "column": "kind", "values": ["a b", "c", "d"] } + { + "type": "accepted_values", + "column": "kind", + "values": [ + "a b", + "c", + "d" + ] + } ] } }, @@ -363,7 +493,12 @@ "freshness": null, "tag": null, "retry": null, - "data_tests": [{ "type": "custom", "path": "f/tests/orders_amount_sane" }] + "data_tests": [ + { + "type": "custom", + "path": "f/tests/orders_amount_sane" + } + ] } }, { @@ -399,7 +534,12 @@ "freshness": null, "tag": null, "retry": null, - "data_tests": [{ "type": "unique", "column": "id" }] + "data_tests": [ + { + "type": "unique", + "column": "id" + } + ] } }, { @@ -413,7 +553,12 @@ "freshness": null, "tag": null, "retry": null, - "data_tests": [{ "type": "unique", "column": "id" }] + "data_tests": [ + { + "type": "unique", + "column": "id" + } + ] } }, { @@ -446,7 +591,11 @@ { "column": "user_name", "inputs": [ - { "from_kind": "datatable", "from_path": "prod/users", "from_column": "name" } + { + "from_kind": "datatable", + "from_path": "prod/users", + "from_column": "name" + } ] } ] @@ -506,5 +655,119 @@ } ] } + }, + { + "name": "bare macros marker", + "code": "// macros\nCREATE MACRO dbl(a) AS a * 2;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "macros": true + } + }, + { + "name": "macros with trailing prose is not a marker", + "code": "// macros are defined below\nCREATE MACRO dbl(a) AS a * 2;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "macros marker with sql comment prefix", + "code": "-- macros\n-- pipeline\nCREATE MACRO dbl(a) AS a * 2;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "macros": true + } + }, + { + "name": "use accumulates in order and dedups", + "code": "// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "use_libs": [ + "f/lib/stats", + "f/lib/dates" + ] + } + }, + { + "name": "use requires a slashed single token", + "code": "// use this script to compute stuff\n// use standalone\n// use f/lib/ok extra\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "use stops at first code line", + "code": "// pipeline\nSELECT 1;\n-- use f/lib/late\n", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "s3 triple-slash default-storage trigger canonicalizes to bare key", + "code": "// pipeline\n// on s3:///exports/x\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [ + "s3object:exports/x" + ], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "s3 quad-slash trigger strips all leading slashes to the bare key", + "code": "// pipeline\n// on s3:////x\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [ + "s3object:x" + ], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } } ] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 883ddebcbc..7adb07b55c 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -47,6 +47,12 @@ struct Expected { // against `to_value(got.column_lineage)`. Absent === []. #[serde(default)] column_lineage: Vec, + // `// macros` marker (strict, alone on the line). Absent === false. + #[serde(default)] + macros: bool, + // `// use ` accumulation, declaration order, deduped. Absent === []. + #[serde(default)] + use_libs: Vec, } #[derive(Deserialize)] @@ -59,6 +65,19 @@ struct ExpectedMaterialize { append: bool, #[serde(default)] unique_key: Option, + #[serde(default)] + scd2: bool, + #[serde(default)] + track: Vec, + #[serde(default)] + close_deleted: bool, + // "warn" | "ignore"; absent === "warn" (the default). + #[serde(default = "default_on_schema_change")] + on_schema_change: String, +} + +fn default_on_schema_change() -> String { + "warn".to_string() } #[derive(Deserialize)] @@ -191,6 +210,22 @@ fn pipeline_annotation_fixtures_match() { assert_eq!(m.manual, e.manual, "{ctx}: materialize manual"); assert_eq!(m.append, e.append, "{ctx}: materialize append"); assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key"); + assert_eq!(m.scd2, e.scd2, "{ctx}: materialize scd2"); + assert_eq!(m.track, e.track, "{ctx}: materialize track"); + assert_eq!( + m.close_deleted, e.close_deleted, + "{ctx}: materialize close_deleted" + ); + let osc = match m.on_schema_change { + windmill_parser::asset_parser::OnSchemaChange::Warn => "warn", + windmill_parser::asset_parser::OnSchemaChange::Ignore => "ignore", + windmill_parser::asset_parser::OnSchemaChange::Fail => "fail", + windmill_parser::asset_parser::OnSchemaChange::Sync => "sync", + }; + assert_eq!( + osc, e.on_schema_change, + "{ctx}: materialize on_schema_change" + ); } (got, want) => panic!( "{ctx}: materialize mismatch — got {:?}, want present={}", @@ -213,5 +248,8 @@ fn pipeline_annotation_fixtures_match() { serde_json::Value::Array(f.expected.column_lineage.clone()), "{ctx}: column lineage" ); + + assert_eq!(got.macros, f.expected.macros, "{ctx}: macros"); + assert_eq!(got.use_libs, f.expected.use_libs, "{ctx}: use_libs"); } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 94b8e2f4f6..0186f271e6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1679,6 +1679,13 @@ async fn process_notify_event( ); windmill_queue::asset_dispatch::ASSET_PRODUCER_WRITES_CACHE.remove(payload); } + "notify_macro_registry_change" => { + tracing::debug!( + "Macro registry change for workspace {}, invalidating macro registry cache", + payload + ); + windmill_common::assets::MACRO_REGISTRY_CACHE.remove(payload); + } "notify_workspace_key_change" => { tracing::info!( "Workspace key change detected, invalidating workspace key cache: {}", @@ -1708,6 +1715,10 @@ async fn process_notify_event( match *source_type { "script" => { windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + // Bundle-cache key resolution for imported scripts; evicted + // together with the content-side caches below so key and + // inlined content flip to the new version in the same window. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&key); // Evict the relative-import latest-hash cache so a redeployed // imported script flips the content cache to its new version // across all replicas within a poll interval (see #6769). Keyed diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 817887eaad..60b4024c0a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -178,6 +178,14 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + // Ops kill switch for the pipeline freshness watchdog (a background + // pusher — being able to stop it without a redeploy matters more than + // for read-only monitors). + pub static ref DISABLE_FRESHNESS_WATCHDOG: bool = std::env::var("DISABLE_FRESHNESS_WATCHDOG") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + pub static ref WORKERS_NAMES: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); @@ -2994,6 +3002,28 @@ pub async fn monitor_db( } }; + // run every ~60s (2 iterations * 30s). Enterprise feature: the active + // `// freshness` backstop lives in windmill-queue's `freshness_watchdog` + // (`private`); OSS gets a no-op stub. Runtime-gated on an Enterprise + // license like the audit export above. Safe on concurrent servers — the + // watchdog claims per-script state rows atomically before pushing. + let pipeline_freshness_watchdog_f = async { + if server_mode + && !*DISABLE_FRESHNESS_WATCHDOG + && iteration.is_some() + && iteration.as_ref().unwrap().should_run(2) + { + if let Some(db) = conn.as_sql() { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) { + windmill_queue::freshness_watchdog::tick(db).await; + } + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -3021,6 +3051,7 @@ pub async fn monitor_db( manage_audit_partitions_f, export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, + pipeline_freshness_watchdog_f, ); } diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 6553abb5cc..380adf679e 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -115,6 +115,10 @@ kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path) kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[]) log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool) +macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts) + FK: (workspace_id) -> workspace(id) +macro_usage: workspace_id(char), consumer_path(char), macro_name(char) + FK: (workspace_id) -> workspace(id) magic_link: email(char), token(char), expiration(ts) mcp_oauth_client: mcp_server_url(text), client_id(text), client_secret(text), client_secret_expires_at(ts), token_endpoint(text), created_at(ts) mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), token_family(uuid), created_at(ts), expires_at(ts), used_at(ts), revoked(bool) @@ -182,7 +186,7 @@ windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) -workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char) +workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) workspace_diff: source_workspace_id(char), fork_workspace_id(char), path(char), kind(char), ahead(int), behind(int), has_changes(bool), exists_in_source(bool), exists_in_fork(bool) diff --git a/backend/tests/asset_trigger_dispatch.rs b/backend/tests/asset_trigger_dispatch.rs index 198472d404..7184d4810d 100644 --- a/backend/tests/asset_trigger_dispatch.rs +++ b/backend/tests/asset_trigger_dispatch.rs @@ -1044,3 +1044,132 @@ async fn reaper_clears_only_stale_join_slots(db: Pool) -> anyhow::Resu Ok(()) } + +/// Seed one `materialized_partition` row (the state a managed `// materialize` +/// write records) so dispatch has a snapshot to look up. +async fn seed_materialization( + db: &Pool, + kind: &str, + asset_path: &str, + partition: &str, + status: &str, + snapshot_id: Option, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO materialized_partition + (workspace_id, asset_kind, asset_path, partition, status, snapshot_id) + VALUES ($1, $2::asset_kind, $3, $4, $5::materialization_status, $6)"#, + ) + .bind(WS) + .bind(kind) + .bind(asset_path) + .bind(partition) + .bind(status) + .bind(snapshot_id) + .execute(db) + .await?; + Ok(()) +} + +/// Dispatch records, on the consumer's `trigger` arg, the latest captured +/// materialization snapshot of each of its direct upstream assets +/// (`upstream_snapshots`) — the forensic "what did this run see" record. +/// Covered here: +/// - latest = highest `snapshot_id` with status `materialized` (a stale +/// partition and a failed/no-snapshot row are both passed over), +/// - whole-table (sentinel '') upstreams omit `partition`, +/// - upstreams with no captured snapshot produce no entry, +/// - a consumer with no materialized upstream at all gets no +/// `upstream_snapshots` key. +#[sqlx::test(fixtures("base"))] +async fn upstream_snapshots_recorded_on_dispatch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + seed_script(&db, SUB_S3, "echo lake consumer", "bash").await?; + seed_script(&db, SUB_RES, "echo raw consumer", "bash").await?; + seed_asset_write(&db, PRODUCER, "ducklake", "analytics/orders").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/raw").await?; + + // SUB_S3's direct upstream set: the firing ducklake asset, a second + // materialized ducklake dimension (written by some other producer), and a + // plain s3 object that is never materialized. + seed_subscription(&db, SUB_S3, "script", "ducklake://analytics/orders").await?; + seed_subscription(&db, SUB_S3, "script", "ducklake://analytics/customers").await?; + seed_subscription(&db, SUB_S3, "script", "s3://f/raw").await?; + // SUB_RES subscribes only to the raw (non-materialized) asset. + seed_subscription(&db, SUB_RES, "script", "s3://f/raw").await?; + + // orders: an older partition, the latest one, and a failed slice with no + // snapshot — only 2026-06-19 @ 42 must be recorded. + seed_materialization( + &db, + "ducklake", + "analytics/orders", + "2026-06-18", + "materialized", + Some(41), + ) + .await?; + seed_materialization( + &db, + "ducklake", + "analytics/orders", + "2026-06-19", + "materialized", + Some(42), + ) + .await?; + seed_materialization( + &db, + "ducklake", + "analytics/orders", + "2026-06-20", + "failed", + None, + ) + .await?; + // customers: unpartitioned (sentinel '') → entry without `partition`. + seed_materialization( + &db, + "ducklake", + "analytics/customers", + "", + "materialized", + Some(7), + ) + .await?; + + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!( + r.dispatched.len(), + 3, + "SUB_S3 fired for both written assets, SUB_RES for the raw one" + ); + + let expected_snaps = json!([ + { "asset": "ducklake://analytics/customers", "snapshot_id": 7 }, + { "asset": "ducklake://analytics/orders", "snapshot_id": 42, "partition": "2026-06-19" }, + ]); + for (path, _, args) in fetch_dispatched(&db).await? { + let trigger = args + .as_ref() + .and_then(|a| a.get("trigger")) + .cloned() + .expect("dispatched job carries a trigger arg"); + match path.as_str() { + SUB_S3 => assert_eq!( + trigger.get("upstream_snapshots"), + Some(&expected_snaps), + "latest materialized snapshot per upstream, sorted by ref, raw asset absent" + ), + SUB_RES => assert!( + trigger.get("upstream_snapshots").is_none(), + "no materialized upstream → no upstream_snapshots key" + ), + other => panic!("unexpected dispatched path {other}"), + } + } + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index d51e51f58b..fc766253bc 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,3 +1,4 @@ +use futures::StreamExt; use sqlx::postgres::Postgres; use sqlx::Pool; use uuid::Uuid; @@ -818,6 +819,126 @@ export function main() { Ok(()) } +// ============================================================================ +// Bundle cache invalidation on transitive relative-import change +// ============================================================================ + +async fn insert_deployed_bun_script(db: &Pool, path: &str, hash: i64, content: &str) { + // What gen_bun_lockfile stores for a script with no npm dependencies; a + // bare '' lock fails split_lockfile when the script is run directly. + const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n"; + // Runtime query to avoid touching the sqlx offline cache. + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"properties\":{},\"required\":[],\"type\":\"object\"}', '', '', $2, $3, 'bun', $4)", + ) + .bind(content) + .bind(path) + .bind(hash) + .bind(EMPTY_BUN_LOCK) + .execute(db) + .await + .unwrap(); +} + +fn run_main_script_job(hash: i64) -> RunJob { + RunJob::from(JobPayload::ScriptHash { + path: "f/stale_bundle/main_script".to_string(), + hash: windmill_common::scripts::ScriptHash(hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bun, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }) +} + +/// Editing a script that a runnable imports only TRANSITIVELY (main -> mid -> +/// leaf) must invalidate the runnable's cached bundle: the leaf's code is +/// inlined in the bundle, so the cache key has to cover the whole closure, not +/// just direct imports. +#[sqlx::test(fixtures("base"))] +async fn test_bun_transitive_import_change_rebundles(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Hashes/paths unique across this test binary: the script/hash caches are + // process-global while parallel tests each run in their own DB. + const LEAF_V1: i64 = 41230001; + const MID: i64 = 41230002; + const MAIN: i64 = 41230003; + const LEAF_V2: i64 = 41230004; + + insert_deployed_bun_script( + &db, + "f/stale_bundle/leaf", + LEAF_V1, + r#"export function leafValue() { return "V1_FROM_LEAF"; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/mid", + MID, + r#"import { leafValue } from "./leaf"; +export function midValue() { return `M(${leafValue()})`; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/main_script", + MAIN, + r#"import { midValue } from "./mid"; +export function main() { return midValue(); }"#, + ) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + in_test_worker( + &db, + async move { + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V1_FROM_LEAF)")); + + // Deploy a new version of ONLY the leaf; main_script and mid keep + // their hash, content, and lock byte-identical. + insert_deployed_bun_script( + &db2, + "f/stale_bundle/leaf", + LEAF_V2, + r#"export function leafValue() { return "V2_FROM_LEAF"; }"#, + ) + .await; + + // Tests don't run the notify_event poll loop, so replay what its + // `notify_runnable_version_change` handler (main.rs) does on deploy: + // evict the leaf's latest-hash cache entries. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&( + "test-workspace".to_string(), + "f/stale_bundle/leaf".to_string(), + )); + windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE + .remove(&format!("test-workspace:f/stale_bundle/leaf")); + + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V2_FROM_LEAF)")); + }, + port, + ) + .await; + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/tests/freshness_watchdog.rs b/backend/tests/freshness_watchdog.rs new file mode 100644 index 0000000000..17851d0a1e --- /dev/null +++ b/backend/tests/freshness_watchdog.rs @@ -0,0 +1,294 @@ +//! End-to-end tests for the pipeline freshness watchdog (Enterprise). +//! +//! `windmill_queue::freshness_watchdog::tick` is called directly against +//! seeded `script` / `v2_job(_completed)` rows — no worker or API server is +//! needed, since the watchdog's job ends at the push (the pushed job sitting +//! in `v2_job_queue` is itself part of the assertions). Covers: staleness on +//! never-ran and aged-out members, the fresh short-circuit + state reset, +//! the in-flight suppression, the backoff claim, and the skip rules +//! (partitioned, malformed window, non-pipeline scripts). + +#![cfg(feature = "private")] + +use sqlx::{Pool, Postgres}; +use windmill_queue::freshness_watchdog::tick; +use windmill_test_utils::initialize_tracing; + +const WS: &str = "test-workspace"; +const PATH: &str = "u/test-user/freshness_producer"; + +/// Seed a deployed pipeline-member script. Mirrors the deploy path's output: +/// `auto_kind = 'pipeline'`, empty (non-NULL) lock so run-by-path resolution +/// treats it as deployed, hash derived from path+content for uniqueness. +async fn seed_pipeline_script( + db: &Pool, + path: &str, + content: &str, +) -> anyhow::Result<()> { + let mut h = 0i64; + for b in path.bytes().chain(content.bytes()) { + h = h.wrapping_mul(31).wrapping_add(b as i64); + } + sqlx::query( + r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, tag, lock, auto_kind) + VALUES ($1, $2, $3, '', '', $4, 'test-user', 'bash'::script_lang, 'bash', '', 'pipeline') + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(h) + .bind(path) + .bind(content) + .execute(db) + .await?; + // Process-global deployed-script caches are keyed by (workspace, path) / + // (workspace, hash) and would leak between #[sqlx::test] isolated DBs + // that reuse both — resolve everything from this test's own DB. + windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) +} + +/// Seed a completed root run of `path` that finished `age_s` seconds ago. +async fn seed_completed_run( + db: &Pool, + path: &str, + age_s: i64, + success: bool, +) -> anyhow::Result<()> { + let id = uuid::Uuid::new_v4(); + sqlx::query( + r#"INSERT INTO v2_job (id, workspace_id, runnable_path, kind, created_at, + created_by, permissioned_as, permissioned_as_email, tag) + VALUES ($1, $2, $3, 'script'::job_kind, + now() - ($4::bigint::text || ' seconds')::interval, + 'test-user', 'u/test-user', 'test@windmill.dev', 'bash')"#, + ) + .bind(id) + .bind(WS) + .bind(path) + .bind(age_s) + .execute(db) + .await?; + sqlx::query( + r#"INSERT INTO v2_job_completed (id, workspace_id, duration_ms, status, started_at, completed_at) + VALUES ($1, $2, 0, CASE WHEN $3 THEN 'success'::job_status ELSE 'failure'::job_status END, + now() - ($4::bigint::text || ' seconds')::interval, + now() - ($4::bigint::text || ' seconds')::interval)"#, + ) + .bind(id) + .bind(WS) + .bind(success) + .bind(age_s) + .execute(db) + .await?; + Ok(()) +} + +/// Jobs the watchdog pushed: (path, created_by, args) rows attributed to +/// `trigger_kind = 'freshness'`. +async fn fetch_pushed( + db: &Pool, +) -> anyhow::Result)>> { + let rows = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", created_by AS "created_by!", + args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'freshness' + ORDER BY created_at"#, + WS, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.runnable_path, r.created_by, r.args.map(|a| a.0))) + .collect()) +} + +async fn state_row(db: &Pool, path: &str) -> anyhow::Result> { + Ok(sqlx::query_scalar!( + "SELECT attempts FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path, + ) + .fetch_optional(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn never_ran_member_is_pushed_once(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 30s\necho hi\n").await?; + + tick(&db).await; + + let pushed = fetch_pushed(&db).await?; + assert_eq!(pushed.len(), 1, "one watchdog push expected"); + let (path, created_by, args) = &pushed[0]; + assert_eq!(path, PATH); + assert_eq!(created_by, &format!("freshness-{PATH}")); + let args = args.as_ref().expect("args recorded"); + assert_eq!( + args.get("_wmill_skip_asset_dispatch"), + Some(&serde_json::json!(true)), + "watchdog runs must not re-fire the cascade" + ); + assert_eq!( + args.pointer("/trigger/kind"), + Some(&serde_json::json!("freshness")) + ); + assert_eq!(state_row(&db, PATH).await?, Some(1), "claim row recorded"); + + // Second tick: the pushed job is queued-and-due, so the in-flight guard + // suppresses a duplicate regardless of backoff. + tick(&db).await; + assert_eq!( + fetch_pushed(&db).await?.len(), + 1, + "no duplicate while queued" + ); + + // Simulate the queued job vanishing without a completion: the backoff + // claim (next_attempt_at in the future) now carries the suppression. + sqlx::query!("DELETE FROM v2_job_queue WHERE workspace_id = $1", WS) + .execute(&db) + .await?; + tick(&db).await; + assert_eq!(fetch_pushed(&db).await?.len(), 1, "backoff holds the retry"); + + // Force the backoff window open: the watchdog retries and escalates. + sqlx::query!( + "UPDATE pipeline_freshness_state SET next_attempt_at = now() - interval '1 second' + WHERE workspace_id = $1 AND script_path = $2", + WS, + PATH, + ) + .execute(&db) + .await?; + tick(&db).await; + assert_eq!(fetch_pushed(&db).await?.len(), 2, "due retry pushed"); + assert_eq!(state_row(&db, PATH).await?, Some(2), "attempts escalated"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn fresh_member_is_skipped_and_state_reset(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 1h\necho hi\n").await?; + seed_completed_run(&db, PATH, 10, true).await?; + // Leftover backoff row from an earlier staleness episode. + sqlx::query!( + "INSERT INTO pipeline_freshness_state (workspace_id, script_path, attempts) VALUES ($1, $2, 3)", + WS, + PATH, + ) + .execute(&db) + .await?; + + tick(&db).await; + + assert!( + fetch_pushed(&db).await?.is_empty(), + "fresh member not pushed" + ); + assert_eq!(state_row(&db, PATH).await?, None, "backoff reset on fresh"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn aged_out_member_is_pushed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 1h\necho hi\n").await?; + // Old success outside the window + a recent failure: still stale. + seed_completed_run(&db, PATH, 7200, true).await?; + seed_completed_run(&db, PATH, 60, false).await?; + + tick(&db).await; + + assert_eq!(fetch_pushed(&db).await?.len(), 1, "aged-out member pushed"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn partitioned_malformed_and_plain_members_are_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Partitioned: freshness means partition-gap detection, out of scope. + seed_pipeline_script( + &db, + "u/test-user/partitioned", + "# pipeline\n# partitioned daily\n# freshness 1h\necho hi\n", + ) + .await?; + // Malformed window: fails safe to unwatched. + seed_pipeline_script( + &db, + "u/test-user/malformed", + "# pipeline\n# freshness soonish\necho hi\n", + ) + .await?; + // Freshness only in prose (parser must reject; ILIKE prefilter passes). + seed_pipeline_script( + &db, + "u/test-user/prose", + "# pipeline\n# ensure freshness of data below\necho hi\n", + ) + .await?; + + tick(&db).await; + + assert!(fetch_pushed(&db).await?.is_empty(), "no member is watched"); + let rows = sqlx::query_scalar!( + r#"SELECT COUNT(*) AS "count!" FROM pipeline_freshness_state WHERE workspace_id = $1"#, + WS, + ) + .fetch_one(&db) + .await?; + assert_eq!(rows, 0, "no state rows for unwatched members"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn archived_workspace_is_not_resurrected(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // Workspace archival stops all execution but keeps script rows for + // unarchival — the watchdog must not keep pushing runs there. + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 30s\necho hi\n").await?; + sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", WS) + .execute(&db) + .await?; + + tick(&db).await; + + assert!( + fetch_pushed(&db).await?.is_empty(), + "no pushes into an archived workspace" + ); + assert_eq!( + state_row(&db, PATH).await?, + None, + "no state bookkeeping either" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn state_of_unwatched_member_is_cleaned_up(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // A stale-bookkeeping row whose script no longer declares freshness + // (e.g. annotation removed and redeployed) must not survive the sweep. + sqlx::query!( + "INSERT INTO pipeline_freshness_state (workspace_id, script_path) VALUES ($1, $2)", + WS, + "u/test-user/gone", + ) + .execute(&db) + .await?; + + tick(&db).await; + + assert_eq!(state_row(&db, "u/test-user/gone").await?, None); + Ok(()) +} diff --git a/backend/tests/preserve_on_behalf_of.rs b/backend/tests/preserve_on_behalf_of.rs index 7ed5fc7697..d808348937 100644 --- a/backend/tests/preserve_on_behalf_of.rs +++ b/backend/tests/preserve_on_behalf_of.rs @@ -2704,7 +2704,9 @@ async fn test_schedule_permissions_workspace_admin(db: Pool) -> anyhow Ok(()) } -/// Superadmin NOT in workspace creates a schedule — uses email as permissioned_as +/// Superadmin NOT in workspace creates a schedule — uses their instance-derived +/// username (`password.username`) as permissioned_as, not the raw email. The +/// email is still stored directly on the schedule for downstream resolution. #[sqlx::test(fixtures("preserve_on_behalf_of"))] async fn test_schedule_permissions_superadmin_not_in_workspace( db: Pool, @@ -2758,16 +2760,16 @@ async fn test_schedule_permissions_superadmin_not_in_workspace( .fetch_one(&db) .await?; - // Superadmin not in workspace: username_to_permissioned_as uses the email directly - // since the authed username for a superadmin not in workspace IS the email + // Superadmin not in workspace: the authed username is now their instance-derived + // username (`password.username` = 'superadmin-external'), so permissioned_as is + // `u/` rather than the raw email. The email is still stored directly. assert_eq!( schedule.email, "superadmin-external@windmill.dev", "schedule email should be superadmin email" ); assert_eq!( - schedule.permissioned_as, - schedule.email.clone(), - "permissioned_as should match email for superadmin not in workspace" + schedule.permissioned_as, "u/superadmin-external", + "permissioned_as should use the instance-derived username, not the email" ); // Update by the same superadmin diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index 8aa9addcae..02ab702d31 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -857,22 +857,91 @@ pub fn bedrock_stream_event_is_block_stop(event: &ConverseStreamOutput) -> bool matches!(event, ConverseStreamOutput::ContentBlockStop(_)) } -/// Convert accumulated streaming tool calls to OpenAI format -pub fn streaming_tool_calls_to_openai(tool_calls: Vec) -> Vec { +/// Convert accumulated streaming tool calls to OpenAI format. +/// +/// When thinking is enabled, `reasoning` carries the turn's Claude reasoning +/// block; it is attached to the first tool call so the next request replays it +/// before `toolUse` (required by Claude when thinking is enabled — see +/// [`convert_message`]). When `None` (thinking off), tool calls carry no extra +/// content, byte-identical to the pre-feature output. +pub fn streaming_tool_calls_to_openai( + tool_calls: Vec, + reasoning: Option, +) -> Vec { + let mut reasoning = reasoning; tool_calls .into_iter() - .map(|tc| OpenAIToolCall { - id: tc.id, - function: OpenAIFunction { name: tc.name, arguments: tc.arguments }, - r#type: FUNCTION_TYPE.to_string(), - // Worker agent requests never enable thinking, so there is no - // reasoning block to round-trip here (the chat proxy path does — - // see providers/bedrock.rs). - extra_content: None, + .map(|tc| { + // `take` so only the first tool call carries the reasoning block. + let extra_content = reasoning.take().map(|block| crate::ai_types::ExtraContent { + bedrock: Some(block), + ..Default::default() + }); + OpenAIToolCall { + id: tc.id, + function: OpenAIFunction { name: tc.name, arguments: tc.arguments }, + r#type: FUNCTION_TYPE.to_string(), + extra_content, + } }) .collect() } +/// Fold a Bedrock `ReasoningContent` stream delta into an accumulating reasoning +/// block (text + signature, or redacted bytes). Returns the readable text delta +/// when the event carried one, so the caller can stream it as reasoning content. +/// Mirrors the proxy path's accumulation in `providers/bedrock.rs`. +pub fn bedrock_stream_event_to_reasoning_delta( + event: &ConverseStreamOutput, + reasoning: &mut Option, +) -> Option { + let ConverseStreamOutput::ContentBlockDelta(delta_event) = event else { + return None; + }; + let aws_sdk_bedrockruntime::types::ContentBlockDelta::ReasoningContent(rc) = + delta_event.delta()? + else { + return None; + }; + + let entry = reasoning.get_or_insert_with(Default::default); + match rc { + aws_sdk_bedrockruntime::types::ReasoningContentBlockDelta::Text(text) => { + entry + .reasoning_text + .get_or_insert_with(String::new) + .push_str(text); + Some(text.clone()) + } + aws_sdk_bedrockruntime::types::ReasoningContentBlockDelta::Signature(signature) => { + entry + .signature + .get_or_insert_with(String::new) + .push_str(signature); + None + } + aws_sdk_bedrockruntime::types::ReasoningContentBlockDelta::RedactedContent(blob) => { + // Base64 of concatenated fragments != concatenated base64 fragments, + // so accumulate raw bytes and re-encode. + let mut bytes = entry + .redacted_content + .as_deref() + .and_then(|existing| { + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, existing) + .ok() + }) + .unwrap_or_default(); + bytes.extend_from_slice(blob.as_ref()); + entry.redacted_content = Some(base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + bytes, + )); + None + } + _ => None, + } +} + // ============================================================================ // Tool Configuration Builder // ============================================================================ @@ -1028,6 +1097,101 @@ mod tests { assert!(matches!(&content[1], ContentBlock::ToolUse(_))); } + #[test] + fn streaming_tool_calls_to_openai_attaches_reasoning_to_first_call_only() { + let calls = vec![ + StreamingToolCall { + id: "call_1".to_string(), + name: "a".to_string(), + arguments: "{}".to_string(), + }, + StreamingToolCall { + id: "call_2".to_string(), + name: "b".to_string(), + arguments: "{}".to_string(), + }, + ]; + let reasoning = crate::ai_types::BedrockExtraContent { + reasoning_text: Some("thinking".to_string()), + signature: Some("sig".to_string()), + redacted_content: None, + }; + + let openai = streaming_tool_calls_to_openai(calls, Some(reasoning)); + + let with_reasoning = openai + .iter() + .filter(|tc| { + tc.extra_content + .as_ref() + .and_then(|e| e.bedrock.as_ref()) + .is_some() + }) + .count(); + assert_eq!( + with_reasoning, 1, + "exactly one tool call carries the reasoning block" + ); + let block = openai[0] + .extra_content + .as_ref() + .and_then(|e| e.bedrock.as_ref()) + .expect("first tool call carries reasoning"); + assert_eq!(block.reasoning_text.as_deref(), Some("thinking")); + assert_eq!(block.signature.as_deref(), Some("sig")); + } + + #[test] + fn streaming_tool_calls_to_openai_without_reasoning_has_no_extra_content() { + let calls = vec![StreamingToolCall { + id: "call_1".to_string(), + name: "a".to_string(), + arguments: "{}".to_string(), + }]; + + let openai = streaming_tool_calls_to_openai(calls, None); + + assert!(openai[0].extra_content.is_none()); + } + + #[test] + fn bedrock_reasoning_delta_accumulates_text_and_signature() { + use aws_sdk_bedrockruntime::types::{ + ContentBlockDelta, ContentBlockDeltaEvent, ConverseStreamOutput, + ReasoningContentBlockDelta, + }; + + let mut reasoning = None; + let text_event = ConverseStreamOutput::ContentBlockDelta( + ContentBlockDeltaEvent::builder() + .content_block_index(0) + .delta(ContentBlockDelta::ReasoningContent( + ReasoningContentBlockDelta::Text("let me ".to_string()), + )) + .build() + .unwrap(), + ); + assert_eq!( + bedrock_stream_event_to_reasoning_delta(&text_event, &mut reasoning).as_deref(), + Some("let me ") + ); + + let sig_event = ConverseStreamOutput::ContentBlockDelta( + ContentBlockDeltaEvent::builder() + .content_block_index(0) + .delta(ContentBlockDelta::ReasoningContent( + ReasoningContentBlockDelta::Signature("sig".to_string()), + )) + .build() + .unwrap(), + ); + assert!(bedrock_stream_event_to_reasoning_delta(&sig_event, &mut reasoning).is_none()); + + let block = reasoning.expect("reasoning accumulated"); + assert_eq!(block.reasoning_text.as_deref(), Some("let me ")); + assert_eq!(block.signature.as_deref(), Some("sig")); + } + #[test] fn openai_messages_to_bedrock_skips_cache_points_when_disabled() { let messages = vec![ diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index 94d3ef65e0..4374cb8628 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -312,7 +312,7 @@ impl GeminiToolCallEvent { pub fn to_extra_content(&self) -> Option { self.thought_signature.as_ref().map(|sig| ExtraContent { google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }), - bedrock: None, + ..Default::default() }) } } @@ -1001,10 +1001,8 @@ mod tests { #[test] fn gemini_streaming_without_usage_emits_no_usage_chunk() { - let parsed = GeminiParsedEvent { - text: Some("the answer".to_string()), - ..Default::default() - }; + let parsed = + GeminiParsedEvent { text: Some("the answer".to_string()), ..Default::default() }; let mut tool_call_index = 0; let chunks = gemini_event_to_openai_sse_chunks( diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index 5d8e763b69..551ba3129b 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -49,6 +49,8 @@ pub enum AIProvider { OpenAI, #[serde(rename = "azure_openai")] AzureOpenAI, + #[serde(rename = "azure_foundry")] + AzureFoundry, Anthropic, Mistral, DeepSeek, @@ -114,9 +116,12 @@ impl AIProvider { AIProvider::TogetherAI => Ok("https://api.together.xyz/v1".to_string()), AIProvider::Anthropic => Ok("https://api.anthropic.com/v1".to_string()), AIProvider::Mistral => Ok("https://api.mistral.ai/v1".to_string()), - p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI) => Err(Error::BadRequest( - format!("{:?} provider requires a base URL in the resource", p), - )), + p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI | AIProvider::AzureFoundry) => { + Err(Error::BadRequest(format!( + "{:?} provider requires a base URL in the resource", + p + ))) + } AIProvider::AWSBedrock => { // AWS Bedrock uses the SDK directly, not HTTP base URL Err(Error::internal_err( @@ -131,24 +136,84 @@ impl AIProvider { matches!(self, AIProvider::Anthropic) } - /// Check if this provider/URL combination represents Azure OpenAI - pub fn is_azure_openai(&self, base_url: &str) -> bool { + /// Check whether this provider/URL combination uses Azure conventions + /// (the `api-key` auth header and Azure URL building). This covers Azure + /// OpenAI, Azure AI Foundry, and the `OpenAI` provider pointed at an Azure + /// base path override. + pub fn is_azure(&self, base_url: &str) -> bool { (matches!(self, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL) - || matches!(self, AIProvider::AzureOpenAI) + || matches!(self, AIProvider::AzureOpenAI | AIProvider::AzureFoundry) } - /// Build Azure OpenAI URL with deployment model path + /// Build an Azure-style OpenAI-compatible URL (Azure OpenAI / Azure AI Foundry) + /// for the given path. The resource base URL may be stored as the bare resource + /// root (e.g. `https://.services.ai.azure.com`) or with a legacy `/openai` + /// or `/openai/v1` suffix (older Foundry resources shipped that way); those forms + /// resolve to the canonical `/openai/v1/`. Any other explicit path + /// (e.g. an Azure OpenAI `.../openai/deployments/` base) is preserved as-is + /// with only `/` appended. pub fn build_azure_openai_url(base_url: &str, path: &str) -> String { let base_url = base_url.trim_end_matches('/'); - if base_url.ends_with("/openai") { + if base_url.ends_with("/openai/v1") { + format!("{}/{}", base_url, path) + } else if base_url.ends_with("/openai") { format!("{}/v1/{}", base_url, path) } else if base_url.ends_with("/deployments") { format!("{}/v1/{}", base_url.trim_end_matches("/deployments"), path) + } else if Self::is_bare_host(base_url) { + // A resource root with no path (Foundry convention, or an Azure OpenAI + // resource root) targets the OpenAI-compatible v1 surface. + format!("{}/openai/v1/{}", base_url, path) } else { + // Any other explicit base path (e.g. an Azure OpenAI deployment URL + // `.../openai/deployments/`) is kept intact. format!("{}/{}", base_url, path) } } + /// Whether the URL is a bare scheme+host with no path component, e.g. + /// `https://x.services.ai.azure.com` (vs `https://x.openai.azure.com/openai/deployments/y`). + fn is_bare_host(url: &str) -> bool { + let after_scheme = url.split_once("://").map(|(_, rest)| rest).unwrap_or(url); + !after_scheme.contains('/') + } + + /// Strip any known Foundry API sub-path from the resource base URL to recover the + /// resource root, so the correct per-model-family path can be appended. Handles + /// both the current root-URL convention and legacy `/openai/v1`-style values. + fn azure_foundry_root(base_url: &str) -> &str { + let base_url = base_url.trim_end_matches('/'); + for suffix in [ + "/openai/v1", + "/anthropic/v1", + "/openai", + "/anthropic", + "/models", + ] { + if let Some(root) = base_url.strip_suffix(suffix) { + return root.trim_end_matches('/'); + } + } + base_url + } + + /// Build an Azure AI Foundry Anthropic Messages API URL. Claude deployments on + /// Foundry are served only through `/anthropic/v1/...`, not the + /// OpenAI-compatible `/openai/v1` surface. + pub fn build_azure_foundry_anthropic_url(base_url: &str, path: &str) -> String { + format!( + "{}/anthropic/v1/{}", + Self::azure_foundry_root(base_url), + path + ) + } + + /// Whether a Foundry deployment name refers to an Anthropic (Claude) model, + /// which requires the Anthropic Messages API rather than OpenAI chat completions. + pub fn is_anthropic_model(model: &str) -> bool { + model.to_lowercase().starts_with("claude") + } + /// Extract model from request body (needed for Azure deployments) pub fn extract_model_from_body(body: &[u8]) -> Result { #[derive(serde::Deserialize)] @@ -186,3 +251,81 @@ pub struct ProviderModel { pub model: String, pub provider: AIProvider, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn azure_openai_url_handles_root_and_legacy_suffixes() { + // Current convention: resource stores the bare root. + assert_eq!( + AIProvider::build_azure_openai_url( + "https://wm-test-ai.services.ai.azure.com", + "chat/completions" + ), + "https://wm-test-ai.services.ai.azure.com/openai/v1/chat/completions" + ); + // Legacy resources that already baked in /openai/v1 must still resolve. + assert_eq!( + AIProvider::build_azure_openai_url( + "https://wm-test-ai.services.ai.azure.com/openai/v1/", + "chat/completions" + ), + "https://wm-test-ai.services.ai.azure.com/openai/v1/chat/completions" + ); + // Azure OpenAI resources typically end in /openai. + assert_eq!( + AIProvider::build_azure_openai_url( + "https://example.openai.azure.com/openai", + "chat/completions" + ), + "https://example.openai.azure.com/openai/v1/chat/completions" + ); + assert_eq!( + AIProvider::build_azure_openai_url( + "https://example.openai.azure.com/openai/deployments", + "chat/completions" + ), + "https://example.openai.azure.com/openai/v1/chat/completions" + ); + // An Azure OpenAI base that pins a specific deployment must be preserved + // as-is (not have /openai/v1 appended after the deployment id). + assert_eq!( + AIProvider::build_azure_openai_url( + "https://example.openai.azure.com/openai/deployments/my-deployment", + "chat/completions" + ), + "https://example.openai.azure.com/openai/deployments/my-deployment/chat/completions" + ); + } + + #[test] + fn azure_foundry_anthropic_url_from_root_and_legacy() { + // Root URL. + assert_eq!( + AIProvider::build_azure_foundry_anthropic_url( + "https://wm-test-ai.services.ai.azure.com", + "messages" + ), + "https://wm-test-ai.services.ai.azure.com/anthropic/v1/messages" + ); + // Legacy /openai/v1 base is normalized back to the root, then routed to + // the Anthropic Messages API. + assert_eq!( + AIProvider::build_azure_foundry_anthropic_url( + "https://wm-test-ai.services.ai.azure.com/openai/v1", + "messages" + ), + "https://wm-test-ai.services.ai.azure.com/anthropic/v1/messages" + ); + } + + #[test] + fn detects_anthropic_models() { + assert!(AIProvider::is_anthropic_model("claude-sonnet-5")); + assert!(AIProvider::is_anthropic_model("Claude-Opus-4-8")); + assert!(!AIProvider::is_anthropic_model("gpt-4o")); + assert!(!AIProvider::is_anthropic_model("DeepSeek-R1")); + } +} diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index c6f171daf3..b80e237920 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -118,6 +118,22 @@ pub struct BedrockExtraContent { pub redacted_content: Option, } +/// Native-Anthropic reasoning block emitted in the same assistant turn as a +/// tool call. Like Bedrock, Anthropic requires the thinking block (text + +/// unmodified signature, or redacted bytes) to precede `tool_use` on replay when +/// thinking is enabled, so it is round-tripped through the OpenAI-shaped tool call. +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +pub struct AnthropicExtraContent { + #[serde(skip_serializing_if = "Option::is_none")] + pub thinking: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature: Option, + /// Base64 `data` of a redacted (encrypted) thinking block, when the provider + /// returned one instead of readable text. + #[serde(skip_serializing_if = "Option::is_none")] + pub redacted_thinking: Option, +} + /// Extra content for provider-specific metadata (e.g., Google thought signatures) #[derive(Deserialize, Serialize, Clone, Debug, Default)] pub struct ExtraContent { @@ -125,6 +141,8 @@ pub struct ExtraContent { pub google: Option, #[serde(skip_serializing_if = "Option::is_none")] pub bedrock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub anthropic: Option, } #[derive(Deserialize, Serialize, Clone, Debug)] diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 05398ce212..d71de2f35f 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -92,6 +92,10 @@ pub enum AnthropicRequestContent { #[serde(skip_serializing_if = "Option::is_none")] cache_control: Option, }, + #[serde(rename = "thinking")] + Thinking { thinking: String, signature: String }, + #[serde(rename = "redacted_thinking")] + RedactedThinking { data: String }, #[serde(rename = "image")] Image { source: AnthropicBase64Source }, #[serde(rename = "document")] @@ -131,6 +135,26 @@ pub struct AnthropicMessage { pub content: Vec, } +/// Adaptive thinking config for Anthropic native API. `summarized` display +/// matches the chat proxy path (renders a summarized thinking stream). +#[derive(Serialize, Debug)] +pub struct AnthropicThinking { + pub r#type: &'static str, + pub display: &'static str, +} + +impl AnthropicThinking { + fn adaptive() -> Self { + Self { r#type: "adaptive", display: "summarized" } + } +} + +/// Carries the reasoning effort token alongside adaptive thinking. +#[derive(Serialize, Debug)] +pub struct AnthropicOutputConfig { + pub effort: String, +} + /// Anthropic-specific request structure for standard API #[derive(Serialize)] pub struct AnthropicRequest<'a> { @@ -145,6 +169,10 @@ pub struct AnthropicRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub thinking: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_config: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens: Option, pub stream: bool, } @@ -166,6 +194,10 @@ pub struct AnthropicVertexRequest { #[serde(skip_serializing_if = "Option::is_none")] pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub thinking: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_config: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens: Option, pub stream: bool, } @@ -189,6 +221,25 @@ fn convert_messages_to_anthropic(messages: &[OpenAIMessage]) -> Vec { let mut content: Vec = Vec::new(); + // Replay the turn's thinking block first: when thinking is enabled, + // Claude requires the thinking block (with its unmodified signature) + // to precede tool_use in the assistant turn it was emitted in. It is + // round-tripped on the tool call's extra_content (see AnthropicExtraContent). + if let Some(reasoning) = msg + .tool_calls + .as_ref() + .and_then(|tcs| { + tcs.iter().find_map(|tc| { + tc.extra_content + .as_ref() + .and_then(|ec| ec.anthropic.as_ref()) + }) + }) + .and_then(anthropic_reasoning_block_from_extra) + { + content.push(reasoning); + } + // Add text content if present if let Some(ref c) = msg.content { let text = extract_text_content(c); @@ -246,6 +297,25 @@ fn convert_messages_to_anthropic(messages: &[OpenAIMessage]) -> Vec Option { + if let Some(data) = extra.redacted_thinking.as_deref() { + return Some(AnthropicRequestContent::RedactedThinking { data: data.to_string() }); + } + match (extra.thinking.as_deref(), extra.signature.as_deref()) { + (Some(thinking), Some(signature)) => Some(AnthropicRequestContent::Thinking { + thinking: thinking.to_string(), + signature: signature.to_string(), + }), + _ => None, + } +} + /// Convert OpenAI content to Anthropic content blocks fn convert_content_to_anthropic(content: &Option) -> Vec { let Some(content) = content else { @@ -380,6 +450,13 @@ impl AnthropicQueryBuilder { self.platform == AIPlatform::GoogleVertexAi } + /// Claude models hosted on Azure AI Foundry: the Anthropic Messages API is + /// served under the resource's `/anthropic/v1` path rather than at the + /// Anthropic public base URL. + fn is_azure_foundry(&self) -> bool { + matches!(self.provider_kind, AIProvider::AzureFoundry) + } + fn transform_proxy_body_for_vertex(body: &[u8]) -> Result<(String, Vec), Error> { let mut json_body: std::collections::HashMap = serde_json::from_slice(body).map_err(|e| { @@ -426,6 +503,15 @@ impl AnthropicQueryBuilder { format!("{}/{}:streamRawPredict", base_url, model), transformed_body, ) + } else if self.is_azure_foundry() { + // Claude on Foundry is served under the resource's /anthropic/v1 path, + // not the resource's /openai/v1 base. The Anthropic SDK sends the path + // as "v1/messages", so drop its leading "v1/" before re-appending. + let path = args.path.trim_start_matches("v1/"); + ( + AIProvider::build_azure_foundry_anthropic_url(base_url, path), + body, + ) } else if is_anthropic_sdk { let truncated_base_url = base_url.trim_end_matches("/v1"); (format!("{}/{}", truncated_base_url, args.path), body) @@ -575,6 +661,17 @@ impl AnthropicQueryBuilder { } } + // Adaptive thinking rejects sampling params, so drop temperature when + // reasoning is on (Anthropic returns a hard 400 otherwise). + let (thinking, output_config, temperature) = match args.reasoning_effort { + Some(effort) => ( + Some(AnthropicThinking::adaptive()), + Some(AnthropicOutputConfig { effort: effort.to_string() }), + None, + ), + None => (None, None, args.temperature), + }; + // Build request based on platform if self.is_vertex() { // For Vertex AI: no model field, anthropic_version in body @@ -584,7 +681,9 @@ impl AnthropicQueryBuilder { messages: anthropic_messages, tools: tools_option, tool_choice, - temperature: args.temperature, + temperature, + thinking, + output_config, max_tokens, stream: true, }; @@ -598,7 +697,9 @@ impl AnthropicQueryBuilder { messages: anthropic_messages, tools: tools_option, tool_choice, - temperature: args.temperature, + temperature, + thinking, + output_config, max_tokens, stream: true, }; @@ -688,6 +789,8 @@ impl QueryBuilder for AnthropicQueryBuilder { base_url.trim_end_matches('/'), model ) + } else if self.is_azure_foundry() { + AIProvider::build_azure_foundry_anthropic_url(base_url, "messages") } else { format!("{}/messages", base_url) } @@ -748,8 +851,7 @@ mod tests { #[test] fn builds_standard_anthropic_proxy_request() { let credentials = credentials(AIPlatform::Standard); - let builder = - AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard); + let builder = AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard); let method = Method::POST; let mut headers = HeaderMap::new(); headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01")); @@ -786,8 +888,7 @@ mod tests { let mut credentials = credentials(AIPlatform::GoogleVertexAi); credentials.base_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/publishers/anthropic/models".to_string(); credentials.user = Some("user-1".to_string()); - let builder = - AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::GoogleVertexAi); + let builder = AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::GoogleVertexAi); let method = Method::POST; let mut headers = HeaderMap::new(); headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01")); @@ -827,11 +928,127 @@ mod tests { assert!(has_header(&request.headers, "anthropic-beta", "some-beta")); } + #[test] + fn convert_messages_replays_thinking_block_before_tool_use() { + let tool_call = crate::ai_types::OpenAIToolCall { + id: "call_1".to_string(), + function: crate::ai_types::OpenAIFunction { + name: "lookup".to_string(), + arguments: "{}".to_string(), + }, + r#type: "function".to_string(), + extra_content: Some(crate::ai_types::ExtraContent { + anthropic: Some(crate::ai_types::AnthropicExtraContent { + thinking: Some("let me think".to_string()), + signature: Some("sig-abc".to_string()), + redacted_thinking: None, + }), + ..Default::default() + }), + }; + let assistant = OpenAIMessage { + role: "assistant".to_string(), + tool_calls: Some(vec![tool_call]), + ..Default::default() + }; + + let messages = convert_messages_to_anthropic(&[assistant]); + let content = &messages[0].content; + match &content[0] { + AnthropicRequestContent::Thinking { thinking, signature } => { + assert_eq!(thinking, "let me think"); + assert_eq!(signature, "sig-abc"); + } + other => panic!("expected thinking block first, got {:?}", other), + } + assert!(matches!( + &content[1], + AnthropicRequestContent::ToolUse { .. } + )); + } + + #[test] + fn convert_messages_drops_thinking_without_signature() { + let tool_call = crate::ai_types::OpenAIToolCall { + id: "call_1".to_string(), + function: crate::ai_types::OpenAIFunction { + name: "lookup".to_string(), + arguments: "{}".to_string(), + }, + r#type: "function".to_string(), + extra_content: Some(crate::ai_types::ExtraContent { + anthropic: Some(crate::ai_types::AnthropicExtraContent { + thinking: Some("unsigned".to_string()), + signature: None, + redacted_thinking: None, + }), + ..Default::default() + }), + }; + let assistant = OpenAIMessage { + role: "assistant".to_string(), + tool_calls: Some(vec![tool_call]), + ..Default::default() + }; + + let messages = convert_messages_to_anthropic(&[assistant]); + // An unsigned thinking block can't be replayed, so only tool_use remains. + assert!(matches!( + messages[0].content[0], + AnthropicRequestContent::ToolUse { .. } + )); + } + + #[test] + fn anthropic_request_serializes_adaptive_thinking_without_temperature() { + let request = AnthropicRequest { + model: "claude-opus-4-8", + system: None, + messages: vec![], + tools: None, + tool_choice: None, + temperature: None, + thinking: Some(AnthropicThinking::adaptive()), + output_config: Some(AnthropicOutputConfig { effort: "high".to_string() }), + max_tokens: Some(64000), + stream: true, + }; + + let body: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap(); + assert_eq!(body["thinking"]["type"], "adaptive"); + assert_eq!(body["thinking"]["display"], "summarized"); + assert_eq!(body["output_config"]["effort"], "high"); + // Sampling params are rejected alongside adaptive thinking. + assert!(body.get("temperature").is_none()); + } + + #[test] + fn anthropic_request_omits_thinking_when_reasoning_off() { + let request = AnthropicRequest { + model: "claude-opus-4-8", + system: None, + messages: vec![], + tools: None, + tool_choice: None, + temperature: Some(0.5), + thinking: None, + output_config: None, + max_tokens: Some(64000), + stream: true, + }; + + let body: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap(); + assert!(body.get("thinking").is_none()); + assert!(body.get("output_config").is_none()); + assert_eq!(body["temperature"], 0.5); + } + #[test] fn rejects_vertex_proxy_request_without_model() { let credentials = credentials(AIPlatform::GoogleVertexAi); - let builder = - AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::GoogleVertexAi); + let builder = AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::GoogleVertexAi); let method = Method::POST; let headers = HeaderMap::new(); @@ -847,4 +1064,35 @@ mod tests { assert!(matches!(err, Error::BadRequest(message) if message.contains("Missing 'model'"))); } + + #[test] + fn builds_azure_foundry_anthropic_proxy_request() { + // Foundry resource stored with a legacy /openai/v1 suffix; the Anthropic SDK + // sends the path as "v1/messages". Both must resolve to the resource's + // /anthropic/v1/messages surface. + let mut credentials = credentials(AIPlatform::Standard); + credentials.provider = AIProvider::AzureFoundry; + credentials.base_url = "https://wm-test-ai.services.ai.azure.com/openai/v1".to_string(); + let builder = AnthropicQueryBuilder::new(AIProvider::AzureFoundry, AIPlatform::Standard); + let method = Method::POST; + let mut headers = HeaderMap::new(); + headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01")); + headers.insert("X-Anthropic-SDK", HeaderValue::from_static("true")); + + let request = builder + .build_proxy_request(&ProxyBuildArgs { + method: &method, + path: "v1/messages", + headers: &headers, + body: br#"{"model":"claude-sonnet-5","messages":[]}"#, + credentials: &credentials, + }) + .unwrap(); + + assert_eq!( + request.url, + "https://wm-test-ai.services.ai.azure.com/anthropic/v1/messages" + ); + assert!(has_header(&request.headers, "X-API-Key", "api-key")); + } } diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index ddc5f33998..4405a50dcb 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -9,10 +9,10 @@ use crate::{ ai_bedrock::{ bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop, - bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, - bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start, - bedrock_stream_event_to_tool_start_with_block_index, build_tool_config, - create_inference_config, format_bedrock_error, json_to_document, + bedrock_stream_event_to_reasoning_delta, bedrock_stream_event_to_text, + bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_delta_with_block_index, + bedrock_stream_event_to_tool_start, bedrock_stream_event_to_tool_start_with_block_index, + build_tool_config, create_inference_config, format_bedrock_error, json_to_document, openai_messages_to_bedrock, streaming_tool_calls_to_openai, BearerTokenProvider, BedrockClient, StreamingToolCall, }, @@ -641,57 +641,13 @@ fn bedrock_sse_chunks_for_event( /// Fold a `ReasoningContent` stream delta into the state's pending reasoning /// block. Returns the text delta (for a `reasoning_content` SSE chunk) when the -/// event carried readable reasoning text. +/// event carried readable reasoning text. Shares the worker path's folding so +/// the two never drift. fn accumulate_reasoning_delta( event: &aws_sdk_bedrockruntime::types::ConverseStreamOutput, state: &mut BedrockSseStreamState, ) -> Option { - let aws_sdk_bedrockruntime::types::ConverseStreamOutput::ContentBlockDelta(delta_event) = event - else { - return None; - }; - let aws_sdk_bedrockruntime::types::ContentBlockDelta::ReasoningContent(reasoning) = - delta_event.delta()? - else { - return None; - }; - - let entry = state.reasoning.get_or_insert_with(Default::default); - match reasoning { - aws_sdk_bedrockruntime::types::ReasoningContentBlockDelta::Text(text) => { - entry - .reasoning_text - .get_or_insert_with(String::new) - .push_str(text); - Some(text.clone()) - } - aws_sdk_bedrockruntime::types::ReasoningContentBlockDelta::Signature(signature) => { - entry - .signature - .get_or_insert_with(String::new) - .push_str(signature); - None - } - aws_sdk_bedrockruntime::types::ReasoningContentBlockDelta::RedactedContent(blob) => { - // Base64 of concatenated fragments != concatenated base64 fragments, - // so accumulate raw bytes and re-encode. - let mut bytes = entry - .redacted_content - .as_deref() - .and_then(|existing| { - base64::Engine::decode(&base64::engine::general_purpose::STANDARD, existing) - .ok() - }) - .unwrap_or_default(); - bytes.extend_from_slice(blob.as_ref()); - entry.redacted_content = Some(base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - bytes, - )); - None - } - _ => None, - } + bedrock_stream_event_to_reasoning_delta(event, &mut state.reasoning) } async fn handle_bedrock_sdk_non_streaming( @@ -942,6 +898,7 @@ impl BedrockQueryBuilder { tools: Option<&[ToolDef]>, model: &str, temperature: Option, + reasoning_effort: Option<&str>, max_tokens: Option, api_key: &str, region: &str, @@ -977,6 +934,9 @@ impl BedrockQueryBuilder { let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&prepared_messages, enable_prompt_caching)?; + // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. + let temperature = reasoning_effort.is_none().then_some(temperature).flatten(); + // Build inference configuration using shared helper let inference_config = create_inference_config(temperature, max_tokens.map(|t| t as i32)); @@ -994,6 +954,7 @@ impl BedrockQueryBuilder { system_prompts, inference_config, tool_config, + reasoning_effort, stream_event_sink, ) .await @@ -1008,6 +969,7 @@ impl BedrockQueryBuilder { system_prompts: Vec, inference_config: Option, tool_config: Option, + reasoning_effort: Option<&str>, stream_event_sink: Option>, ) -> Result { tracing::debug!( @@ -1035,6 +997,11 @@ impl BedrockQueryBuilder { request_builder = request_builder.set_tool_config(Some(config)); } + if let Some(effort) = reasoning_effort { + request_builder = + request_builder.additional_model_request_fields(bedrock_thinking_fields(effort)); + } + let mut stream = request_builder .send() .await @@ -1053,11 +1020,32 @@ impl BedrockQueryBuilder { let mut accumulated_tool_calls: HashMap = HashMap::new(); let mut current_tool_use_id: Option = None; let mut usage: Option = None; + // Claude reasoning block for the turn (only populated when thinking is on), + // attached to the first tool call for replay before toolUse. + let mut reasoning: Option = None; // Process stream events using shared parsing functions loop { match stream.recv().await { Ok(Some(event)) => { + // Fold reasoning deltas into the turn's reasoning block (for + // replay before toolUse, required by Claude when thinking is + // on) and stream the readable summary as a thinking affordance. + if let Some(reasoning_delta) = + bedrock_stream_event_to_reasoning_delta(&event, &mut reasoning) + { + if let Some(processor) = stream_event_sink.as_ref() { + processor + .send( + StreamingEvent::ReasoningTokenDelta { + content: reasoning_delta, + }, + &mut events_str, + ) + .await?; + } + } + // Handle tool use start using shared parser if let Some(tool_call) = bedrock_stream_event_to_tool_start(&event) { current_tool_use_id = Some(tool_call.id.clone()); @@ -1143,8 +1131,10 @@ impl BedrockQueryBuilder { Some(accumulated_text) }; - let tool_calls = - streaming_tool_calls_to_openai(accumulated_tool_calls.into_values().collect()); + let tool_calls = streaming_tool_calls_to_openai( + accumulated_tool_calls.into_values().collect(), + reasoning, + ); Ok(ParsedResponse::Text { content, diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 650e11c0c7..21428903fa 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -153,13 +153,18 @@ impl GoogleAIQueryBuilder { (None, None) }; + // Map the effort token onto Gemini's native thinking controls; without + // one, leave provider defaults untouched. + let thinking_config = args + .reasoning_effort + .map(|effort| gemini_thinking_config(args.model, effort)); + build_gemini_generation_config( args.temperature, args.max_tokens, response_mime_type, response_schema, - // Worker AI-agent requests carry no reasoning knob; keep provider defaults. - None, + thinking_config, ) } } diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index 80fbc8fcca..ab4f670b17 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -7,7 +7,9 @@ pub mod openrouter; pub mod other; use crate::{ - ai_providers::AIProvider, credentials::ProviderCredentials, query_builder::QueryBuilder, + ai_providers::{AIPlatform, AIProvider}, + credentials::ProviderCredentials, + query_builder::QueryBuilder, }; use self::{ @@ -16,7 +18,15 @@ use self::{ }; /// Factory function to create the appropriate query builder from resolved credentials. -pub fn create_query_builder(credentials: &ProviderCredentials) -> Box { +/// +/// `model` is the deployment/model name of the request. It matters only for Azure AI +/// Foundry, which fronts multiple model families under one resource: Claude +/// deployments speak the Anthropic Messages API while everything else is +/// OpenAI-compatible, so the builder is chosen per model rather than per provider. +pub fn create_query_builder( + credentials: &ProviderCredentials, + model: &str, +) -> Box { match credentials.provider { AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new(credentials.platform.clone())), AIProvider::OpenAI => Box::new(OpenAIQueryBuilder::new(credentials.provider.clone())), @@ -25,6 +35,9 @@ pub fn create_query_builder(credentials: &ProviderCredentials) -> Box Box::new(OpenRouterQueryBuilder::new()), + AIProvider::AzureFoundry if AIProvider::is_anthropic_model(model) => Box::new( + AnthropicQueryBuilder::new(AIProvider::AzureFoundry, AIPlatform::Standard), + ), _ => Box::new(OtherQueryBuilder::new(credentials.provider.clone())), } } diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index efd7229ebb..bdb882679b 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -192,6 +192,15 @@ pub struct ResponsesApiTextFormat { pub format: ResponsesApiTextFormatConfig, } +/// Reasoning config for the Responses API (`reasoning: { effort }`). +/// The summary is intentionally not requested, mirroring the copilot chat: OpenAI +/// gates reasoning summaries behind organization verification, so asking for one +/// would fail the request for unverified orgs. +#[derive(Serialize)] +pub struct ResponsesApiReasoning { + pub effort: String, +} + #[derive(Serialize)] pub struct ResponsesApiRequest<'a> { pub model: &'a str, @@ -204,6 +213,8 @@ pub struct ResponsesApiRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub max_output_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub text: Option, @@ -409,6 +420,9 @@ impl OpenAIQueryBuilder { tools, stream: Some(true), temperature: args.temperature, + reasoning: args + .reasoning_effort + .map(|effort| ResponsesApiReasoning { effort: effort.to_string() }), max_output_tokens: args.max_tokens, text, }; @@ -464,6 +478,7 @@ impl OpenAIQueryBuilder { tools, stream: None, // Image generation doesn't use streaming temperature: args.temperature, + reasoning: None, // Image generation models don't take a reasoning effort max_output_tokens: args.max_tokens, text: None, // No structured output for image generation }; diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index 650ce1c19e..f641daec64 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -35,6 +35,12 @@ pub struct OpenAICompletionRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option<&'a str>, + /// DeepSeek disables reasoning via a `thinking` object rather than an effort + /// token (`reasoning_effort: "none"` is rejected by their API). + #[serde(skip_serializing_if = "Option::is_none")] + pub thinking: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub max_completion_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub response_format: Option, @@ -51,6 +57,32 @@ pub struct BuiltRequests { pub without_usage: String, } +/// Provider-specific reasoning translation for OpenAI-compatible providers, +/// mirroring the chat proxy's `applyReasoningToConfig`. Returns the effort token +/// to send, an optional `thinking` object, and the temperature to keep: +/// - DeepSeek disables reasoning via `thinking: {type: disabled}` (the effort +/// token `"none"` is rejected by their API). +/// - Mistral rejects sampling params when reasoning is on (greedy sampling +/// requires `top_p == 1`), so temperature is dropped. +fn provider_reasoning_fields<'a>( + provider: &AIProvider, + reasoning_effort: Option<&'a str>, + temperature: Option, +) -> (Option<&'a str>, Option, Option) { + let (effort, thinking) = match provider { + AIProvider::DeepSeek if reasoning_effort == Some("none") => { + (None, Some(serde_json::json!({ "type": "disabled" }))) + } + _ => (reasoning_effort, None), + }; + let temperature = if *provider == AIProvider::Mistral && reasoning_effort.is_some() { + None + } else { + temperature + }; + (effort, thinking, temperature) +} + /// Query builder for providers using the OpenAI-compatible completion endpoint /// (Mistral, DeepSeek, Groq, TogetherAI, CustomAI, etc.) pub struct OtherQueryBuilder { @@ -109,12 +141,17 @@ impl OtherQueryBuilder { None }; + let (reasoning_effort, thinking, temperature) = + provider_reasoning_fields(&self.provider_kind, args.reasoning_effort, args.temperature); + // Build request with stream_options for usage tracking let request_with_usage = OpenAICompletionRequest { model: args.model, messages: &prepared_messages, tools: args.tools, - temperature: args.temperature, + temperature, + reasoning_effort, + thinking: thinking.clone(), max_completion_tokens: args.max_tokens, response_format: response_format.clone(), tool_choice: tool_choice.clone(), @@ -127,7 +164,9 @@ impl OtherQueryBuilder { model: args.model, messages: &prepared_messages, tools: args.tools, - temperature: args.temperature, + temperature, + reasoning_effort, + thinking, max_completion_tokens: args.max_tokens, response_format, tool_choice, @@ -248,7 +287,7 @@ impl QueryBuilder for OtherQueryBuilder { } fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { - if self.provider_kind.is_azure_openai(base_url) { + if self.provider_kind.is_azure(base_url) { AIProvider::build_azure_openai_url(base_url, "chat/completions") } else { format!("{}/chat/completions", base_url) @@ -261,10 +300,57 @@ impl QueryBuilder for OtherQueryBuilder { base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - if self.provider_kind.is_azure_openai(base_url) { + if self.provider_kind.is_azure(base_url) { vec![("api-key", api_key.to_string())] } else { vec![("Authorization", format!("Bearer {}", api_key))] } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deepseek_off_translates_to_thinking_disabled() { + let (effort, thinking, temperature) = + provider_reasoning_fields(&AIProvider::DeepSeek, Some("none"), Some(0.5)); + // "none" is rejected as an effort token, so it becomes a thinking disable. + assert_eq!(effort, None); + assert_eq!(thinking, Some(serde_json::json!({ "type": "disabled" }))); + assert_eq!(temperature, Some(0.5)); + } + + #[test] + fn deepseek_level_passes_effort_through() { + let (effort, thinking, _) = + provider_reasoning_fields(&AIProvider::DeepSeek, Some("high"), None); + assert_eq!(effort, Some("high")); + assert!(thinking.is_none()); + } + + #[test] + fn mistral_drops_temperature_when_reasoning_on() { + let (effort, thinking, temperature) = + provider_reasoning_fields(&AIProvider::Mistral, Some("high"), Some(0.7)); + assert_eq!(effort, Some("high")); + assert!(thinking.is_none()); + assert_eq!(temperature, None); + } + + #[test] + fn mistral_keeps_temperature_when_reasoning_off() { + let (_, _, temperature) = provider_reasoning_fields(&AIProvider::Mistral, None, Some(0.7)); + assert_eq!(temperature, Some(0.7)); + } + + #[test] + fn other_providers_pass_reasoning_and_temperature_through() { + let (effort, thinking, temperature) = + provider_reasoning_fields(&AIProvider::OpenRouter, Some("high"), Some(0.3)); + assert_eq!(effort, Some("high")); + assert!(thinking.is_none()); + assert_eq!(temperature, Some(0.3)); + } +} diff --git a/backend/windmill-ai/src/proxy.rs b/backend/windmill-ai/src/proxy.rs index 891fa89486..fe4a0229d4 100644 --- a/backend/windmill-ai/src/proxy.rs +++ b/backend/windmill-ai/src/proxy.rs @@ -51,6 +51,7 @@ pub fn supports_openai_compatible_proxy(provider: &AIProvider) -> bool { provider, AIProvider::OpenAI | AIProvider::AzureOpenAI + | AIProvider::AzureFoundry | AIProvider::Mistral | AIProvider::DeepSeek | AIProvider::Groq @@ -64,6 +65,7 @@ pub fn proxy_execution_mode(provider: &AIProvider) -> ProxyExecutionMode { match provider { AIProvider::OpenAI | AIProvider::AzureOpenAI + | AIProvider::AzureFoundry | AIProvider::Anthropic | AIProvider::Mistral | AIProvider::DeepSeek @@ -89,7 +91,7 @@ pub fn build_openai_compatible_proxy_request(args: &ProxyBuildArgs<'_>) -> Resul }; let base_url = credentials.base_url.trim_end_matches('/'); - let is_azure = credentials.provider.is_azure_openai(base_url); + let is_azure = credentials.provider.is_azure(base_url); let url = if is_azure { AIProvider::build_azure_openai_url(base_url, args.path) } else { @@ -201,6 +203,7 @@ mod tests { let cases = [ (AIProvider::OpenAI, ProxyExecutionMode::HttpForward), (AIProvider::AzureOpenAI, ProxyExecutionMode::HttpForward), + (AIProvider::AzureFoundry, ProxyExecutionMode::HttpForward), (AIProvider::Anthropic, ProxyExecutionMode::HttpForward), (AIProvider::Mistral, ProxyExecutionMode::HttpForward), (AIProvider::DeepSeek, ProxyExecutionMode::HttpForward), @@ -253,6 +256,35 @@ mod tests { .contains(&("api-key".to_string(), "api-key".to_string()))); } + #[test] + fn builds_azure_foundry_proxy_request() { + // Foundry's OpenAI-compatible endpoint uses the same Azure conventions + // (api-key header, /openai -> /openai/v1 path) as Azure OpenAI. + let credentials = credentials( + AIProvider::AzureFoundry, + "https://example.services.ai.azure.com/openai", + ); + let method = Method::POST; + let headers = HeaderMap::new(); + + let request = build_openai_compatible_proxy_request(&ProxyBuildArgs { + method: &method, + path: "chat/completions", + headers: &headers, + body: br#"{"model":"gpt-4o","messages":[]}"#, + credentials: &credentials, + }) + .unwrap(); + + assert_eq!( + request.url, + "https://example.services.ai.azure.com/openai/v1/chat/completions" + ); + assert!(request + .headers + .contains(&("api-key".to_string(), "api-key".to_string()))); + } + #[test] fn injects_user_into_proxy_body() { let mut credentials = credentials(AIProvider::OpenAI, "https://api.openai.com/v1"); @@ -273,4 +305,31 @@ mod tests { assert_eq!(body["user"], "user-1"); assert_eq!(body["model"], "gpt-4o"); } + + #[test] + fn foundry_routes_claude_to_anthropic_messages_api() { + use crate::providers::create_query_builder; + use crate::types::OutputType; + + let creds = credentials( + AIProvider::AzureFoundry, + "https://wm-test-ai.services.ai.azure.com/openai/v1", + ); + + // Claude deployment -> Anthropic Messages API surface + x-api-key auth. + let claude = create_query_builder(&creds, "claude-sonnet-5"); + assert_eq!( + claude.get_endpoint(&creds.base_url, "claude-sonnet-5", &OutputType::Text), + "https://wm-test-ai.services.ai.azure.com/anthropic/v1/messages" + ); + let auth = claude.get_auth_headers("api-key", &creds.base_url, &OutputType::Text); + assert!(auth.contains(&("x-api-key", "api-key".to_string()))); + + // OpenAI-compatible deployment -> chat completions surface. + let gpt = create_query_builder(&creds, "gpt-4o"); + assert_eq!( + gpt.get_endpoint(&creds.base_url, "gpt-4o", &OutputType::Text), + "https://wm-test-ai.services.ai.azure.com/openai/v1/chat/completions" + ); + } } diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index 9830045f57..79fca4bd7d 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -12,6 +12,9 @@ pub struct BuildRequestArgs<'a> { pub tools: Option<&'a [ToolDef]>, pub model: &'a str, pub temperature: Option, + /// Provider-native reasoning effort token (e.g. `low`, `high`, `none`). + /// Each `build_request` maps it onto the provider's thinking config. + pub reasoning_effort: Option<&'a str>, pub max_tokens: Option, pub output_schema: Option<&'a OpenAPISchema>, pub output_type: &'a OutputType, diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index bb1b9d9731..66ff7da9de 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -9,7 +9,9 @@ use windmill_common::{error::Error, utils::rd_string}; use crate::{ ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, ai_types::UrlCitation, - ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}, + ai_types::{ + AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall, + }, query_builder::StreamEventSink, types::StreamingEvent, }; @@ -30,6 +32,10 @@ pub struct OpenAIChoiceDeltaToolCall { #[derive(Deserialize)] pub struct OpenAIChoiceDelta { pub content: Option, + /// Reasoning summary streamed by providers that expose it (e.g. DeepSeek's + /// `reasoning_content`). Rendered as a "thinking" affordance. + #[serde(default)] + pub reasoning_content: Option, pub tool_calls: Option>, } @@ -142,6 +148,13 @@ impl SSEParser for OpenAISSEParser { if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { if let Some(delta) = choices.remove(0).delta { + if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { + let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + if let Some(content) = delta.content.filter(|s| !s.is_empty()) { self.accumulated_content.push_str(&content); let event = StreamingEvent::TokenDelta { content }; @@ -216,6 +229,16 @@ pub enum AnthropicContentBlockStart { id: String, name: String, }, + #[serde(rename = "thinking")] + Thinking { + #[serde(default)] + thinking: String, + }, + #[serde(rename = "redacted_thinking")] + RedactedThinking { + #[serde(default)] + data: String, + }, #[serde(other)] Unknown, } @@ -240,6 +263,10 @@ pub enum AnthropicDelta { InputJsonDelta { partial_json: String }, #[serde(rename = "citations_delta")] CitationsDelta { citation: AnthropicCitationDelta }, + #[serde(rename = "thinking_delta")] + ThinkingDelta { thinking: String }, + #[serde(rename = "signature_delta")] + SignatureDelta { signature: String }, #[serde(other)] Unknown, } @@ -292,6 +319,7 @@ pub enum AnthropicSSEEvent { #[allow(dead_code)] enum ContentBlockState { Text, + Thinking, ToolUse { id: String, name: String }, Unknown, } @@ -310,6 +338,11 @@ pub struct AnthropicSSEParser { pub used_websearch: bool, /// Token usage from message_delta event pub usage: Option, + /// Claude thinking block accumulated from `thinking`/`signature` deltas + /// (or a redacted block). Attached to the first tool call of the turn so it + /// can be replayed before `tool_use` (required by Claude when thinking is on). + pending_reasoning: Option, + reasoning_attached: bool, } impl AnthropicSSEParser { @@ -323,6 +356,8 @@ impl AnthropicSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + pending_reasoning: None, + reasoning_attached: false, } } } @@ -363,6 +398,17 @@ impl SSEParser for AnthropicSSEParser { self.stream_event_processor .send(event, &mut self.events_str) .await?; + // Attach the turn's thinking block to the first tool call so it + // can be replayed before tool_use on the next request. + let extra_content = if self.reasoning_attached { + None + } else { + self.reasoning_attached = true; + self.pending_reasoning.take().map(|anthropic| ExtraContent { + anthropic: Some(anthropic), + ..Default::default() + }) + }; // Initialize tool call accumulator self.accumulated_tool_calls.insert( index as i64, @@ -370,10 +416,35 @@ impl SSEParser for AnthropicSSEParser { id, function: OpenAIFunction { name, arguments: String::new() }, r#type: "function".to_string(), - extra_content: None, + extra_content, }, ); } + AnthropicContentBlockStart::Thinking { thinking } => { + self.content_blocks + .insert(index, ContentBlockState::Thinking); + let entry = self.pending_reasoning.get_or_insert_with(Default::default); + if !thinking.is_empty() { + entry + .thinking + .get_or_insert_with(String::new) + .push_str(&thinking); + self.stream_event_processor + .send( + StreamingEvent::ReasoningTokenDelta { content: thinking }, + &mut self.events_str, + ) + .await?; + } + } + AnthropicContentBlockStart::RedactedThinking { data } => { + // Redacted blocks arrive whole (no deltas). + self.content_blocks + .insert(index, ContentBlockState::Unknown); + self.pending_reasoning + .get_or_insert_with(Default::default) + .redacted_thinking = Some(data); + } AnthropicContentBlockStart::ServerToolUse { name, .. } => { // Detect websearch tool usage if name == "web_search" { @@ -417,6 +488,34 @@ impl SSEParser for AnthropicSSEParser { title: citation.title, }); } + AnthropicDelta::ThinkingDelta { thinking } => { + if let Some(ContentBlockState::Thinking) = + self.content_blocks.get(&index) + { + self.pending_reasoning + .get_or_insert_with(Default::default) + .thinking + .get_or_insert_with(String::new) + .push_str(&thinking); + self.stream_event_processor + .send( + StreamingEvent::ReasoningTokenDelta { content: thinking }, + &mut self.events_str, + ) + .await?; + } + } + AnthropicDelta::SignatureDelta { signature } => { + if let Some(ContentBlockState::Thinking) = + self.content_blocks.get(&index) + { + self.pending_reasoning + .get_or_insert_with(Default::default) + .signature + .get_or_insert_with(String::new) + .push_str(&signature); + } + } AnthropicDelta::Unknown => {} } } @@ -495,6 +594,15 @@ impl SSEParser for GeminiSSEParser { return Ok(()); }; + if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { + self.stream_event_processor + .send( + StreamingEvent::ReasoningTokenDelta { content: reasoning }, + &mut self.events_str, + ) + .await?; + } + if let Some(text) = parsed.text { self.accumulated_content.push_str(&text); self.stream_event_processor @@ -522,7 +630,7 @@ impl SSEParser for GeminiSSEParser { let extra_content = tool_call.thought_signature.map(|sig| ExtraContent { google: Some(GoogleExtraContent { thought_signature: Some(sig) }), - bedrock: None, + ..Default::default() }); self.accumulated_tool_calls.insert( @@ -805,3 +913,76 @@ impl SSEParser for OpenAIResponsesSSEParser { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reasoning_token_delta_serializes_with_snake_case_tag() { + let event = StreamingEvent::ReasoningTokenDelta { content: "hmm".to_string() }; + let json: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&event).unwrap()).unwrap(); + assert_eq!(json["type"], "reasoning_token_delta"); + assert_eq!(json["content"], "hmm"); + } + + #[test] + fn openai_delta_parses_reasoning_content() { + // DeepSeek and similar stream reasoning under `reasoning_content`. + let delta: OpenAIChoiceDelta = + serde_json::from_str(r#"{"reasoning_content":"let me think"}"#).unwrap(); + assert_eq!(delta.reasoning_content.as_deref(), Some("let me think")); + } + + #[test] + fn parses_anthropic_thinking_events() { + let start: AnthropicSSEEvent = serde_json::from_str( + r#"{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"x"}}"#, + ) + .unwrap(); + assert!(matches!( + start, + AnthropicSSEEvent::ContentBlockStart { + content_block: AnthropicContentBlockStart::Thinking { .. }, + .. + } + )); + + let thinking_delta: AnthropicSSEEvent = serde_json::from_str( + r#"{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"more"}}"#, + ) + .unwrap(); + assert!(matches!( + thinking_delta, + AnthropicSSEEvent::ContentBlockDelta { + delta: AnthropicDelta::ThinkingDelta { .. }, + .. + } + )); + + let signature_delta: AnthropicSSEEvent = serde_json::from_str( + r#"{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig"}}"#, + ) + .unwrap(); + assert!(matches!( + signature_delta, + AnthropicSSEEvent::ContentBlockDelta { + delta: AnthropicDelta::SignatureDelta { .. }, + .. + } + )); + + let redacted: AnthropicSSEEvent = serde_json::from_str( + r#"{"type":"content_block_start","index":0,"content_block":{"type":"redacted_thinking","data":"abc"}}"#, + ) + .unwrap(); + assert!(matches!( + redacted, + AnthropicSSEEvent::ContentBlockStart { + content_block: AnthropicContentBlockStart::RedactedThinking { .. }, + .. + } + )); + } +} diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 8f864468db..6991e0e0d3 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -203,6 +203,10 @@ pub struct ProviderWithResource { pub kind: AIProvider, pub resource: ProviderResource, pub model: String, + /// Provider-native reasoning effort token (e.g. `low`, `high`, `none`). + /// Belongs to the model selection, so it rides on the provider config. + #[serde(default)] + pub reasoning_effort: Option, } impl ProviderWithResource { @@ -214,6 +218,12 @@ impl ProviderWithResource { &self.model } + /// The reasoning effort to thread to the provider, treating an empty string + /// (e.g. a cleared flow input) as unset. + pub fn get_reasoning_effort(&self) -> Option<&str> { + self.reasoning_effort.as_deref().filter(|s| !s.is_empty()) + } + pub async fn get_base_url(&self, db: &DB) -> Result { self.kind .get_base_url(self.resource.base_url.clone(), db) @@ -365,6 +375,10 @@ pub struct AIAgentResult<'a> { pub enum StreamingEvent { /// Individual token from the AI response TokenDelta { content: String }, + /// Individual token of the model's reasoning / thinking summary. Emitted + /// before the answer when reasoning is enabled; renderable as a "thinking" + /// affordance (thinking tokens bill regardless of whether they are shown). + ReasoningTokenDelta { content: String }, /// Tool call has started ToolCall { call_id: String, function_name: String }, /// Tool call arguments are complete diff --git a/backend/windmill-api-assets/Cargo.toml b/backend/windmill-api-assets/Cargo.toml index c6e7676b6b..c75715a3ac 100644 --- a/backend/windmill-api-assets/Cargo.toml +++ b/backend/windmill-api-assets/Cargo.toml @@ -8,6 +8,9 @@ edition.workspace = true name = "windmill_api_assets" path = "src/lib.rs" +[features] +private = ["windmill-common/private"] + [dependencies] windmill-api-auth.workspace = true windmill-common = { workspace = true, default-features = false } diff --git a/backend/windmill-api-assets/src/backfill_oss.rs b/backend/windmill-api-assets/src/backfill_oss.rs new file mode 100644 index 0000000000..b5f0da1d58 --- /dev/null +++ b/backend/windmill-api-assets/src/backfill_oss.rs @@ -0,0 +1,18 @@ +//! OSS fallback: backfilling a range of partitions is an enterprise +//! feature (the resolution/enumeration logic lives in `windmill-ee-private`, +//! `windmill-api-assets/src/backfill_ee.rs`). Single-partition runs with an +//! explicit `partition` arg remain available in OSS. + +use windmill_common::error::{Error, Result}; + +use crate::{PartitionsInRangeQuery, PartitionsInRangeResponse}; + +pub(crate) async fn partitions_in_range( + _tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + _w_id: &str, + _q: &PartitionsInRangeQuery, +) -> Result { + Err(Error::BadRequest( + "Backfilling a range of partitions is an enterprise feature".to_string(), + )) +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 29122bd793..997879e155 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -13,7 +13,19 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; + +// Partition-range backfill preview. The logic (producer resolution, range +// enumeration, status join) is enterprise: the `private` build compiles the +// EE module, the public build a stub that errors. +#[cfg(feature = "private")] +mod backfill_ee; +#[cfg(feature = "private")] +use backfill_ee as backfill; +#[cfg(not(feature = "private"))] +mod backfill_oss; +#[cfg(not(feature = "private"))] +use backfill_oss as backfill; pub fn workspaced_service() -> Router { Router::new() @@ -23,8 +35,66 @@ pub fn workspaced_service() -> Router { .route("/graph", get(asset_graph)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) + .route("/partitions_in_range", get(list_partitions_in_range)) .route("/asset_schemas", get(list_asset_schemas)) .route("/record_materialization", post(record_materialization)) + .route("/macros", get(list_macros)) +} + +// One registry macro, with its full definition — drives the macro-explorer +// drawer (body preview) and the DuckDB editor autocomplete (signatures). +#[derive(Serialize)] +struct MacroListItem { + name: String, + params: String, + body: String, + is_table: bool, + provider_path: String, +} + +// Every workspace macro (`// macros` libraries), grouped client-side by +// provider. Small by construction — one row per macro definition. The rows +// copy script body text, so visibility must match reading the provider +// script itself: the EXISTS join runs under the user_db transaction (script +// RLS filters libraries the caller can't read) and the scope predicate +// covers path-scoped tokens, mirroring `list_scripts`. +async fn list_macros( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let scope_allowed = build_scope_path_predicate(&authed, "scripts", "read"); + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#"SELECT m.name AS "name!", m.params AS "params!", m.body AS "body!", + m.is_table_macro AS "is_table_macro!", m.provider_path AS "provider_path!" + FROM macro_definition m + WHERE m.workspace_id = $1 + AND EXISTS ( + SELECT 1 FROM script s + WHERE s.workspace_id = m.workspace_id + AND s.path = m.provider_path + AND s.archived = false + AND s.deleted = false + ) + ORDER BY m.provider_path, m.name"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json( + rows.into_iter() + .filter(|r| scope_allowed(&r.provider_path)) + .map(|r| MacroListItem { + name: r.name, + params: r.params, + body: r.body, + is_table: r.is_table_macro, + provider_path: r.provider_path, + }) + .collect(), + )) } #[derive(Deserialize)] @@ -54,6 +124,55 @@ async fn list_partitions( Ok(Json(rows)) } +// Only the EE `backfill` module reads the fields; the OSS stub errors without +// touching them. +#[cfg_attr(not(feature = "private"), allow(dead_code))] +#[derive(Deserialize)] +struct PartitionsInRangeQuery { + // The materialized ducklake asset path (`/
`). + path: String, + // Inclusive calendar-day range (YYYY-MM-DD), local to the producer's + // partition tz. + from: chrono::NaiveDate, + to: chrono::NaiveDate, +} + +#[derive(Serialize)] +struct PartitionInRange { + partition: String, + // `missing` | `running` | `materialized` | `failed` — `missing` means no + // materialization was ever recorded for the slice. + status: &'static str, +} + +#[derive(Serialize)] +struct PartitionsInRangeResponse { + // The pipeline script that materializes the asset (managed `// materialize` + // target, or a partitioned writer using the SDK helpers) — the runnable a + // backfill launches (with an explicit `partition` arg per slice). + producer_path: String, + partition_kind: String, + partitions: Vec, +} + +// Backfill range preview: every partition the producer's `// partitioned` spec +// expects in `[from, to]`, joined with what `materialized_partition` records — +// the missing/failed subset is the backfill worklist. The logic is in the +// `backfill` module pair: EE resolves and enumerates, the OSS stub errors +// (single-partition runs stay available everywhere; fanning out over a range +// is enterprise). +async fn list_partitions_in_range( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let res = backfill::partitions_in_range(&mut tx, &w_id, &q).await?; + tx.commit().await?; + Ok(Json(res)) +} + // Per-asset captured output schema versions for a ducklake asset (gap #2a) — // the schema-evolution history persisted after each managed `// materialize`. // Newest version first; materialization targets are ducklake-only in v1, so the @@ -495,6 +614,21 @@ struct GraphQuery { struct GraphAssetNode { kind: AssetKind, path: String, + // Fork workspaces only: 'fork' when the fork has materialized this ducklake asset itself, + // 'deferred' when reads fall back to the parent workspace's current table (defer view). + // Absent outside forks, for non-ducklake assets, and for assets never materialized + // anywhere. Lockstep with TS `AssetGraphAssetNode.fork_materialization`. + #[serde(skip_serializing_if = "Option::is_none")] + fork_materialization: Option, + // The base dimension this asset is the SCD2 `_current` companion view + // of — set only on a `ducklake://…/_current` node whose producer + // declares `// materialize … history` on ``. The producer edge already + // links it to that script (both writes are registered at deploy); this lets + // the canvas render it as a derived "current view" of the base rather than an + // unrelated table. Absent for every other asset. Lockstep with TS + // `AssetGraphAssetNode.derived_from`. + #[serde(skip_serializing_if = "Option::is_none")] + derived_from: Option, } #[derive(Serialize, Debug)] @@ -513,6 +647,14 @@ struct GraphRunnableNode { partition_kind: Option, #[serde(skip_serializing_if = "Option::is_none", default)] freshness: Option, + // Completion time of the most recently started successful run of this + // pipeline member. The canvas checks it against the `// freshness` window + // to color the badge fresh/stale. The badge itself is passive; on EE the + // freshness watchdog (windmill-queue) separately re-runs stale + // unpartitioned producers. Absent when no successful run is visible to + // the caller (job RLS applies). + #[serde(skip_serializing_if = "Option::is_none", default)] + last_success_at: Option>, #[serde(skip_serializing_if = "Option::is_none", default)] tag: Option, #[serde(skip_serializing_if = "Option::is_none", default)] @@ -537,6 +679,39 @@ struct GraphRunnableNode { // `merge` / any partitioned write INSERTs into a fixed-schema table. #[serde(skip_serializing_if = "Option::is_none", default)] materialize_strategy: Option, + // `on_schema_change=ignore` on the managed materialize — the producer's + // opt-out from downstream schema-contract warnings. Threaded to the editor + // so its client-side contract mirror suppresses the same warnings the + // server check does. Only serialized when set to `ignore` (default `warn` + // is absent). Lockstep with TS `AssetGraphRunnableNode.materialize_on_schema_change`. + #[serde(skip_serializing_if = "Option::is_none", default)] + materialize_on_schema_change: Option, + // Macros this script provides to the workspace registry (deployed + // `// macros` library). Drives the library node state + details-pane + // signature list. Lockstep with TS `AssetGraphRunnableNode.macros`. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + macros: Vec, +} + +// One macro of a `// macros` library, as surfaced on its graph node. +#[derive(Serialize, Debug, Clone)] +struct MacroInfo { + name: String, + // Verbatim parameter list, for the `name(params)` signature display. + params: String, + is_table: bool, +} + +// A macro-library → consumer edge: the consumer calls `macro_names` of +// `lib_path`'s macros (deploy-recorded detection), or pulls in the whole +// library via `// use` (`via_use`, in which case `macro_names` lists all of +// the library's macros). +#[derive(Serialize, Debug)] +struct MacroEdge { + lib_path: String, + consumer_path: String, + macro_names: Vec, + via_use: bool, } // The output asset a producer's column lineage belongs to (the `// materialize` @@ -631,18 +806,42 @@ enum TriggerEdge { }, } +// Ordering-only "must-run-after" edge: `runnable_path`'s data test reads +// `asset` (a `// data_test relationships` ref, or a custom test whose body +// reads a known pipeline asset), so the asset's in-pipeline producer must +// materialize before `runnable_path` runs. NOT a data-consumption edge — the +// tested script doesn't ingest the asset's rows, it only needs the table to +// exist at test time. Rendered dashed (like macro edges) and fed into the +// cascade topo-sort so a cold cascade orders the referenced dimension first. +// Only emitted when the referenced asset has a producer in the graph; an +// external table (no producer) adds no edge — the runtime error stands. +#[derive(Serialize, Debug)] +struct TestEdge { + producer_kind: AssetUsageKind, + producer_path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + asset_kind: AssetKind, + asset_path: String, +} + #[derive(Serialize, Debug)] struct AssetGraphResponse { assets: Vec, runnables: Vec, edges: Vec, triggers: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + macro_edges: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + test_edges: Vec, } async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, Extension(user_db): Extension, + Extension(db): Extension, Query(q): Query, ) -> JsonResult { let mut tx = user_db.begin(&authed).await?; @@ -773,6 +972,46 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // Newest successful completed run per pipeline member, for the passive + // freshness status on the canvas. Correlated per-path lookup walks + // ix_job_root_job_index_by_path_2 newest-first until the first success, + // so cost is bounded by the member count, not run history. "Newest" is + // by created_at (the index order), not completed_at: with overlapping + // runs of one path this can pick an earlier completion, erring toward + // stale — never toward false-fresh. Inside the user tx so job-visibility + // RLS applies — a caller who can't see the runs gets no timestamp rather + // than leaked completion times. + let member_paths: Vec = pipeline_member_paths + .iter() + .map(|r| r.path.clone()) + .collect(); + let last_success_rows = sqlx::query!( + r#" + SELECT p.path AS "path!", + (SELECT c.completed_at + FROM v2_job j + JOIN v2_job_completed c ON c.id = j.id + WHERE j.workspace_id = $1 + AND j.runnable_path = p.path + AND j.parent_job IS NULL + -- No 'singlestepflow': flows may share a script's path, and + -- a same-path flow run must not read as the script being + -- fresh (false-fresh). Script retries land as native + -- 'script' jobs; only the rare flow-wrapper fallback is + -- missed, which errs stale. Kept in lockstep with the + -- freshness watchdog's queries (freshness_watchdog_ee). + AND j.kind IN ('script', 'preview') + AND c.status = 'success' + ORDER BY j.created_at DESC + LIMIT 1) AS last_success_at + FROM unnest($2::text[]) AS p(path) + "#, + &w_id, + &member_paths, + ) + .fetch_all(&mut *tx) + .await?; + // Existing scripts / flows in the workspace. Used to filter out // orphan trigger rows whose `script_path` no longer resolves — those // would otherwise be added to `runnable_set` below and surface as @@ -794,6 +1033,31 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // Workspace macro registry + deploy-recorded call edges. Definitions are + // fetched unfiltered so an out-of-folder library still appears as the + // provider endpoint of in-scope consumers' edges; consumers honor the + // folder filter like every other runnable query. + let macro_def_rows = sqlx::query!( + r#"SELECT name AS "name!", provider_path AS "provider_path!", + params AS "params!", is_table_macro AS "is_table_macro!" + FROM macro_definition + WHERE workspace_id = $1 + ORDER BY provider_path, name"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + let macro_usage_rows = sqlx::query!( + r#"SELECT consumer_path AS "consumer_path!", macro_name AS "macro_name!" + FROM macro_usage + WHERE workspace_id = $1 + AND ($2::text IS NULL OR consumer_path LIKE $2)"#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; // Parse each pipeline member's body once into its badge annotations, keyed @@ -838,6 +1102,11 @@ async fn asset_graph( (r.path.clone(), lineage) }) .collect(); + let last_success_by_path: std::collections::HashMap> = + last_success_rows + .into_iter() + .filter_map(|r| r.last_success_at.map(|t| (r.path, t))) + .collect(); let pipeline_member_script_paths: std::collections::HashSet = pipeline_member_paths.into_iter().map(|r| r.path).collect(); let existing_script_paths: std::collections::HashSet = @@ -950,9 +1219,272 @@ async fn asset_graph( triggers.push(edge); } + // Macro libraries + lib→consumer edges. Group per-provider macro lists, + // resolve each usage row's name to its provider (names are + // workspace-unique), and merge `// use` whole-lib edges from the parsed + // member annotations. Both endpoints are forced into the runnable set so + // an out-of-folder library still renders as the edge's provider node. + let mut macros_by_provider: std::collections::HashMap> = + Default::default(); + let mut provider_by_name: std::collections::HashMap = Default::default(); + for r in macro_def_rows { + provider_by_name.insert(r.name.clone(), r.provider_path.clone()); + macros_by_provider + .entry(r.provider_path) + .or_default() + .push(MacroInfo { name: r.name, params: r.params, is_table: r.is_table_macro }); + } + let mut macro_edge_map: std::collections::BTreeMap< + (String, String), + (std::collections::BTreeSet, bool), + > = Default::default(); + for u in macro_usage_rows { + // Same orphan filter as the other edge loops. + if !runnable_exists(AssetUsageKind::Script, &u.consumer_path) { + continue; + } + let Some(lib) = provider_by_name.get(&u.macro_name) else { + continue; + }; + let e = macro_edge_map + .entry((lib.clone(), u.consumer_path)) + .or_default(); + e.0.insert(u.macro_name); + } + for (path, ann) in &annotations_by_path { + for lib in &ann.use_libs { + // An undeployed `// use` target has no registry rows — the live + // draft overlay is the only surface that can render it. + let Some(lib_macros) = macros_by_provider.get(lib) else { + continue; + }; + let e = macro_edge_map + .entry((lib.clone(), path.clone())) + .or_default(); + e.1 = true; + e.0.extend(lib_macros.iter().map(|m| m.name.clone())); + } + } + let macro_edges: Vec = macro_edge_map + .into_iter() + // Same orphan filter as the other edge families: a registry row whose + // provider script no longer exists must not synthesize a phantom + // library node (consumers were filtered above, but the `// use` pass + // re-adds them, so re-check both endpoints). + .filter(|((lib_path, consumer_path), _)| { + runnable_exists(AssetUsageKind::Script, lib_path) + && runnable_exists(AssetUsageKind::Script, consumer_path) + }) + .map(|((lib_path, consumer_path), (names, via_use))| MacroEdge { + lib_path, + consumer_path, + macro_names: names.into_iter().collect(), + via_use, + }) + .collect(); + for e in ¯o_edges { + runnable_set.insert((AssetUsageKind::Script, e.lib_path.clone())); + runnable_set.insert((AssetUsageKind::Script, e.consumer_path.clone())); + } + + // Data-test ordering edges. A `// data_test relationships -> ` + // (and, best-effort, a custom `// data_test + +{#if !provider || !model} +
Select a model to configure reasoning effort.
+{:else if !capability.supported} +
The selected model does not support reasoning effort.
+{:else} + + {#snippet buttonReplacement()} + + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 9edd80dd54..ef632f5738 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -991,13 +991,15 @@ {:else if step == 2 && manual}
- + {#if deployTo}
{/snippet} + {#snippet groupActions(groupItems)} + + {@const groupConflicts = groupItems.filter((i) => { + const d = i.diff as WorkspaceItemDiff + return d.ahead > 0 && d.behind > 0 + }).length} + {#if groupConflicts > 0} + + + {groupConflicts} conflict{groupConflicts !== 1 ? 's' : ''} + + {/if} + {/snippet} {#if allCiTests.length > 0}
@@ -846,22 +1168,31 @@ {/if} - {#if !comparison.all_ahead_items_visible || !comparison.all_behind_items_visible} - - {#if !comparison.all_ahead_items_visible && !comparison.all_behind_items_visible} - This fork is ahead and behind its parent - {:else if !comparison.all_behind_items_visible} - This fork is behind of its parent - {:else if !comparison.all_ahead_items_visible} - This fork is ahead of its parent - {/if} - and some of the changes are not visible by you. Only a user with access to the whole context - may deploy or update this fork. You can share the link to this page to someone with proper - permissions to get it deployed. + {@const hiddenDir = mergeIntoParent ? comparison.hidden_ahead : comparison.hidden_behind} + {#if hiddenDir.items.length > 0} + + + {hiddenDir.items.length} + {mergeIntoParent ? 'ahead' : 'behind'} item{hiddenDir.items.length !== 1 ? 's' : ''} + {hiddenDir.items.length !== 1 ? 'are' : 'is'} excluded from the list below — they are not + resolvable as live items and are most likely stale/phantom diff rows: +
    + {#each hiddenDir.items as it} +
  • {hiddenKindLabel(it.kind)} · {it.path}
    • + {/each} +
+ + {:else if mergeIntoParent ? !comparison.all_ahead_items_visible : !comparison.all_behind_items_visible} + + {hiddenDir.total} + {mergeIntoParent ? 'ahead' : 'behind'} item{hiddenDir.total !== 1 ? 's' : ''} + ({formatHiddenByKind(hiddenDir.by_kind)}) + {hiddenDir.total !== 1 ? 'are' : 'is'} not visible to your user and + {hiddenDir.total !== 1 ? 'are' : 'is'} excluded from the list below. You can still + {mergeIntoParent ? 'deploy' : 'update'} the items you can see — share this page with someone + who has full access to include the rest. {/if} {/snippet} @@ -1016,6 +1347,27 @@ Show diff
+ {#if diff.kind === 'resource' || diff.kind === 'variable'} + {#if isPinned(diff.kind, diff.path)} + + workspace-specific + + {:else} + + {/if} + {/if} {/if} {/snippet} @@ -1025,44 +1377,51 @@
- {#if comparison.all_behind_items_visible && comparison.all_ahead_items_visible} -
- {#if mergeIntoParent && !hasOpenDeploymentRequest && !deploymentRequestPanel?.isDialogOpen()} - - {/if} + +
+ {#if mergeIntoParent && !hasOpenDeploymentRequest && !deploymentRequestPanel?.isDialogOpen()} -
- {#if !(mergeIntoParent && !canDeployToParent) && hasUnselectedOnBehalfOf} - - You must set the "on behalf of" user for all items before deploying - - The "run on behalf of" field defines which user's permissions will be - applied during execution. Make sure this is set to an appropriate user - before deploying. - - {/if} + +
+ {#if !deployPerm.ok} + {deployPerm.reason} + {/if} + {#if !(mergeIntoParent && !canDeployToParent) && hasUnselectedOnBehalfOf} + + You must set the "on behalf of" user for all items before deploying + + The "run on behalf of" field defines which user's permissions will be applied + during execution. Make sure this is set to an appropriate user before + deploying. + + {/if} {#if deploymentErrorMessage != ''} @@ -1097,6 +1456,53 @@
+ + {#if pinnedItems.length > 0} +
+
+ Workspace-specific items +
+

+ These resources and variables keep their own value in each environment and are excluded + from the diff. An item that exists on only one side can be seeded onto the other with + "Create in …" (copies the current value, including secrets); it will never overwrite an + existing value. +

+
+ {#each pinnedItems as it (`${it.item_kind}:${it.path}`)} + {@const missingSide = + it.onCurrent && !it.onParent + ? parentWorkspaceId + : !it.onCurrent && it.onParent + ? currentWorkspaceId + : undefined} +
+ {it.item_kind} + {it.path} + {#if missingSide} + + {/if} + +
+ {/each} +
+
+ {/if}
@@ -1129,6 +1535,27 @@
+ + { + const it = createConfirm + createConfirm = undefined + if (it) createOnRemote(it) + }} + onCanceled={() => (createConfirm = undefined)} + > +

+ This copies the current value of {createConfirm?.path} + (including any secret value) from + {createConfirm?.onCurrent ? currentWorkspaceId : parentWorkspaceId} + into {createConfirm?.onCurrent ? parentWorkspaceId : currentWorkspaceId}. It stays + workspace-specific afterward, so later promotes won't overwrite it. If it already exists + there, it's left untouched and just marked workspace-specific. +

+ {:else}
No comparison data available
diff --git a/frontend/src/lib/components/CronInput.svelte b/frontend/src/lib/components/CronInput.svelte index b7febcdaca..d2b629d33e 100644 --- a/frontend/src/lib/components/CronInput.svelte +++ b/frontend/src/lib/components/CronInput.svelte @@ -204,18 +204,20 @@
-
- +
+
+ +
{#if !disabled}
{@render cronBuilder()} diff --git a/frontend/src/lib/components/DataTestsResult.svelte b/frontend/src/lib/components/DataTestsResult.svelte index cba28f3a82..385af187df 100644 --- a/frontend/src/lib/components/DataTestsResult.svelte +++ b/frontend/src/lib/components/DataTestsResult.svelte @@ -1,15 +1,22 @@
    {#each tests as t (t.test)} -
  • - {#if t.violating > 0} - - failed: - {t.test} - — {t.violating} violating row{t.violating === 1 ? '' : 's'} - {:else} - - passed: - {t.test} +
  • +
    + {#if t.violating > 0} + + failed: + {t.test} + — {t.violating} violating row{t.violating === 1 ? '' : 's'} + {#if t.sample && t.sample.length > 0} + + {/if} + {:else} + + passed: + {t.test} + {/if} +
    + {#if t.violating > 0 && t.sample && t.sample.length > 0 && expanded.has(t.test)} +
    +
    + sample of the violating rows ({t.sample.length} of {t.violating}, unordered) +
    + +
    {/if}
    • {/each} diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte new file mode 100644 index 0000000000..bdba23c57f --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -0,0 +1,191 @@ + + +{#if isDev && parentId} +
    +

    + This is a dev workspace paired with root workspace {parentId}. Promote changes + from the home page banner or the Compare & Deploy page. +

    +
    + +
    +
    +{:else if pairedDev} +
    +

    + This workspace's dev workspace is {pairedDev.name} ({pairedDev.id}). Edits to this + workspace are redirected there. +

    +
    + {#if pairedDev.isMember} + + {/if} + +
    +
    +{:else if parentId} +

    + Dev workspace pairing is only available for root workspaces. This workspace is a fork of + {parentId}. +

    +{:else} +
    +

    + Pair this workspace with a dev workspace: the same code with a different environment (resource + and variable values). Edits are made in the dev workspace and promoted here. +

    +
    + Attach an existing workspace as dev + onNativeInput(e.currentTarget.value)} + /> + {#if metadataError} + + + + The // partitioned header has {metadataError} — fix it in the + script; no default is filled in. + + + {:else if beforeStart} + + Partitioning starts {spec.start} — defaulted to the first partition. + + {/if} + {:else} + value ?? '', (v) => (value = v)} + size="sm" + inputProps={{ + placeholder: spec.kind === 'dynamic' ? 'Partition key value' : 'Partition bucket' + }} + /> + + {#if spec.kind === 'dynamic'} + Dynamic partition — leave blank to let the run resolve it from the payload. + {:else} + Custom partition format — enter the bucket exactly as the producer renders it. + {/if} + + {/if} + + + {#if calendarPicker && materializeTarget?.kind === 'ducklake' && value} + {#if selectedMaterialized} + + + {value} is already materialized — running replaces it. + + {:else} + + + {value} not materialized yet — running creates it. + + {/if} + {/if} + + + {#if upstreamMissing} + + + + No upstream data for {value} yet — this run may materialize an + empty partition. + + + {/if} + + + {#if recentlyMissing.length > 0} +
    + + {recentlyMissing.length} of the last {recentWindow(spec.kind)} + {spec.kind} partitions are not materialized: + +
    + {#each recentlyMissing.slice(0, MAX_MISSING_CHIPS) as b (b)} + + {/each} + {#if recentlyMissing.length > MAX_MISSING_CHIPS} + + +{recentlyMissing.length - MAX_MISSING_CHIPS} more + + {/if} +
    +
    + {/if} +
    diff --git a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte index 863130833d..4be0bb8b6c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte @@ -1,10 +1,12 @@
    Materialized partitions -
    - +
    + {#if backfillRunning} + + {:else if backfillSlices?.length} + + {/if} +
    + +
    @@ -131,4 +196,14 @@
    - + (backfillSlices = undefined)} +/> diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineActivityPanel.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineActivityPanel.svelte index 90e531e858..7e6c34adc2 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineActivityPanel.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineActivityPanel.svelte @@ -36,8 +36,12 @@ // History preload hit its page cap before the days cutoff. truncated?: boolean error?: string | undefined - days: number - onDaysChange: (days: number) => void + days?: number + onDaysChange?: (days: number) => void + // Live-only surfaces (local-dev preview) have no historical fetch: hide + // the day-range Select + histogram (both are history-window concepts) and + // show only the live run stream. + liveOnly?: boolean // Hover a run row → emphasize its node(s) on the canvas; a group header // passes the whole cascade's paths. `undefined` clears. onHoverRun?: (paths: string[] | undefined) => void @@ -51,8 +55,9 @@ loading = false, truncated = false, error, - days, + days = 30, onDaysChange, + liveOnly = false, onHoverRun, onSelectRun }: Props = $props() @@ -99,7 +104,7 @@ // Quick reset: drop the brush and return to the default 30-day window. function resetWindow() { selectedRange = undefined - if (days !== 30) onDaysChange(30) + if (days !== 30) onDaysChange?.(30) } function fmtRange(r: { from: number; to: number }): string { const opt: Intl.DateTimeFormatOptions = { @@ -236,7 +241,9 @@ { label: 'Last 30 days', value: 30 }, { label: 'Last 90 days', value: 90 } ] - let windowLabel = $derived(DAY_OPTIONS.find((o) => o.value === days)?.label ?? `Last ${days} days`) + let windowLabel = $derived( + DAY_OPTIONS.find((o) => o.value === days)?.label ?? `Last ${days} days` + ) // Excludes future-scheduled queued jobs (a schedule's next planned run // is not activity) — see isActiveEvent. @@ -341,17 +348,19 @@ {/if} -
    - days, (v) => onDaysChange?.(v ?? 30)} + /> +
    + {/if}
    - {#if events.length > 0} + {#if events.length > 0 && !liveOnly}
    {:else}
    - No runs in this window ({windowLabel.toLowerCase()}) — executions of this pipeline will - appear here live. + {#if liveOnly} + No runs yet — executions of this pipeline will appear here live. + {:else} + No runs in this window ({windowLabel.toLowerCase()}) — executions of this pipeline will + appear here live. + {/if}
    {/if} {:else} @@ -555,7 +568,8 @@ title={`Join fed by ${g.extraTriggers + 1} triggers`}>+{g.extraTriggers} {/if} - {g.members.length} runs + {g.members.length} runs {ago(g.latestAt)} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte new file mode 100644 index 0000000000..8b10e54ce3 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte @@ -0,0 +1,411 @@ + + +
    +
    + + f/{folder} + · local pipeline dev + + + {wsState === 'open' + ? 'watching' + : wsState === 'connecting' + ? 'connecting…' + : 'disconnected'} + +
    +
    + {#if !bundle} +
    + + Connecting to wmill pipeline dev… +
    + {:else if displayGraph.runnables.length === 0} +
    + No // pipeline scripts found in f/{folder}. Mark a script with a + bare + // pipeline comment. +
    + {:else} + (panelHidden = !panelHidden)} + onRunProducer={runProducer} + onRunByPath={(path, args) => runNode(path, args)} + onRunCascadeByPath={(path, args) => runCascadeFrom(path, args)} + downstreamSubscribers={selectionDownstreamCount} + {resolveLocalScript} + localScriptsVersion={bundle} + {selectionProducers} + canRunByPath + onRunCompleted={() => { + activeRunnable = undefined + activeRunnableJobId = undefined + }} + onSelect={handleCanvasSelect} + onClose={() => (pe.selection = undefined)} + > + {#snippet idlePane()} + (activityHoverPaths = p ?? [])} + onSelectRun={(p) => (activitySelectPaths = p ?? [])} + /> + {/snippet} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineFolderList.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineFolderList.svelte new file mode 100644 index 0000000000..d35b4f21d6 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineFolderList.svelte @@ -0,0 +1,111 @@ + + +
    +
    +

    + Existing pipelines +

    + {#if pipelines.loading && !pipelines.current} +
    + + Loading… +
    + {:else if pipelines.error} +
    Failed: {pipelines.error.message}
    + {:else if visiblePipelines.length === 0} +
    + {currentFolder + ? 'No other pipelines in this workspace.' + : 'No pipelines yet. A pipeline is any folder whose scripts carry pipeline annotations.'} +
    + {:else} +
    + {#each visiblePipelines as p (p.folder)} + + {/each} +
    + {/if} +
    + + {#if !$userStore?.operator} + +
    +

    + Pick or create a folder +

    +
    +
    + +
    + +
    +
    + {/if} +
    diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte new file mode 100644 index 0000000000..cc44d0b41a --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -0,0 +1,546 @@ + + +
    + + +
    + + {#if boundBar}{@render boundBar()}{/if} + {#if mode === 'edit'} + + {/if} + {#if prefetchingAssets} +
    + + Parsing assets… +
    + {/if} + {#if onTogglePanelHidden && (mode !== 'edit' || editor.selection != undefined || editor.activeDraftPath != undefined)} +
    +
    + {/if} +
    + {#if detailsPaneOpen && workspace} + + {#if idleView && idlePane} + {@render idlePane()} + {:else} + onStartBoundedRunForOpen?.(editor.openScriptPath!) + : undefined} + {onRunCompleted} + {onTestStateChange} + {requestRemoveSignal} + {requestRunSignal} + {requestRunCascadeSignal} + {focusUploadSignal} + draftScript={activeDraft?.script} + {pathPrefix} + {onDraftPathChange} + {workspace} + onAnnotationsChange={editor.handleAnnotationsChange} + onAssetsChange={editor.handleAssetsChange} + onContentChange={editor.handleContentChange} + onDraftPersist={editor.handleDraftPersist} + onclose={onClose} + onHide={onTogglePanelHidden} + {onDiscard} + {onDraftSaved} + {onPersistedSaved} + {onScriptRenamed} + {onScriptRemoved} + /> + {/if} + + {/if} +     +
    diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte index ded1013e25..b2183eec77 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte @@ -1,13 +1,6 @@ -
    - {#if visiblePipelines.length > 0 || pipelines.loading} -
    -

    - Existing pipelines -

    - {#if pipelines.loading && !pipelines.current} -
    - - Loading… -
    - {:else if pipelines.error} -
    Failed: {pipelines.error.message}
    - {:else} -
    - {#each visiblePipelines as p} - - {/each} -
    - {/if} -
    - {/if} - - {#if !$userStore?.operator} - -
    -

    - Pick or create a folder -

    -
    -
    - -
    - -
    -
    - {/if} -
    + (open = false)} />     diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte new file mode 100644 index 0000000000..4384f125ba --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte @@ -0,0 +1,73 @@ + + +
    + {#if partitionSpec} + args.partition, (v) => (args.partition = v)} + {workspace} + {materializeTarget} + {upstreamAssets} + /> + {/if} + +
    diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte index bb0fbd4d27..79a1cf6d47 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte @@ -1,12 +1,13 @@ + +{#if show} +
    +
    + +
    +

    Finish setting up pipelines

    +

    + Pipelines materialize data into DuckLake tables backed by object storage. Configure the + following before your first pipeline can run. +

    +
    +
    + +
      + {#each steps as step (step.title)} + {@const Icon = step.icon} +
    • + {#if step.done === true} + + {:else if step.done === false} + + {:else} + + {/if} +
      + {step.title} + {step.description} +
      + {#if step.done !== true} + + {step.cta} + + + {:else} + Configured + {/if} +
      • + {/each} +
    +
    +{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index c709866a5f..4cb65ee8f3 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -10,6 +10,7 @@ Loader2, Play, RotateCw, + SquareFunction, Tag, Target, Timer, @@ -21,12 +22,13 @@ import { preventDefault, stopPropagation } from 'svelte/legacy' import type { GraphUsageKind } from './types' import type { RunnableRunState } from './activeRunnables.svelte' + import { parseDurationSecs } from './parsePipelineAnnotations' import { NODE } from '$lib/components/graph/util' import DropdownV2 from '$lib/components/DropdownV2.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' import type { Item } from '$lib/utils' import { workspaceStore } from '$lib/stores' - import { sendUserToast } from '$lib/utils' + import { sendUserToast, msToReadableTimeShort } from '$lib/utils' interface Props { data: { @@ -35,8 +37,14 @@ in_pipeline?: boolean partition_kind?: 'daily' | 'hourly' | 'weekly' | 'monthly' | 'dynamic' freshness?: string + // Completion time (ISO) of the newest successful run visible to + // the caller. With `freshness`, drives the fresh/stale chip state. + last_success_at?: string tag?: string retry?: { count: number; delay?: string } + // Macros this script provides (deployed/drafted `// macros` library). + // Non-empty renders the ƒ chip marking the node as a macro library. + macros?: { name: string; params: string; is_table: boolean }[] // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState @@ -118,6 +126,44 @@ } } + // Freshness verdict: newest successful run (server `last_success_at`, + // or a newer one the session poll observed) vs the `// freshness` + // window. No verdict (undefined) for drafts — no run history — and for + // unparseable windows; the chip then stays neutral like the other + // annotation chips. + let freshnessWindowS = $derived(data.freshness ? parseDurationSecs(data.freshness) : undefined) + // Ticks so a node crosses fresh→stale while the canvas stays open (the + // graph payload is static between refetches). Armed only when a verdict + // is rendered. + let nowMs = $state(Date.now()) + $effect(() => { + if (freshnessWindowS === undefined || data.unsaved) return + const id = setInterval(() => (nowMs = Date.now()), 30_000) + return () => clearInterval(id) + }) + let lastSuccessMs = $derived.by(() => { + const server = data.last_success_at ? new Date(data.last_success_at).getTime() : undefined + const polled = data.runState?.lastSuccessAt + ? new Date(data.runState.lastSuccessAt).getTime() + : undefined + if (server === undefined) return polled + return polled === undefined ? server : Math.max(server, polled) + }) + let freshnessState = $derived.by((): 'fresh' | 'stale' | undefined => { + if (freshnessWindowS === undefined || data.unsaved) return undefined + if (lastSuccessMs === undefined) return 'stale' + return nowMs - lastSuccessMs <= freshnessWindowS * 1000 ? 'fresh' : 'stale' + }) + let freshnessTooltip = $derived.by(() => { + const base = `// freshness ${data.freshness}` + if (freshnessState === undefined) return base + if (lastSuccessMs === undefined) return `${base} — stale: no successful run yet` + const ago = msToReadableTimeShort(Math.max(0, nowMs - lastSuccessMs)) + return freshnessState === 'fresh' + ? `${base} — fresh: last successful run ${ago} ago` + : `${base} — stale: last successful run ${ago} ago` + }) + // Cascade + bounded-run options live on the Run button's caret popover // (whenever there's a cascade OR a bounded-run start — see `hasCaret` // below), so the kebab menu stays focused on lifecycle actions only. @@ -160,8 +206,8 @@ + guidelines). Only the freshness chip (when it has a verdict) + and the run-state chip below use semantic colors. --> {#if data.partition_kind}
    {data.partition_kind}
    {/if} + {#if data.freshness}
    {data.freshness} @@ -199,6 +257,17 @@ ×{r.count}
    {/if} + {#if data.macros && data.macros.length > 0} +
    1 ? 's' : ''}:\n${data.macros + .map((m) => `• ${m.name}(${m.params})${m.is_table ? ' → table' : ''}`) + .join('\n')}`} + > + + ×{data.macros.length} +
    + {/if} {#if data.runState} {@const rs = data.runState}
    - Run downstream up to… + Run + downstream… - Pick end node(s) on the graph, then run only the cascade between this script - and them. + Run this script and everything downstream. Or pick end node(s) on the graph to + bound the cascade between this script and them.
    diff --git a/frontend/src/lib/components/assets/AssetGraph/SchemaHistoryPanel.svelte b/frontend/src/lib/components/assets/AssetGraph/SchemaHistoryPanel.svelte index fbccccf860..fc49fd0f33 100644 --- a/frontend/src/lib/components/assets/AssetGraph/SchemaHistoryPanel.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/SchemaHistoryPanel.svelte @@ -11,7 +11,7 @@ // fixed-schema table, so the schema is pinned at first materialize; there's // only ever one version, shown as a single current-schema table. import { resource } from 'runed' - import { OpenAPI } from '$lib/gen' + import { AssetService, type AssetSchemaVersion } from '$lib/gen' import { Button } from '$lib/components/common' import { Loader2, RefreshCw, Lock } from 'lucide-svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -26,23 +26,12 @@ } let { path, workspace, canEvolve = true }: Props = $props() + // One column of a captured schema version (the generated `columns` element). type SchemaColumn = { name: string; type: string } - type AssetSchemaVersion = { - version: number - columns: SchemaColumn[] - snapshot_id?: number | null - job_id?: string | null - captured_at: string - } - let schemas = resource([() => workspace, () => path], async ([ws, p], _prev, { signal }) => { + let schemas = resource([() => workspace, () => path], async ([ws, p]) => { if (!ws || !p) return [] as AssetSchemaVersion[] - const res = await fetch( - `${OpenAPI.BASE ?? ''}/w/${ws}/assets/asset_schemas?path=${encodeURIComponent(p)}`, - { credentials: 'include', signal } - ) - if (!res.ok) throw new Error(`GET /assets/asset_schemas → ${res.status}`) - return (await res.json()) as AssetSchemaVersion[] + return await AssetService.listAssetSchemas({ workspace: ws, path: p }) }) // The user's explicit pick (undefined until they click). Falls back to the diff --git a/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte b/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte index caf7d90482..98307013c4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte @@ -60,7 +60,7 @@ import { Handle, Position } from '@xyflow/svelte' import { NODE } from '$lib/components/graph/util' import { twMerge } from 'tailwind-merge' - import { AlertTriangle, EllipsisVertical, Target, Trash2 } from 'lucide-svelte' + import { AlertTriangle, CheckCircle2, EllipsisVertical, Target, Trash2 } from 'lucide-svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { stopPropagation, preventDefault } from 'svelte/legacy' import type { Item } from '$lib/utils' @@ -101,6 +101,9 @@ // auto-generated S3 picker lets the user upload + run) instead of // rendering a "missing" placeholder. onOpenDataUpload?: (scriptPath: string) => void + // True once a file is staged for this data_upload entry — renders the + // node green so the user knows the pipeline can run (see WIN-2129). + ready?: boolean // Page-supplied dispatcher to open the matching native trigger // drawer in edit mode for an attached (non-missing) trigger. // `triggerPath` is the trigger row's path (e.g. the mqtt_trigger @@ -149,6 +152,10 @@ // data_upload routes through its own handler — clicking opens the target // script's run form (with the auto-generated S3 picker). let canOpenDataUpload = $derived(isDataUpload && !!data.runnable_path && !!data.onOpenDataUpload) + // A staged upload turns the node green (ready to run); before that it stays + // on the neutral surface with the "upload a file" prompt. + let dataUploadReady = $derived(isDataUpload && data.ready === true) + let DataUploadIcon = $derived(dataUploadReady ? CheckCircle2 : style.icon) // Schedule + the other native kinds all have dedicated editors. Webhook and // data_upload are excluded — they route through their own open handlers. let canCreate = $derived( @@ -183,7 +190,7 @@ ...(data.onStartBoundedRun ? [ { - displayName: 'Run downstream up to…', + displayName: 'Run + downstream…', icon: Target, action: () => data.onStartBoundedRun?.() } @@ -306,26 +313,47 @@ {:else if canOpenDataUpload} + S3 picker lets the user upload a file and run the pipeline. Goes + green once a file is staged (ready), so "Run pipeline" can proceed; + until then it stays neutral with an "upload a file" prompt. Never + the red "missing" state. --> {:else} diff --git a/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts index 917e21766a..f6682804c7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts @@ -1,8 +1,15 @@ import { JobService } from '$lib/gen' export type RunStatus = 'running' | 'success' | 'failure' -/** Per-runnable badge state: latest run status + runs observed this session. */ -export type RunnableRunState = { status: RunStatus; runs: number } +/** + * Per-runnable badge state: latest run status + runs observed this session. + * `lastSuccessAt` is the completion time (start + duration when the listing + * carries it, else start as a conservative lower bound) of the newest + * successful run seen by the poll — lets the freshness chip go green right + * after an in-session run, ahead of the next graph refetch (whose + * `last_success_at` would carry it). + */ +export type RunnableRunState = { status: RunStatus; runs: number; lastSuccessAt?: string } export type EventStatus = 'queued' | 'running' | 'success' | 'failure' /** One folder activity-log row (a job observed by the poll). */ @@ -14,6 +21,12 @@ export type PipelineEvent = { /** What started it, as far as the job listing reveals. */ source: 'schedule' | 'run' at: string + /** + * Completion time (start + duration) for completed rows. The freshness + * chip compares against completion — `at` is the start time and would + * read a long run as older than its output actually is. + */ + completedAt?: string /** * Queued jobs: when the job is due to start. A future value means a * scheduled run waiting for its cron tick, not pipeline activity. @@ -45,7 +58,8 @@ function statesEq(a: Map, b: Map() const countedJobIds = new Set() // Job ids we've observed in-flight at least once. The catch-up pulse is @@ -241,10 +255,25 @@ export function useActiveRunnableIds( const prev = completedHistory.get(id) const status: RunStatus = (j as any).success === true ? 'success' : 'failure' const ts = startedTs ?? new Date(pollStartedMs).toISOString() + // Freshness compares against COMPLETION time (that's + // when the output materialized — the server-side + // last_success_at is completed_at too). The listing + // only carries started_at, so add duration_ms; when + // absent, the start is a conservative lower bound + // (errs stale, never false-fresh). + const durationMs = (j as any).duration_ms + const doneTs = + typeof durationMs === 'number' && startedTs + ? new Date(new Date(startedTs).getTime() + durationMs).toISOString() + : ts completedHistory.set(id, { runs: (prev?.runs ?? 0) + 1, lastStatus: !prev || ts >= prev.lastTs ? status : prev.lastStatus, - lastTs: !prev || ts >= prev.lastTs ? ts : prev.lastTs + lastTs: !prev || ts >= prev.lastTs ? ts : prev.lastTs, + lastSuccessTs: + status === 'success' && (!prev?.lastSuccessTs || doneTs >= prev.lastSuccessTs) + ? doneTs + : prev?.lastSuccessTs }) } } @@ -267,6 +296,10 @@ export function useActiveRunnableIds( : 'failure', source: (j as any).schedule_path ? 'schedule' : 'run', at: startedTs ?? new Date(pollStartedMs).toISOString(), + completedAt: + !isQueued && typeof (j as any).duration_ms === 'number' && startedTs + ? new Date(new Date(startedTs).getTime() + (j as any).duration_ms).toISOString() + : undefined, scheduledFor: isQueued ? ((j as any).scheduled_for as string | undefined) : undefined }) } @@ -289,7 +322,11 @@ export function useActiveRunnableIds( // previous badge state until a worker picks the job up. const snap = new Map() for (const [id, h] of completedHistory) { - snap.set(id, { status: runningThisTick.has(id) ? 'running' : h.lastStatus, runs: h.runs }) + snap.set(id, { + status: runningThisTick.has(id) ? 'running' : h.lastStatus, + runs: h.runs, + lastSuccessAt: h.lastSuccessTs + }) } for (const id of runningThisTick) { if (!snap.has(id)) snap.set(id, { status: 'running', runs: 0 }) diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts index a5a92f61c2..24e7d4c2da 100644 --- a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts @@ -38,7 +38,14 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => { // stays strictly left of every node of the right branch. const pos = layoutAssetGraph({ nodes: [n('root'), n('a'), n('b'), n('a1'), n('a2'), n('a3'), n('b1')], - edges: [e('root', 'a'), e('root', 'b'), e('a', 'a1'), e('a', 'a2'), e('a', 'a3'), e('b', 'b1')] + edges: [ + e('root', 'a'), + e('root', 'b'), + e('a', 'a1'), + e('a', 'a2'), + e('a', 'a3'), + e('b', 'b1') + ] }) const leftMax = Math.max(...['a', 'a1', 'a2', 'a3'].map((id) => pos.get(id)!.x)) const rightMin = Math.min(...['b', 'b1'].map((id) => pos.get(id)!.x)) @@ -83,14 +90,37 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => { } }) - it('falls back to a grid on cyclic input', () => { + it('lays a 2-cycle out as a chain (feedback edge dropped, no grid)', () => { const pos = layoutAssetGraph({ nodes: [n('a'), n('b')], edges: [e('a', 'b'), e('b', 'a')] }) - expect(pos.size).toBe(2) - expect(pos.get('a')).toBeDefined() - expect(pos.get('b')).toBeDefined() + // First-in-input wins the top slot; the b→a feedback edge is ignored. + expect(pos.get('a')!.y).toBeLessThan(pos.get('b')!.y) + expect(pos.get('a')!.x).toBe(pos.get('b')!.x) + }) + + it('keeps the acyclic part of a graph layered when one cycle exists', () => { + // root → a ⇄ b → leaf: the a⇄b cycle must not degrade root/leaf layering. + const pos = layoutAssetGraph({ + nodes: [n('root'), n('a'), n('b'), n('leaf')], + edges: [e('root', 'a'), e('a', 'b'), e('b', 'a'), e('b', 'leaf')] + }) + expect(pos.get('root')!.y).toBeLessThan(pos.get('a')!.y) + expect(pos.get('a')!.y).toBeLessThan(pos.get('b')!.y) + expect(pos.get('b')!.y).toBeLessThan(pos.get('leaf')!.y) + // A linear chain stays in one column. + expect(new Set(['root', 'a', 'b', 'leaf'].map((id) => pos.get(id)!.x)).size).toBe(1) + }) + + it('handles a longer cycle without dropping nodes', () => { + const pos = layoutAssetGraph({ + nodes: [n('a'), n('b'), n('c')], + edges: [e('a', 'b'), e('b', 'c'), e('c', 'a')] + }) + expect(pos.size).toBe(3) + const ys = ['a', 'b', 'c'].map((id) => pos.get(id)!.y) + expect(new Set(ys).size).toBe(3) }) it('packs disjoint components side by side without overlap', () => { diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts index cf55db6e0a..c9d7d60355 100644 --- a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts @@ -53,8 +53,8 @@ interface Band { // // y comes from longest-path layering (same top-down orientation as before: // producers above, assets in the middle, consumers below). Returns positions -// (band centers) normalized so the component's min x,y = 0. Throws on cyclic -// input (caller falls back to a grid for the whole graph). +// (band centers) normalized so the component's min x,y = 0. Cyclic input is +// handled by dropping feedback edges (see the Kahn step below). function layoutComponent( nodes: GraphInput['nodes'], edges: GraphInput['edges'] @@ -76,21 +76,42 @@ function layoutComponent( if (!parents.get(e.target)!.includes(e.source)) parents.get(e.target)!.push(e.source) } - // Kahn topological order — also the cycle guard. + // Kahn topological order. Cycles don't abort the layout: when the queue + // drains with nodes left, the unplaced node with the fewest outstanding + // parents (first in input order on ties) is forced into the order and its + // not-yet-placed parent edges are dropped as feedback edges — layering and + // tree-building then operate on the resulting DAG while the rendered graph + // keeps every edge. (The caller already resolves write⇄read 2-cycles by + // omitting the read direction; this handles any longer cycle.) const indeg = new Map() for (const n of nodes) indeg.set(n.id, parents.get(n.id)!.length) const queue = nodes.filter((n) => indeg.get(n.id) === 0).map((n) => n.id) + const placed = new Set() const topo: string[] = [] - while (queue.length) { + while (topo.length < nodes.length) { + if (queue.length === 0) { + let pick: string | undefined + for (const n of nodes) { + if (placed.has(n.id)) continue + if (pick === undefined || indeg.get(n.id)! < indeg.get(pick)!) pick = n.id + } + parents.set( + pick!, + parents.get(pick!)!.filter((p) => placed.has(p)) + ) + queue.push(pick!) + } const cur = queue.shift()! + if (placed.has(cur)) continue + placed.add(cur) topo.push(cur) for (const c of children.get(cur)!) { + if (placed.has(c)) continue const d = indeg.get(c)! - 1 indeg.set(c, d) if (d === 0) queue.push(c) } } - if (topo.length !== nodes.length) throw new Error('cyclic asset graph') // Longest-path layering: a node sits one layer below its lowest parent. const layer = new Map() @@ -141,8 +162,7 @@ function layoutComponent( out.set(id, { x: left + w / 2, y: layer.get(id)! * LAYER_H }) const kids = treeChildren.get(id)! if (kids.length === 0) return - const kidsW = - kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1) + const kidsW = kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1) let cursor = left + (w - kidsW) / 2 for (const k of kids) { placeTree(k, cursor) @@ -221,7 +241,8 @@ function layoutComponent( // disjoint components, so it's excluded from component detection and instead // re-placed centered one layer above the whole packed graph. // -// Falls back to a stable grid if the component layout throws (cyclic inputs). +// Falls back to a stable grid if the component layout throws (defensive — +// cycles are already absorbed by feedback-edge dropping in layoutComponent). export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map { const byId = new Map() if (graph.nodes.length === 0) return byId diff --git a/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts b/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts new file mode 100644 index 0000000000..69fc6f9728 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest' +import { runBackfill, type BackfillSliceState } from './backfillRun' + +// Deterministic fake backend: launch resolves with `job:`, +// waitTerminal resolves per the `results` table (default success), recording +// launch order. +function fakeRunner(results: Record = {}) { + const launched: string[] = [] + return { + launched, + launch: async (partition: string) => { + launched.push(partition) + return `job:${partition}` + }, + waitTerminal: async (jobId: string) => results[jobId.slice(4)] ?? ('success' as const) + } +} + +describe('runBackfill', () => { + it('runs slices sequentially in order and reports ok', async () => { + const r = fakeRunner() + const res = await runBackfill({ + partitions: ['2026-06-26', '2026-06-27', '2026-06-29'], + launch: r.launch, + waitTerminal: r.waitTerminal + }) + expect(r.launched).toEqual(['2026-06-26', '2026-06-27', '2026-06-29']) + expect(res.ok).toBe(true) + expect(res.cancelled).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'success', 'success']) + expect(res.slices.map((s) => s.jobId)).toEqual([ + 'job:2026-06-26', + 'job:2026-06-27', + 'job:2026-06-29' + ]) + }) + + it('continues past a failed slice — each slice is independent', async () => { + const r = fakeRunner({ '2026-06-27': 'failure' }) + const res = await runBackfill({ + partitions: ['2026-06-26', '2026-06-27', '2026-06-29'], + launch: r.launch, + waitTerminal: r.waitTerminal + }) + expect(r.launched).toEqual(['2026-06-26', '2026-06-27', '2026-06-29']) + expect(res.ok).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'failure', 'success']) + }) + + it('records a launch error as slice failure and keeps going', async () => { + const r = fakeRunner() + const res = await runBackfill({ + partitions: ['a', 'b'], + launch: async (p) => { + if (p === 'a') throw new Error('boom') + return r.launch(p) + }, + waitTerminal: r.waitTerminal + }) + expect(res.slices[0]).toMatchObject({ status: 'failure', error: 'boom' }) + expect(res.slices[1].status).toBe('success') + }) + + it('stops before the next launch when cancelled, leaving the rest pending', async () => { + const r = fakeRunner() + let done = 0 + const res = await runBackfill({ + partitions: ['a', 'b', 'c'], + launch: r.launch, + waitTerminal: async (id) => { + done++ + return r.waitTerminal(id) + }, + isCancelled: () => done >= 1 + }) + expect(r.launched).toEqual(['a']) + expect(res.cancelled).toBe(true) + expect(res.ok).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'pending', 'pending']) + }) + + it('cancels a job whose launch raced the cancellation', async () => { + let cancelled = false + const cancelledJobs: string[] = [] + const res = await runBackfill({ + partitions: ['a', 'b'], + launch: async (p) => { + // The user clicks cancel while the launch request is in flight — + // there is no job id to cancel yet. + cancelled = true + return `job:${p}` + }, + waitTerminal: async () => 'failure', + isCancelled: () => cancelled, + cancelJob: async (id) => { + cancelledJobs.push(id) + } + }) + expect(cancelledJobs).toEqual(['job:a']) + expect(res.cancelled).toBe(true) + expect(res.slices.map((s) => s.status)).toEqual(['failure', 'pending']) + }) + + it('emits a snapshot per transition, never mutating earlier snapshots', async () => { + const r = fakeRunner() + const snapshots: BackfillSliceState[][] = [] + await runBackfill({ + partitions: ['a'], + launch: r.launch, + waitTerminal: r.waitTerminal, + onUpdate: (s) => snapshots.push(s) + }) + // initial pending, running, running+jobId, terminal + expect(snapshots.map((s) => s[0].status)).toEqual(['pending', 'running', 'running', 'success']) + expect(snapshots[1][0].jobId).toBeUndefined() + expect(snapshots[2][0].jobId).toBe('job:a') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts b/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts new file mode 100644 index 0000000000..51fbc9664b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts @@ -0,0 +1,83 @@ +// Client-side orchestration of a partition-range backfill (enterprise): one +// deployed run of the producing script per slice, launched with an explicit +// `partition` arg (the worker only resolves a partition when the arg is +// absent, so the caller-provided value wins). Slices run sequentially — +// concurrent materializations of the same ducklake table would contend on +// the catalog commit — and a failed slice does not stop the rest: each slice +// is independent, and the missing/failed set is simply the next worklist. +// +// Pure module (no Svelte runes) so the loop is unit-testable; reactive +// progress is delivered via `onUpdate` snapshots, mirroring +// `cascadeOrchestrator.ts`. + +export type BackfillSliceStatus = 'pending' | 'running' | 'success' | 'failure' + +export type BackfillSliceState = { + partition: string + status: BackfillSliceStatus + jobId?: string + error?: string +} + +export type BackfillRunOptions = { + /** Partition values to materialize, in run order. */ + partitions: string[] + /** Launch one run of the producer with the given partition arg; returns the job id. */ + launch: (partition: string) => Promise + /** Resolve once the job reaches a terminal state. */ + waitTerminal: (jobId: string) => Promise<'success' | 'failure'> + /** Snapshot of all slice states, emitted on every transition. */ + onUpdate?: (slices: BackfillSliceState[]) => void + /** Checked before each launch; a true stop leaves the remaining slices 'pending'. */ + isCancelled?: () => boolean + /** + * Cancel a job whose launch raced the cancellation — the cancel click had + * no job id to act on yet, so the loop cancels it as soon as the id + * arrives. Must not throw (the job may already be terminal). + */ + cancelJob?: (jobId: string) => Promise +} + +export type BackfillRunResult = { + /** True when every slice ran and succeeded. */ + ok: boolean + /** True when the loop stopped early on `isCancelled`. */ + cancelled: boolean + slices: BackfillSliceState[] +} + +export async function runBackfill(opts: BackfillRunOptions): Promise { + const { partitions, launch, waitTerminal, onUpdate, isCancelled, cancelJob } = opts + const slices: BackfillSliceState[] = partitions.map((partition) => ({ + partition, + status: 'pending' + })) + const emit = () => onUpdate?.(slices.map((s) => ({ ...s }))) + emit() + let cancelled = false + for (const slice of slices) { + if (isCancelled?.()) { + cancelled = true + break + } + slice.status = 'running' + emit() + try { + slice.jobId = await launch(slice.partition) + emit() + if (isCancelled?.() && cancelJob) { + await cancelJob(slice.jobId) + } + slice.status = await waitTerminal(slice.jobId) + } catch (e) { + slice.status = 'failure' + slice.error = e instanceof Error ? e.message : String(e) + } + emit() + } + return { + ok: !cancelled && slices.every((s) => s.status === 'success'), + cancelled, + slices + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts index fa735682b7..e4692de6b8 100644 --- a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts @@ -7,24 +7,29 @@ import { buildLineageDag, buildLineageDownstreamMap, descendants, + nonAutorunTriggerScripts, + reachableCutting, scriptNodeId, scriptsOf, - validStarts + validStarts, + validFromStarts } from './boundedCascade' import { computeInducedSchedule } from './graphTraversal' type W = [script: string, asset: string] // producer write edge (datatable) type R = [script: string, asset: string] // pure-read edge (datatable) type S = [script: string, asset: string] // `// on ` subscription +type T = [producer: string, tested: string, asset: string] // `// data_test` ordering edge function graph(opts: { scripts?: string[] writes?: W[] reads?: R[] subs?: S[] + tests?: T[] native?: Array<[kind: NativeTriggerKind, script: string]> }): AssetGraphResponse { - const { scripts = [], writes = [], reads = [], subs = [], native = [] } = opts + const { scripts = [], writes = [], reads = [], subs = [], tests = [], native = [] } = opts const triggers: AssetGraphTrigger[] = [ ...subs.map( ([s, a]) => @@ -60,7 +65,15 @@ function graph(opts: { access_type: 'r' as const })) ], - triggers + triggers, + test_edges: tests.map(([producer, tested, a]) => ({ + producer_kind: 'script' as const, + producer_path: producer, + runnable_kind: 'script' as const, + runnable_path: tested, + asset_kind: 'datatable' as const, + asset_path: a + })) } } @@ -95,6 +108,22 @@ describe('buildLineageDag', () => { expect([...(dag.down.get(sn('u')) ?? [])]).toEqual([asset('x')]) expect(dag.up.get(sn('u'))).toBeUndefined() // asset is not upstream of its own writer }) + + it('routes a data_test ordering edge through the referenced asset', () => { + // prod writes x; tested has a `// data_test` against x (prod → tested edge). + // The DAG must place x (and thus prod) upstream of tested so a cascade + // materializes x first. + const g = graph({ + scripts: ['prod', 'tested'], + writes: [['prod', 'x']], + tests: [['prod', 'tested', 'x']] + }) + const dag = buildLineageDag(g) + // asset x → tested (routed through the asset, not a direct prod → tested hop) + expect([...(dag.down.get(asset('x')) ?? [])]).toEqual([sn('tested')]) + // prod → x → tested makes prod an ancestor of tested. + expect(ancestors(dag, sn('tested'))).toEqual(new Set([asset('x'), sn('prod')])) + }) }) describe('ancestors / descendants', () => { @@ -247,6 +276,165 @@ describe('validStarts', () => { }) }) +describe('validFromStarts (mid-DAG selective execution)', () => { + // a → x → sub (subscriber) → y → reader (pure read). `k` is event-triggered. + const g = () => + graph({ + scripts: ['a', 'sub', 'reader', 'k'], + writes: [ + ['a', 'x'], + ['sub', 'y'] + ], + reads: [['reader', 'y']], + subs: [['sub', 'x']], + native: [['kafka', 'k']] + }) + + it('includes mid-DAG subscribers and pure readers, not just roots', () => { + const from = validFromStarts(g()) + expect(from.has(sn('a'))).toBe(true) // root + expect(from.has(sn('sub'))).toBe(true) // mid-DAG subscriber — NOT a validStart + expect(from.has(sn('reader'))).toBe(true) // pure reader + // The old root-only gate would have rejected the mid-DAG nodes. + expect(validStarts(g()).has(sn('sub'))).toBe(false) + }) + + it('excludes event-triggered scripts (no run-now gesture)', () => { + expect(validFromStarts(g()).has(sn('k'))).toBe(false) + }) + + it('excludes webhook/data_upload mid-DAG subscribers (need caller input)', () => { + // a → x → upload_mid (subscribes x AND `// on data_upload`) → y → consumer. + // upload_mid must NOT be an eligible start (empty-arg run has no S3Object), + // and must be a barrier so `consumer` isn't run with a skipped producer. + const g2 = graph({ + scripts: ['a', 'upload_mid', 'hook_mid', 'consumer'], + writes: [ + ['a', 'x'], + ['upload_mid', 'y'] + ], + subs: [ + ['upload_mid', 'x'], + ['hook_mid', 'x'], + ['consumer', 'y'] + ], + native: [ + ['data_upload', 'upload_mid'], + ['webhook', 'hook_mid'] + ] + }) + const from = validFromStarts(g2) + expect(from.has(sn('upload_mid'))).toBe(false) + expect(from.has(sn('hook_mid'))).toBe(false) + expect(from.has(sn('a'))).toBe(true) // the plain root is still eligible + // and they're barriers, so running downstream from `a` cuts them + consumer. + const bars = nonAutorunTriggerScripts(g2) + expect(bars.has(sn('upload_mid'))).toBe(true) + expect(bars.has(sn('hook_mid'))).toBe(true) + expect(scriptsOf(reachableCutting(buildLineageDag(g2), [sn('a')], bars)).sort()).toEqual(['a']) + }) + + it('keeps a scheduled root that also carries an event trigger', () => { + // schedule wins over the secondary kafka trigger in validStarts, so the + // scheduled root stays --from-eligible (regression guard). + const g2 = graph({ + scripts: ['sched_evt', 'consumer'], + writes: [['sched_evt', 'x']], + subs: [['consumer', 'x']], + native: [ + ['schedule', 'sched_evt'], + ['kafka', 'sched_evt'] + ] + }) + expect(validStarts(g2).has(sn('sched_evt'))).toBe(true) + expect(nonAutorunTriggerScripts(g2).has(sn('sched_evt'))).toBe(true) + expect(validFromStarts(g2).has(sn('sched_evt'))).toBe(true) // union with roots + }) + + it('runs a mid-DAG start plus its downstream WITHOUT re-running upstream', () => { + // Starting at `sub`, the unbounded downstream is {sub, y, reader} — `a`/`x` + // upstream are never pulled in (dbt `--select sub+`). + const dag = buildLineageDag(g()) + const downstream = new Set([sn('sub'), ...descendants(dag, sn('sub'))]) + expect(scriptsOf(downstream).sort()).toEqual(['reader', 'sub']) + expect(downstream.has(sn('a'))).toBe(false) + expect(downstream.has(asset('x'))).toBe(false) + }) +}) + +describe('reachableCutting (barrier cut for "Run + downstream")', () => { + // a → x → k(kafka, also reads x) → z → consumer. Starting at `a` and running + // downstream must cut the event handler `k` AND `consumer` (only reachable + // through it) — else they'd launch with empty args. Mirrors the CLI cut. + const g = () => + graph({ + scripts: ['a', 'k', 'consumer'], + writes: [ + ['a', 'x'], + ['k', 'z'] + ], + reads: [['k', 'x']], + subs: [['consumer', 'z']], + native: [['kafka', 'k']] + }) + + it('detects event-triggered scripts as barriers', () => { + expect(nonAutorunTriggerScripts(g()).has(sn('k'))).toBe(true) + expect(nonAutorunTriggerScripts(g()).has(sn('a'))).toBe(false) + }) + + it('cuts an event descendant and its event-only downstream from a run set', () => { + const dag = buildLineageDag(g()) + const barriers = nonAutorunTriggerScripts(g()) + const runNodes = reachableCutting(dag, [sn('a')], barriers) + expect(scriptsOf(runNodes).sort()).toEqual(['a']) // k + consumer cut + expect(runNodes.has(sn('k'))).toBe(false) + expect(runNodes.has(sn('consumer'))).toBe(false) + }) + + it('protects an explicit start even if it carries an event trigger', () => { + const dag = buildLineageDag(g()) + // start = k itself (user named it); it runs, and so does its downstream. + const barriers = new Set([...nonAutorunTriggerScripts(g())].filter((id) => id !== sn('k'))) + const runNodes = reachableCutting(dag, [sn('k')], barriers) + expect(scriptsOf(runNodes).sort()).toEqual(['consumer', 'k']) + }) + + it('keeps a scheduled event root (and its downstream) reachable from an upstream start', () => { + // a → x → sched_evt(schedule + kafka) → y → consumer. Running downstream + // from `a`, sched_evt is a scheduled root so it must NOT be a barrier even + // though it carries an event trigger — matching the CLI, which excludes all + // valid roots (`starts`), not just the picked start. Mirror of the page's + // barrier construction: nonAutorunTriggerScripts minus validStarts minus start. + const g2 = graph({ + scripts: ['a', 'sched_evt', 'consumer'], + writes: [ + ['a', 'x'], + ['sched_evt', 'y'] + ], + subs: [ + ['sched_evt', 'x'], + ['consumer', 'y'] + ], + native: [ + ['schedule', 'sched_evt'], + ['kafka', 'sched_evt'] + ] + }) + const dag = buildLineageDag(g2) + const roots = validStarts(g2) + const barriers = new Set( + [...nonAutorunTriggerScripts(g2)].filter((id) => !roots.has(id) && id !== sn('a')) + ) + const runNodes = reachableCutting(dag, [sn('a')], barriers) + expect(scriptsOf(runNodes).sort()).toEqual(['a', 'consumer', 'sched_evt']) + // Without the validStarts exclusion, sched_evt (a kafka handler) would be a + // barrier and `consumer` would be dropped — the bug this guards. + const naiveBarriers = new Set([...nonAutorunTriggerScripts(g2)].filter((id) => id !== sn('a'))) + expect(scriptsOf(reachableCutting(dag, [sn('a')], naiveBarriers)).sort()).toEqual(['a']) + }) +}) + describe('buildLineageDownstreamMap (read-aware scheduling)', () => { // a writes x; c only *reads* x (no `// on x`). c must still run after a. const g = graph({ @@ -271,6 +459,47 @@ describe('buildLineageDownstreamMap (read-aware scheduling)', () => { expect(readAware.indegree.get('c')).toBe(1) expect(readAware.nodes).toEqual(['a', 'c']) }) + + it('orders a disjoint-root producer before a data_test that references it', () => { + // Two disjoint roots (the HD-1 repro): `dim` produces the dimension + // `dimc`; `fct` produces `fcto` and has a `// data_test relationships` + // against `dimc`. Without the test edge they are unordered and a cold + // cascade can run `fct` first ("table dimc does not exist"). The test + // edge must place `dim` strictly before `fct`. + const g = graph({ + scripts: ['dim', 'fct'], + writes: [ + ['dim', 'dimc'], + ['fct', 'fcto'] + ], + tests: [['dim', 'fct', 'dimc']] + }) + const selected = new Set(['dim', 'fct']) + const map = buildLineageDownstreamMap(g) + expect([...(map.get('dim') ?? [])]).toEqual(['fct']) + const schedule = computeInducedSchedule(g, selected, map) + expect(schedule.roots).toEqual(['dim']) + expect(schedule.indegree.get('fct')).toBe(1) + expect(schedule.nodes).toEqual(['dim', 'fct']) + expect(schedule.cyclic).toEqual([]) + }) + + it('adds no ordering when the referenced asset has no in-pipeline producer', () => { + // `fct` tests against an external `ext` table nothing produces — the + // backend emits no test edge, so the frontend sees none and `fct` stays + // an independent root (the runtime error stands, as designed). + const g = graph({ + scripts: ['dim', 'fct'], + writes: [['fct', 'fcto']], + tests: [] // no producer for `ext` ⇒ backend omitted the edge + }) + const schedule = computeInducedSchedule( + g, + new Set(['dim', 'fct']), + buildLineageDownstreamMap(g) + ) + expect(schedule.roots.sort()).toEqual(['dim', 'fct']) + }) }) describe('assetUriToNodeId', () => { @@ -280,4 +509,15 @@ describe('assetUriToNodeId', () => { expect(assetUriToNodeId('ducklake://lake/t')).toBe('ducklake:lake/t') expect(assetUriToNodeId('not-a-uri')).toBeUndefined() }) + it('strips leading slashes from S3 keys so s3:/// and s3:// share a node', () => { + // Mirror of Rust `parse_asset_syntax`: `--to s3:///exports/x` must resolve + // to the same canonical node as the graph's `s3object:exports/x`. + expect(assetUriToNodeId('s3:///exports/x')).toBe('s3object:exports/x') + expect(assetUriToNodeId('s3:///exports/x')).toBe(assetUriToNodeId('s3://exports/x')) + // All leading slashes are stripped so a canonical key never starts with + // `/` (the quad-slash `S3Object(s3="/x")` form collapses to `x`). + expect(assetUriToNodeId('s3:////x')).toBe('s3object:x') + // Hive-partition keys and non-S3 kinds are untouched. + expect(assetUriToNodeId('s3:///t/y=2024/f.parquet')).toBe('s3object:t/y=2024/f.parquet') + }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts index 44a26d9c71..6fcabbe71f 100644 --- a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts +++ b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts @@ -38,14 +38,16 @@ export function assetUriToNodeId(uri: string): string | undefined { // `s3` is the URI prefix for the `s3object` asset kind (mirrors the CLI // `assetUri` and the canvas). All other kinds use their name verbatim. const kind = prefix === 's3' ? 's3object' : prefix - return `${kind}:${m[2]}` + // Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys so + // `s3:///key` (default storage) and `s3://key` resolve to the same node id + // and a canonical key never starts with `/`. + const path = kind === 's3object' ? m[2].replace(/^\/+/, '') : m[2] + return `${kind}:${path}` } // Native trigger kinds that fan out *per event*: a single event always flows // through the whole reactive downstream, so "run up to X now" is not a // meaningful gesture and these are never offered as bounded-run starts. -// `webhook`/`data_upload` have no trigger row in `/assets/graph`, so a root -// whose only entry is one of those reads as a *manual* root below. const EVENT_TRIGGER_KINDS: ReadonlySet = new Set([ 'kafka', 'mqtt', @@ -56,6 +58,20 @@ const EVENT_TRIGGER_KINDS: ReadonlySet = new Set([ 'email' ]) +// Trigger kinds a cascade must NOT auto-run: the per-event kinds above PLUS the +// input-only entrypoints `webhook`/`data_upload`, which need caller-supplied +// input (a request body / an uploaded S3Object) and would run the wrong thing +// with empty args. Mirror of the CLI `NON_AUTORUN_TRIGGER_KINDS`. When the +// deployed `/assets/graph` omits a `webhook`/`data_upload` row (it has none for +// them), such a script simply reads as a manual root here — but whenever the +// marker IS visible (editor overlay / draft), it's excluded from bounded-run +// starts and cut as a barrier, exactly as the CLI does. +const NON_AUTORUN_TRIGGER_KINDS: ReadonlySet = new Set([ + ...EVENT_TRIGGER_KINDS, + 'webhook', + 'data_upload' +]) + export type LineageDag = { /** upstream node id → set of direct downstream node ids. */ down: Map> @@ -70,10 +86,17 @@ export type LineageDag = { * - producer script → asset (write / rw edges) * - asset → reader script (pure-read edges — a data dependency) * - asset → subscriber script (`// on ` triggers) + * - asset → testing script (`// data_test` ordering edges) * * An `rw` edge is treated as production only (script → asset); emitting the * reverse asset → script too would make every upsert a 2-cycle through its own * asset. + * + * `test_edges` are modeled through the referenced asset (asset → testing + * script), NOT as a direct producer → testing-script hop: the producer already + * has a write edge to that asset, so this yields producer → asset → testing + * script and keeps the two-hop (script → asset → script) invariant that + * `buildLineageDownstreamMap` relies on. */ export function buildLineageDag(g: AssetGraphResponse): LineageDag { const down = new Map>() @@ -107,6 +130,13 @@ export function buildLineageDag(g: AssetGraphResponse): LineageDag { if (t.trigger_kind !== 'asset' || t.runnable_kind !== 'script') continue addEdge(assetKey(t), scriptNodeId(t.runnable_path)) } + // Data-test ordering edges: the referenced asset must exist before the + // tested script runs. Routed through the asset node so the existing + // producer → asset write edge extends into producer → asset → testing script. + for (const t of g.test_edges ?? []) { + if (t.runnable_kind !== 'script') continue + addEdge(assetKey(t), scriptNodeId(t.runnable_path)) + } return { down, up, nodes } } @@ -138,6 +168,37 @@ export function ancestors(dag: LineageDag, n: string): Set { return closure(dag.up, n) } +/** + * Nodes reachable from `starts` over the lineage DAG, treating `barriers` as cut + * points: a barrier node is neither included NOR traversed through, so a node + * reachable ONLY via a barrier is excluded while one also reachable via another + * path stays. Mirror of the CLI `reachableCutting`. Used to keep event handlers + * (and their event-only downstream) out of a cascade run — running such a + * consumer whose producer was skipped would feed it missing/stale inputs. + */ +export function reachableCutting( + dag: LineageDag, + starts: Iterable, + barriers: Set +): Set { + const seen = new Set() + const queue: string[] = [] + for (const s of starts) { + if (barriers.has(s) || seen.has(s)) continue + seen.add(s) + queue.push(s) + } + while (queue.length > 0) { + const n = queue.shift()! + for (const next of dag.down.get(n) ?? []) { + if (barriers.has(next) || seen.has(next)) continue + seen.add(next) + queue.push(next) + } + } + return seen +} + export type BoundedResult = { /** Path-between node set (scripts + assets), always including `start`. */ nodes: Set @@ -201,6 +262,56 @@ export function validStarts(g: AssetGraphResponse): Set { return out } +/** + * Script node ids eligible as an EXPLICIT bounded-run start from *anywhere* in + * the DAG (dbt's `--select model+`): every script that can run with empty args — + * i.e. all scripts except event-triggered ones (kafka/mqtt/nats/postgres/sqs/ + * gcp/email fan out per event and have no "run now" gesture). Unlike + * `validStarts` (schedule/manual roots only), this INCLUDES mid-DAG asset + * subscribers and pure readers, so "Run + downstream" can begin at any model — + * that node plus its transitive downstream runs, upstream is never re-run. + * (`webhook`/`data_upload` have no trigger row here — same as `validStarts` they + * read as manual roots and are already included.) + */ +export function validFromStarts(g: AssetGraphResponse): Set { + const nonAutorunScripts = new Set() + for (const t of g.triggers ?? []) { + if (t.runnable_kind !== 'script') continue + if (NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) nonAutorunScripts.add(t.runnable_path) + } + // Seed with schedule/manual roots: `validStarts` lets a schedule identity win + // over a secondary non-autorun trigger, so a scheduled root that also carries + // e.g. a `// on kafka` stays `--from`-eligible. The mid-DAG loop then adds only + // scripts that can run with empty args — a `webhook`/`data_upload` subscriber + // is NOT added (it needs caller-supplied input). + const out = new Set(validStarts(g)) + for (const r of g.runnables ?? []) { + if (r.usage_kind !== 'script') continue + if (!nonAutorunScripts.has(r.path)) out.add(scriptNodeId(r.path)) + } + return out +} + +/** + * Script node ids carrying a non-autorun trigger (event kinds kafka/mqtt/…/email + * PLUS input-only webhook/data_upload) — they fan out per event or need + * caller-supplied input, so a cascade must cut them (as `barriers` for + * `reachableCutting`) even when they're a lineage descendant of the start. + * Mirror of the CLI `nonAutorunTriggerScripts`. Only detects what the graph + * surfaces: the deployed `/assets/graph` omits `webhook`/`data_upload` rows, so + * such a handler is only cut when its marker is visible (editor overlay / draft) + * — the same limitation as `validStarts`; the CLI closes it via graph enrichment. + */ +export function nonAutorunTriggerScripts(g: AssetGraphResponse): Set { + const out = new Set() + for (const t of g.triggers ?? []) { + if (t.runnable_kind === 'script' && NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) { + out.add(scriptNodeId(t.runnable_path)) + } + } + return out +} + /** Project a node-id set to the script paths it contains (run targets). */ export function scriptsOf(nodes: Iterable): string[] { const out: string[] = [] diff --git a/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.test.ts b/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.test.ts index 92d7a977fe..3a0fd4fbd3 100644 --- a/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.test.ts @@ -201,7 +201,48 @@ describe('runSelection', () => { expect(r.launched.indexOf('c')).toBeGreaterThan(r.launched.indexOf('b')) }) - it('stops scheduling after a failure', async () => { + it('skips a failed node’s descendants but keeps independent branches running', async () => { + // Two independent chains: a → b and c → d. `a` fails; `b` (its descendant) + // must be skipped, but `d` depends only on the successful `c`, so it must + // still run — a failure must not stall unrelated branches. + const sched = schedule( + [ + ['a', 'b'], + ['c', 'd'] + ], + ['a', 'b', 'c', 'd'], + ['a', 'c'] + ) + const r = fakeRunner({ a: 'failure' }) + const res = await runSelection({ schedule: sched, ...r }) + expect(res.ok).toBe(false) + expect(res.statuses.get('a')?.status).toBe('failure') + expect(res.statuses.get('b')?.status).toBe('skipped') + expect(res.statuses.get('c')?.status).toBe('success') + expect(res.statuses.get('d')?.status).toBe('success') + expect(r.launched).toContain('d') + expect(r.launched).not.toContain('b') + }) + + it('skips a join node when any one of its upstreams fails', async () => { + // {a, b} → c. `a` fails; `c` needs both, so it must be skipped even though + // `b` succeeds — a poisoned lineage isn’t rescued by a sibling success. + const sched = schedule( + [ + ['a', 'c'], + ['b', 'c'] + ], + ['a', 'b', 'c'], + ['a', 'b'] + ) + const r = fakeRunner({ a: 'failure' }) + const res = await runSelection({ schedule: sched, ...r }) + expect(res.ok).toBe(false) + expect(res.statuses.get('c')?.status).toBe('skipped') + expect(r.launched).not.toContain('c') + }) + + it('stops scheduling a failed node’s chain', async () => { const sched = schedule([['a', 'b']], ['a', 'b'], ['a']) const r = fakeRunner({ a: 'failure' }) const res = await runSelection({ schedule: sched, ...r }) diff --git a/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.ts b/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.ts index 16d357af00..424f1581ec 100644 --- a/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.ts +++ b/frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.ts @@ -128,8 +128,9 @@ export type SelectionRunOptions = { * Execute an arbitrary selected set of scripts (e.g. a bounded-cascade * selection) in topological order. Unlike `runCascade` there is no single * privileged root — every `schedule.roots` entry is seeded at once and a node - * runs as soon as its in-set upstreams all succeed. Failure stops *scheduling* - * (in-flight jobs finish); everything not yet started ends 'skipped'. + * runs as soon as its in-set upstreams all succeed. A failure abandons only + * that node's lineage (its transitive descendants end 'skipped'); INDEPENDENT + * branches keep running. In-flight jobs always finish. */ export async function runSelection(opts: SelectionRunOptions): Promise { const { schedule, launch, waitTerminal, onUpdate } = opts @@ -137,10 +138,29 @@ export async function runSelection(opts: SelectionRunOptions): Promise() const inFlight = new Set>() const emit = () => onUpdate?.(new Map(statuses)) + function poison(path: string) { + const stack = [path] + while (stack.length > 0) { + const n = stack.pop()! + for (const s of schedule.edges.get(n) ?? []) { + if (!poisoned.has(s)) { + poisoned.add(s) + stack.push(s) + } + } + } + } + function schedule_(path: string) { const p = runNode(path).finally(() => inFlight.delete(p)) inFlight.add(p) @@ -159,6 +179,7 @@ export async function runSelection(opts: SelectionRunOptions): Promise vi.restoreAllMocks()) + +// The client orchestrates the cascade closure, so every launch must carry +// `_wmill_skip_asset_dispatch: true`. A caller arg (e.g. the run form for the +// root node) must NOT be able to override that guard back to false and let the +// backend also dispatch deployed subscribers. +describe('makeLaunch dispatch guard', () => { + it('local preview: a caller `_wmill_skip_asset_dispatch: false` cannot re-enable dispatch', async () => { + const spy = vi.spyOn(JobService, 'runScriptPreview').mockResolvedValue('job-1' as any) + const launch = makeLaunch({ + workspace: 'w', + resolveLocal: () => ({ content: 'x', language: 'bun' as any }), + argsFor: () => ({ _wmill_skip_asset_dispatch: false, foo: 1 }) + }) + await launch('f/x/root') + const body = (spy.mock.calls[0][0] as any).requestBody + expect(body.args._wmill_skip_asset_dispatch).toBe(true) // guard wins + expect(body.args.foo).toBe(1) // other caller args preserved + }) + + it('deployed by-path: a caller `_wmill_skip_asset_dispatch: false` cannot re-enable dispatch', async () => { + const spy = vi.spyOn(JobService, 'runScriptByPath').mockResolvedValue('job-2' as any) + const launch = makeLaunch({ + workspace: 'w', + argsFor: () => ({ _wmill_skip_asset_dispatch: false }) + }) + await launch('f/x/deployed') + const body = (spy.mock.calls[0][0] as any).requestBody + expect(body._wmill_skip_asset_dispatch).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts b/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts new file mode 100644 index 0000000000..ede66ebe4b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts @@ -0,0 +1,149 @@ +// Reusable cascade-execution primitives shared by the pipeline route page and +// the local-dev preview (`PipelineDevView`). The backend asset-trigger +// dispatcher only resolves DEPLOYED rows, so whenever a run involves local / +// draft content the client must orchestrate the closure itself: topological +// order over the graph the user is looking at, each node launched with +// `_wmill_skip_asset_dispatch` so the backend never double-fires the deployed +// part of a mixed chain. + +import { JobService, type Preview } from '$lib/gen' +import { + runCascade, + runSelection, + type CascadeRunResult, + type CascadeNodeState +} from './cascadeOrchestrator' +import { computeDownstreamClosure, computeInducedSchedule } from './graphTraversal' +import { buildLineageDownstreamMap } from './boundedCascade' +import type { AssetGraphResponse } from './types' + +export const CASCADE_POLL_INTERVAL_MS = 1000 +export const CASCADE_JOB_TIMEOUT_MS = 30 * 60 * 1000 + +export type LocalScriptContent = { + content: string + language: Preview['language'] + // `// tag ` — routes the preview to that worker (deployed parity). + tag?: string +} + +// Poll a launched cascade job to a terminal state. Capped so a never-terminating +// job can't pin a run guard forever; on timeout it throws, surfaced as a chain +// failure by the orchestrator. +export function makeWaitJobTerminal( + workspace: string +): (jobId: string) => Promise<'success' | 'failure'> { + return async function waitJobTerminal(jobId: string): Promise<'success' | 'failure'> { + const deadline = Date.now() + CASCADE_JOB_TIMEOUT_MS + while (Date.now() < deadline) { + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace, + id: jobId, + getStarted: false + }) + if (r.completed) return r.success ? 'success' : 'failure' + } catch { + // transient — retry on the next tick + } + await new Promise((res) => setTimeout(res, CASCADE_POLL_INTERVAL_MS)) + } + throw new Error( + `Timed out after ${Math.round(CASCADE_JOB_TIMEOUT_MS / 60000)}min waiting for job ${jobId} to finish` + ) + } +} + +// Build a per-script launch function. When `resolveLocal(path)` yields content, +// the script runs as a preview of that local content (no deploy); otherwise it +// runs the deployed version by path. Always passes `_wmill_skip_asset_dispatch`. +export function makeLaunch(opts: { + workspace: string + resolveLocal?: (path: string) => LocalScriptContent | undefined + tempScriptRefs?: Record + // Extra run args for a specific node (e.g. the uploaded S3Object bound to a + // `data_upload` cascade root). Merged over `_wmill_skip_asset_dispatch`; all + // other nodes run with empty inputs as before. + argsFor?: (path: string) => Record | undefined + onLaunched?: (path: string, jobId: string) => void +}): (path: string) => Promise { + return async function launch(path: string): Promise { + const local = opts.resolveLocal?.(path) + // Caller args (e.g. the run form for the cascade root) must NOT be able to + // re-enable backend asset dispatch while the client orchestrates the closure + // — that would double-run downstream / run deployed subscribers. Drop any + // `_wmill_skip_asset_dispatch` a caller supplied, and always spread it LAST. + const { _wmill_skip_asset_dispatch: _reserved, ...extra } = opts.argsFor?.(path) ?? {} + void _reserved + let jobId: string + if (local) { + if (!local.content || !local.language) { + throw new Error(`local script ${path} has no content/language`) + } + jobId = await JobService.runScriptPreview({ + workspace: opts.workspace, + requestBody: { + content: local.content, + language: local.language, + path, + args: { ...extra, _wmill_skip_asset_dispatch: true }, + ...(local.tag ? { tag: local.tag } : {}), + ...(opts.tempScriptRefs ? { temp_script_refs: opts.tempScriptRefs } : {}) + } + }) + } else { + jobId = await JobService.runScriptByPath({ + workspace: opts.workspace, + path, + requestBody: { ...extra, _wmill_skip_asset_dispatch: true } + }) + } + opts.onLaunched?.(path, jobId) + return jobId + } +} + +// Run `root` plus its full downstream closure over the given graph. +export async function runDownstreamCascade(opts: { + graph: AssetGraphResponse + root: string + launch: (path: string) => Promise + waitTerminal: (jobId: string) => Promise<'success' | 'failure'> + onUpdate?: (statuses: Map) => void +}): Promise { + const closure = computeDownstreamClosure(opts.graph, opts.root) + const res = await runCascade({ + closure, + root: opts.root, + launch: opts.launch, + waitTerminal: opts.waitTerminal, + onUpdate: opts.onUpdate + }) + return { ...res, cyclic: closure.cyclic } +} + +// Run a bounded selection of scripts (the induced schedule over the lineage DAG). +export async function runBoundedCascade(opts: { + graph: AssetGraphResponse + scripts: Set + launch: (path: string) => Promise + waitTerminal: (jobId: string) => Promise<'success' | 'failure'> + onUpdate?: (statuses: Map) => void +}): Promise { + // Read-aware adjacency (NOT the default write-edge map) so a pure-reader + // member runs after its producer — parity with the route page's bounded run + // and the CLI `topoOrder`. `cyclic` is surfaced so callers can warn instead of + // silently dropping scripts stuck on a dependency cycle. + const schedule = computeInducedSchedule( + opts.graph, + opts.scripts, + buildLineageDownstreamMap(opts.graph) + ) + const res = await runSelection({ + schedule, + launch: opts.launch, + waitTerminal: opts.waitTerminal, + onUpdate: opts.onUpdate + }) + return { ...res, cyclic: schedule.cyclic } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/graphTraversal.ts b/frontend/src/lib/components/assets/AssetGraph/graphTraversal.ts index b89569e754..cc59e8f3a8 100644 --- a/frontend/src/lib/components/assets/AssetGraph/graphTraversal.ts +++ b/frontend/src/lib/components/assets/AssetGraph/graphTraversal.ts @@ -1,6 +1,31 @@ -import type { AssetGraphResponse } from './types' +import type { AssetGraphResponse, AssetGraphSelection } from './types' import { assetKey, buildAssetSubscribers, isWriteEdge } from './lib' +// Scripts that WRITE the selected asset (its producers), from the graph's +// `w`/`rw` lineage edges. `[]` for a non-asset selection. Shared by the route +// page and the dev preview so "who writes this asset" is defined once and can't +// drift between the two surfaces. +export function assetProducers( + graph: AssetGraphResponse, + selection: AssetGraphSelection | undefined +): Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> { + if (!selection || selection.kind !== 'asset') return [] + return graph.edges + .filter((e) => { + const access = e.access_type ?? 'r' + return ( + (access === 'w' || access === 'rw') && + e.asset_kind === selection.asset_kind && + e.asset_path === selection.path + ) + }) + .map((e) => ({ + kind: e.runnable_kind as 'script' | 'flow', + path: e.runnable_path, + unsaved: e.unsaved + })) +} + // Execution-DAG traversal over the resolved asset graph (drafts included). // // The execution edges are producer→subscriber: a script S1 *produces* an diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts index 3eb1e4e574..0ad56c81f9 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts @@ -20,7 +20,9 @@ const ASSERTED_TS_FIELDS: Record = { retry: true, materialize: true, dataTests: true, - columnLineage: true + columnLineage: true, + macros: true, + useLibs: true } // Parser-parity guard: this TS parser (drives the live graph preview) and @@ -67,6 +69,11 @@ type Fixture = { manual?: boolean append?: boolean unique_key?: string | null + scd2?: boolean + track?: string[] + close_deleted?: boolean + // "warn" | "ignore"; absent === "warn" (the default) + on_schema_change?: string } | null // Snake_case form matching the Rust `DataTest` serde output, so the one // corpus drives both sides. The TS parser emits this shape verbatim @@ -75,6 +82,10 @@ type Fixture = { // Snake_case `ColumnLineage` serde shape — TS parser emits it verbatim, // so the comparison is 1:1. Absent === []. column_lineage?: Array> + // `// macros` marker. Absent === false. + macros?: boolean + // `// use ` accumulation, declaration order, deduped. Absent === []. + use_libs?: string[] } } @@ -162,11 +173,27 @@ describe('parsePipelineAnnotations matches the shared Rust fixture corpus', () = expect(got.materialize?.uniqueKey, 'materialize key').toEqual( f.expected.materialize.unique_key ?? undefined ) + expect(got.materialize?.scd2 ?? false, 'materialize scd2').toBe( + f.expected.materialize.scd2 ?? false + ) + expect(got.materialize?.track ?? [], 'materialize track').toEqual( + f.expected.materialize.track ?? [] + ) + expect(got.materialize?.closeDeleted ?? false, 'materialize close_deleted').toBe( + f.expected.materialize.close_deleted ?? false + ) + expect(got.materialize?.onSchemaChange ?? 'warn', 'materialize on_schema_change').toBe( + f.expected.materialize.on_schema_change ?? 'warn' + ) } expect(got.dataTests, 'data tests').toEqual(f.expected.data_tests ?? []) expect(got.columnLineage, 'column lineage').toEqual(f.expected.column_lineage ?? []) + + expect(got.macros, 'macros').toBe(f.expected.macros ?? false) + + expect(got.useLibs, 'use_libs').toEqual(f.expected.use_libs ?? []) }) } }) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts index 5f4b063e3f..e9e23525ec 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { mergeColumnLineage, + parseDurationSecs, parsePipelineAnnotations, type ColumnLineage } from './parsePipelineAnnotations' @@ -95,6 +96,33 @@ describe('parsePipelineAnnotations: retry', () => { }) }) +describe('parsePipelineAnnotations: macros + use', () => { + it('parses the bare macros marker', () => { + const out = parsePipelineAnnotations('// macros\nCREATE MACRO m(a) AS a;') + expect(out.macros).toBe(true) + }) + + it('macros marker is strict — trailing prose and variants rejected', () => { + expect(parsePipelineAnnotations('// macros are defined below\n').macros).toBe(false) + expect(parsePipelineAnnotations('// macros_v2\n').macros).toBe(false) + expect(parsePipelineAnnotations('-- macros \nSELECT 1;').macros).toBe(true) + }) + + it('use accumulates in order and dedups', () => { + const out = parsePipelineAnnotations( + '// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\n' + ) + expect(out.useLibs).toEqual(['f/lib/stats', 'f/lib/dates']) + }) + + it('use rejects prose, slashless and multi-token values', () => { + const out = parsePipelineAnnotations( + '// use this script to compute\n// use standalone\n// use f/lib/ok extra\n' + ) + expect(out.useLibs).toEqual([]) + }) +}) + describe('parsePipelineAnnotations: combined', () => { it('parses all keywords together', () => { const code = [ @@ -161,3 +189,40 @@ describe('mergeColumnLineage', () => { expect(mergeColumnLineage([], annotated)).toEqual(annotated) }) }) + +// Mirror of the Rust `parse_duration_secs` tests (windmill-common assets.rs) +// — the freshness chip's staleness verdict depends on identical parsing. +describe('parseDurationSecs', () => { + it('parses suffixed durations', () => { + expect(parseDurationSecs('30s')).toBe(30) + expect(parseDurationSecs('5m')).toBe(300) + expect(parseDurationSecs('2h')).toBe(7200) + expect(parseDurationSecs('1d')).toBe(86400) + }) + + it('bare integer means seconds', () => { + expect(parseDurationSecs('45')).toBe(45) + }) + + it('tolerates surrounding whitespace', () => { + expect(parseDurationSecs(' 5 m ')).toBe(300) + }) + + it('accepts an explicit plus sign (Rust i64 parsing does)', () => { + expect(parseDurationSecs('+5m')).toBe(300) + expect(parseDurationSecs('+45')).toBe(45) + }) + + it('rejects malformed / non-positive input', () => { + expect(parseDurationSecs('')).toBeUndefined() + expect(parseDurationSecs('h')).toBeUndefined() + expect(parseDurationSecs('1.5h')).toBeUndefined() + expect(parseDurationSecs('-5m')).toBeUndefined() + expect(parseDurationSecs('0')).toBeUndefined() + expect(parseDurationSecs('fast')).toBeUndefined() + }) + + it('rejects values beyond i32 seconds (mirrors backend cap)', () => { + expect(parseDurationSecs('999999999d')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index ccbc746b04..47eea81246 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -67,6 +67,28 @@ export type FreshnessSpec = { duration: string } +// Mirrors backend `parse_duration_secs` (windmill-common assets.rs): a bare +// integer means seconds, otherwise `` with an `s`/`m`/`h`/`d` suffix +// (e.g. `30s`, `5m`, `2h`, `1d`). Returns undefined for malformed or +// non-positive input so a typo'd `// freshness` window fails safe (the chip +// stays neutral instead of guessing a staleness verdict). +export function parseDurationSecs(s: string): number | undefined { + const t = s.trim() + if (!t) return undefined + const last = t[t.length - 1] + const mult = + last === 's' ? 1 : last === 'm' ? 60 : last === 'h' ? 3600 : last === 'd' ? 86400 : undefined + const num = (mult !== undefined ? t.slice(0, -1) : t).trim() + // `+?`: Rust's i64 parsing accepts an explicit plus sign (`+5m`), so the + // mirror must too — divergence here would leave the chip neutral for a + // window the deploy path and watchdog honor. + if (mult === undefined && !/^\+?\d+$/.test(t)) return undefined + if (!/^\+?\d+$/.test(num)) return undefined + const secs = Number(num) * (mult ?? 1) + if (!Number.isSafeInteger(secs) || secs <= 0 || secs > 2147483647) return undefined + return secs +} + // `// retry []` — see backend RetrySpec. Delay is kept as the // raw duration string and resolved to seconds at deploy. export type RetrySpec = { @@ -77,7 +99,9 @@ export type RetrySpec = { // `// materialize [manual] [append] [key=
]` — see backend // MaterializeSpec. Managed by default (the runtime generates the write DDL // around a single SELECT); `manual` opts out (the script writes its own DDL, -// track-only). `append` / `key` are managed-mode strategy options. +// track-only). `append` / `key` / `history` / `track` are managed-mode strategy +// options; `key=history` (or the `scd2` alias) selects SCD type-2 history, +// and `deletes=close` opts scd2 into hard-delete-close. export type MaterializeSpec = { targetKind: AssetKind targetPath: string @@ -85,8 +109,28 @@ export type MaterializeSpec = { manual?: boolean // INSERT-only strategy; absent === false append?: boolean - // merge key; absent === replace (or append) + // merge key; absent === replace (or append). Also the SCD2 natural key. uniqueKey?: string + // SCD2 managed history mode (valid_from/valid_to/is_current); absent === false + scd2?: boolean + // SCD2 tracked columns (change ⇒ new version); empty ⇒ all non-key columns + track?: string[] + // SCD2 hard-delete-close (`deletes=close`): close absent keys; absent === false + closeDeleted?: boolean + // `on_schema_change=ignore` opts the produced asset out of downstream + // schema-contract warnings (save-time metadata only). Default `warn`; + // `fail` is deliberately unrecognized in v1 (saves never hard-block). + onSchemaChange?: 'warn' | 'ignore' +} + +// The `_current` SCD2 companion view this managed materialize also +// produces, or `undefined` when it isn't a managed scd2 target. Mirrors Rust +// `MaterializeSpec::scd2_current_target`: managed scd2 creates the base table +// *and* the `_current` view each run; `manual` mode owns its own DDL and creates +// no companion. The graph surfaces register it as a second write of the producer +// so a read of the view links back instead of orphaning. +export function scd2CurrentTargetPath(m: MaterializeSpec): string | undefined { + return m.scd2 && !m.manual ? `${m.targetPath}_current` : undefined } // `// data_test …` — a data-quality assertion run against the @@ -165,6 +209,12 @@ export type PipelineAnnotations = { dataTests: DataTest[] // `// column <- .[, …]` — accumulating column lineage. columnLineage: ColumnLineage[] + // Bare `// macros` (alone on the line, like `// pipeline`) — marks this + // DuckDB script as a workspace macro library. + macros: boolean + // `// use ` — force-inject the named macro library into + // this script's jobs. Accumulating, declaration order, deduped. + useLibs: string[] } // Tokenize a `key=value [key="quoted value"] ...` option string. Bare @@ -207,7 +257,18 @@ function parseKvOpts(s: string): Map { function parseAssetSyntax(s: string): PipelineTriggerAsset | undefined { for (const [prefix, kind] of ASSET_PREFIXES) { if (s.startsWith(prefix)) { - return { kind, path: s.slice(prefix.length) } + let path = s.slice(prefix.length) + // Mirror the Rust `parse_asset_syntax` S3 canonicalization: strip all + // leading slashes so the SDK object form (`s3:///key`, default + // storage) and DuckDB / `// on s3://key` share one asset path, and a + // canonical key never starts with `/` (so ref reconstruction + // round-trips). Without this the live graph preview would show + // disconnected `/key` and `key` nodes. S3-only; leading slashes only, + // so Hive-partition keys are untouched. + if (kind === 's3object') { + path = path.replace(/^\/+/, '') + } + return { kind, path } } } return undefined @@ -223,18 +284,27 @@ function parseAssetSyntaxDefault(s: string): PipelineTriggerAsset | undefined { return parseAssetSyntax(s) } -// Parse a `// materialize [manual] [append] [key=]` right-hand -// side. Optional leading `manual` word opts out of managed mode; the next token -// is the target asset URI (default-syntax shorthands enabled); the remainder -// are strategy options (`append` flag, `key=`). Missing/empty target → -// undefined (dropped). +// Parse a `// materialize [manual] [append] [key=] [history] +// [track=]` right-hand side. Optional leading `manual` word opts out of +// managed mode; a leading `scd2` word is an alias for the `history` flag; the +// next token is the target asset URI (default-syntax shorthands enabled); the +// remainder are strategy options (`append` flag, `key=`, `history` flag, +// `track=`, `deletes=close`, `on_schema_change=ignore`). Missing/empty +// target → undefined (dropped). function parseMaterializeSpec(s: string): MaterializeSpec | undefined { + // One optional leading mode keyword: `manual` (track-only) or `scd2` (an alias + // for the `history` flag below). let manual = false + let scd2Kw = false let rest = s const afterManual = consumeKeyword(s, 'manual') + const afterScd2 = afterManual === undefined ? consumeKeyword(s, 'scd2') : undefined if (afterManual !== undefined) { manual = true rest = afterManual.trimStart() + } else if (afterScd2 !== undefined) { + scd2Kw = true + rest = afterScd2.trimStart() } rest = rest.trim() const m = rest.match(/^(\S+)(?:\s+(.*))?$/) @@ -242,10 +312,38 @@ function parseMaterializeSpec(s: string): MaterializeSpec | undefined { const asset = parseAssetSyntaxDefault(m[1]) if (!asset || asset.path === '') return undefined const optsStr = m[2] ?? '' - const append = optsStr.split(/\s+/).some((t) => t === 'append') - const key = parseKvOpts(optsStr).get('key') + const optTokens = optsStr.split(/\s+/) + const append = optTokens.some((t) => t === 'append') + // SCD type-2 history: primary spelling is the bare `history` flag on a keyed + // merge; the leading `scd2` keyword is a recognized alias. + const scd2 = scd2Kw || optTokens.some((t) => t === 'history') + const opts = parseKvOpts(optsStr) + const key = opts.get('key') const uniqueKey = key && key !== '' ? key : undefined - return { targetKind: asset.kind, targetPath: asset.path, manual, append, uniqueKey } + // `track=` (scd2): comma-separated tracked columns; empty ⇒ all. + // The value is whitespace-terminated (like every `=`-option), so it must have + // no spaces (`track=a,b`, not `track=a, b` — the rest is dropped). + const track = (opts.get('track') ?? '') + .split(',') + .map((c) => c.trim()) + .filter((c) => c !== '') + // `deletes=close` (scd2 only) opts into hard-delete-close; any other value + // (or absence) keeps the soft-delete default. + const closeDeleted = opts.get('deletes') === 'close' + // `on_schema_change=ignore` suppresses downstream contract warnings; any + // other value (or absence) keeps the `warn` default, fail-safe like `deletes=`. + const onSchemaChange = opts.get('on_schema_change') === 'ignore' ? 'ignore' : 'warn' + return { + targetKind: asset.kind, + targetPath: asset.path, + manual, + append, + uniqueKey, + scd2, + track, + closeDeleted, + onSchemaChange + } } // A single bare identifier token (column name). Rejects empty / multi-token @@ -470,7 +568,9 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { triggerAssets: [], nativeTriggers: [], dataTests: [], - columnLineage: [] + columnLineage: [], + macros: false, + useLibs: [] } for (const rawLine of code.split('\n')) { @@ -490,6 +590,26 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { continue } + const afterMacros = consumeKeyword(inner, 'macros') + if (afterMacros !== undefined) { + // Strict like `pipeline`: keyword alone on the line, so prose such + // as `// macros are defined below` never false-positives. + if (afterMacros.trim() === '') out.macros = true + continue + } + + // `// use ` — accumulating. The argument must be a + // single whitespace-free token containing `/` (all script paths do), + // so prose like `// use this script to …` is dropped fail-safe. + const afterUse = consumeKeyword(inner, 'use') + if (afterUse !== undefined) { + const path = afterUse.trim() + if (path && !/\s/.test(path) && path.includes('/') && !out.useLibs.includes(path)) { + out.useLibs.push(path) + } + continue + } + const afterPart = consumeKeyword(inner, 'partitioned') if (afterPart !== undefined) { if (!out.partition) { diff --git a/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts new file mode 100644 index 0000000000..a8a47af259 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from 'vitest' +import { + bucketFor, + bucketFromInputValue, + defaultBucket, + inputValueFromBucket, + isBeforeStart, + isValidStart, + isValidTimeZone, + partitionInputType, + partitionMetadataError, + recentBuckets, + startBucketOf, + usesCalendarPicker +} from './partitionBuckets' +import type { PartitionSpec } from './parsePipelineAnnotations' + +const spec = (kind: PartitionSpec['kind'], extra: Partial = {}): PartitionSpec => + ({ kind, ...extra }) as PartitionSpec + +// A fixed UTC instant: 2026-07-05 14:37Z (a Sunday — ISO week 27 of 2026). +// Explicit UTC so the assertions are independent of the test runner's TZ, and +// they exercise the same UTC default the backend uses when `spec.tz` is absent. +const at = new Date(Date.UTC(2026, 6, 5, 14, 37, 0)) + +describe('bucketFor mirrors backend default_format (UTC)', () => { + it('daily -> %Y-%m-%d', () => { + expect(bucketFor(spec('daily'), at)).toBe('2026-07-05') + }) + it('hourly -> %Y-%m-%dT%H', () => { + expect(bucketFor(spec('hourly'), at)).toBe('2026-07-05T14') + }) + it('monthly -> %Y-%m', () => { + expect(bucketFor(spec('monthly'), at)).toBe('2026-07') + }) + it('weekly -> ISO %G-W%V', () => { + expect(bucketFor(spec('weekly'), at)).toBe('2026-W27') + }) +}) + +describe('bucketFor honours spec.tz', () => { + // 2026-07-05 02:30Z is still 2026-07-04 in America/New_York (UTC-4 in July). + const nearMidnight = new Date(Date.UTC(2026, 6, 5, 2, 30, 0)) + it('shifts the day boundary by the producer tz', () => { + expect(bucketFor(spec('daily'), nearMidnight)).toBe('2026-07-05') // UTC default + expect(bucketFor(spec('daily', { tz: 'America/New_York' }), nearMidnight)).toBe('2026-07-04') + }) + it('shifts the hour bucket by the producer tz', () => { + // 02:30Z -> 22 (previous day) in New York. + expect(bucketFor(spec('hourly', { tz: 'America/New_York' }), nearMidnight)).toBe( + '2026-07-04T22' + ) + }) +}) + +describe('defaultBucket honours the start= anchor', () => { + // `at` is 2026-07-05. + it('seeds the current bucket when at or after start', () => { + expect(defaultBucket(spec('daily'), at)).toBe('2026-07-05') + expect(defaultBucket(spec('daily', { start: '2026-01-01' }), at)).toBe('2026-07-05') + expect(defaultBucket(spec('daily', { start: '2026-07-05' }), at)).toBe('2026-07-05') // == start, not before + }) + it('seeds the start bucket (never a pre-start one) when before start', () => { + expect(defaultBucket(spec('daily', { start: '2026-08-01' }), at)).toBe('2026-08-01') + expect(defaultBucket(spec('monthly', { start: '2026-08-01' }), at)).toBe('2026-08') + expect(defaultBucket(spec('hourly', { start: '2026-08-01' }), at)).toBe('2026-08-01T00') + }) + it('isBeforeStart mirrors the backend date comparison', () => { + expect(isBeforeStart(spec('daily', { start: '2026-08-01' }), at)).toBe(true) + expect(isBeforeStart(spec('daily', { start: '2026-07-05' }), at)).toBe(false) + expect(isBeforeStart(spec('daily'), at)).toBe(false) + }) + it('startBucketOf renders the anchor in the cadence, undefined when unset', () => { + expect(startBucketOf(spec('daily', { start: '2026-08-01' }))).toBe('2026-08-01') + expect(startBucketOf(spec('monthly', { start: '2026-08-15' }))).toBe('2026-08') + expect(startBucketOf(spec('hourly', { start: '2026-08-01' }))).toBe('2026-08-01T00') + expect(startBucketOf(spec('daily'))).toBeUndefined() + }) +}) + +describe('malformed metadata fails safe (parity with backend validation)', () => { + it('isValidTimeZone rejects garbage, accepts real zones and absence', () => { + expect(isValidTimeZone(undefined)).toBe(true) + expect(isValidTimeZone('UTC')).toBe(true) + expect(isValidTimeZone('America/New_York')).toBe(true) + expect(isValidTimeZone('Not/AZone')).toBe(false) + expect(isValidTimeZone('garbage')).toBe(false) + }) + it('isValidStart rejects malformed and JS-normalized dates', () => { + expect(isValidStart(undefined)).toBe(true) + expect(isValidStart('2026-08-01')).toBe(true) + expect(isValidStart('2026-02-31')).toBe(false) // JS would roll to Mar 3 + expect(isValidStart('2026-13-01')).toBe(false) + expect(isValidStart('08/01/2026')).toBe(false) + }) + it('partitionMetadataError reports the first problem, else undefined', () => { + expect(partitionMetadataError(spec('daily'))).toBeUndefined() + expect( + partitionMetadataError(spec('daily', { tz: 'America/New_York', start: '2026-08-01' })) + ).toBeUndefined() + expect(partitionMetadataError(spec('daily', { tz: 'Not/AZone' }))).toContain('timezone') + expect(partitionMetadataError(spec('daily', { start: '2026-02-31' }))).toContain('start date') + }) + it('bucketFor never throws on an invalid tz (falls back to UTC)', () => { + expect(bucketFor(spec('daily', { tz: 'Not/AZone' }), at)).toBe('2026-07-05') + }) + it('an invalid start is treated as no anchor (isBeforeStart/startBucketOf)', () => { + expect(isBeforeStart(spec('daily', { start: '2026-02-31' }), at)).toBe(false) + expect(startBucketOf(spec('daily', { start: '2026-02-31' }))).toBeUndefined() + }) +}) + +describe('partitionInputType', () => { + it('maps each calendar kind to its native input', () => { + expect(partitionInputType(spec('daily'))).toBe('date') + expect(partitionInputType(spec('hourly'))).toBe('datetime-local') + expect(partitionInputType(spec('weekly'))).toBe('week') + expect(partitionInputType(spec('monthly'))).toBe('month') + }) + it('falls back to text for dynamic and custom-format specs', () => { + expect(partitionInputType(spec('dynamic', { key: '$.tenant' }))).toBe('text') + expect(partitionInputType(spec('daily', { format: '%Y/%m/%d' }))).toBe('text') + expect(usesCalendarPicker(spec('dynamic', { key: '$.tenant' }))).toBe(false) + expect(usesCalendarPicker(spec('daily', { format: '%Y/%m/%d' }))).toBe(false) + }) +}) + +describe('native input <-> bucket round-trip', () => { + it('hourly truncates the datetime-local minutes and restores :00', () => { + expect(bucketFromInputValue(spec('hourly'), '2026-07-05T14:37')).toBe('2026-07-05T14') + expect(inputValueFromBucket(spec('hourly'), '2026-07-05T14')).toBe('2026-07-05T14:00') + }) + it('non-hourly kinds pass through unchanged', () => { + expect(bucketFromInputValue(spec('daily'), '2026-07-05')).toBe('2026-07-05') + expect(inputValueFromBucket(spec('weekly'), '2026-W27')).toBe('2026-W27') + }) + it('empty stays empty', () => { + expect(bucketFromInputValue(spec('hourly'), '')).toBe('') + expect(inputValueFromBucket(spec('daily'), '')).toBe('') + }) +}) + +describe('recentBuckets (UTC)', () => { + it('walks back day by day, most recent first', () => { + expect(recentBuckets(spec('daily'), at, 3)).toEqual(['2026-07-05', '2026-07-04', '2026-07-03']) + }) + it('walks back hour by hour', () => { + expect(recentBuckets(spec('hourly'), at, 3)).toEqual([ + '2026-07-05T14', + '2026-07-05T13', + '2026-07-05T12' + ]) + }) + it('walks back month by month across a year boundary without day roll-over', () => { + const jan31 = new Date(Date.UTC(2026, 0, 31, 0, 0, 0)) + expect(recentBuckets(spec('monthly'), jan31, 3)).toEqual(['2026-01', '2025-12', '2025-11']) + }) + it('walks back week by week', () => { + expect(recentBuckets(spec('weekly'), at, 3)).toEqual(['2026-W27', '2026-W26', '2026-W25']) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts new file mode 100644 index 0000000000..401ef82cae --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts @@ -0,0 +1,261 @@ +// Client-side partition-bucket math for the run form's partition picker. +// Produces the same canonical bucket strings the backend renders in +// `windmill-common/src/partition_ee.rs` (`resolve_time_partition` / +// `default_format`): the instant is localized to the spec's timezone +// (`spec.tz`, defaulting to UTC — NOT the browser's zone) and then formatted. +// Getting the zone right matters — a browser in a non-UTC zone would otherwise +// seed a default bucket, and compare against materialized rows, off by a +// day/hour near every boundary and always for an explicit `tz=` spec. +// +// daily %Y-%m-%d -> 2026-07-05 +// hourly %Y-%m-%dT%H -> 2026-07-05T14 +// weekly %G-W%V (ISO) -> 2026-W27 +// monthly %Y-%m -> 2026-07 +// +// Pure module (no Svelte runes) so the mapping is unit-testable. + +import type { PartitionSpec } from './parsePipelineAnnotations' + +export type PartitionInputType = 'date' | 'month' | 'week' | 'datetime-local' | 'text' + +// A spec carrying a custom strftime `format` can't be reproduced by the native +// date pickers (arbitrary strftime), and `dynamic` partitions are a free-form +// key extracted from the payload — both fall back to a plain text input. +export function usesCalendarPicker(spec: PartitionSpec): boolean { + return spec.kind !== 'dynamic' && !spec.format +} + +export function partitionInputType(spec: PartitionSpec): PartitionInputType { + if (!usesCalendarPicker(spec)) return 'text' + switch (spec.kind) { + case 'monthly': + return 'month' + case 'weekly': + return 'week' + case 'hourly': + return 'datetime-local' + default: + return 'date' + } +} + +function pad(n: number): string { + return String(n).padStart(2, '0') +} + +const ZONED_PARTS_OPTS: Intl.DateTimeFormatOptions = { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hourCycle: 'h23' +} + +// Re-express `at` as a Date whose UTC fields equal the wall-clock in `tz`, so +// all downstream field reads / calendar arithmetic can use the UTC getters and +// stay in the producer's zone. `hourCycle: 'h23'` keeps hours 00–23. A +// malformed `tz` (rejected by `Intl`) falls back to UTC rather than throwing — +// the backend validates `tz=` and is the source of truth for the error; the +// picker must never crash the graph view (`partitionMetadataError` gates +// auto-seeding so a bogus bucket isn't silently sent). +function zonedAsUtc(at: Date, tz: string): Date { + let parts: Intl.DateTimeFormatPart[] + try { + parts = new Intl.DateTimeFormat('en-US', { ...ZONED_PARTS_OPTS, timeZone: tz }).formatToParts( + at + ) + } catch { + parts = new Intl.DateTimeFormat('en-US', { + ...ZONED_PARTS_OPTS, + timeZone: 'UTC' + }).formatToParts(at) + } + const g = (t: string) => Number(parts.find((p) => p.type === t)?.value) + return new Date( + Date.UTC(g('year'), g('month') - 1, g('day'), g('hour'), g('minute'), g('second')) + ) +} + +// `tz=` is valid iff `Intl` accepts it (absent === UTC === valid). +export function isValidTimeZone(tz?: string): boolean { + if (!tz) return true + try { + new Intl.DateTimeFormat('en-US', { timeZone: tz }) + return true + } catch { + return false + } +} + +// `start=` is valid iff it's a real `YYYY-MM-DD` calendar date. The round-trip +// check rejects dates JS would silently normalize (e.g. `2026-02-31` → Mar 3), +// which the backend's `NaiveDate::parse_from_str` also rejects. +export function isValidStart(start?: string): boolean { + if (!start) return true + const m = start.match(/^(\d{4})-(\d{2})-(\d{2})$/) + if (!m) return false + const y = Number(m[1]) + const mo = Number(m[2]) + const d = Number(m[3]) + const dt = new Date(Date.UTC(y, mo - 1, d)) + return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d +} + +// A human-readable reason the `// partitioned` metadata is unusable, or +// undefined when it's sound. Mirrors the backend's `tz=` / `start=` validation +// so the picker can refuse to auto-seed a bucket the backend would reject. +export function partitionMetadataError(spec: PartitionSpec): string | undefined { + if (!isValidTimeZone(spec.tz)) return `invalid timezone "${spec.tz}"` + if (!isValidStart(spec.start)) return `invalid start date "${spec.start}" (want YYYY-MM-DD)` + return undefined +} + +// ISO 8601 week-numbering year + week (chrono's %G / %V) of a UTC-substituted +// date. Standard "nearest Thursday" algorithm, all in UTC. +function isoWeekOf(d: Date): { isoYear: number; week: number } { + const date = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())) + const dayNum = (date.getUTCDay() + 6) % 7 // Mon=0 … Sun=6 + date.setUTCDate(date.getUTCDate() - dayNum + 3) // Thursday of this week + const isoYear = date.getUTCFullYear() + const firstThursday = new Date(Date.UTC(isoYear, 0, 4)) + const fdNum = (firstThursday.getUTCDay() + 6) % 7 + firstThursday.setUTCDate(firstThursday.getUTCDate() - fdNum + 3) + const week = 1 + Math.round((date.getTime() - firstThursday.getTime()) / (7 * 86400000)) + return { isoYear, week } +} + +// Render a UTC-substituted date into its canonical bucket for the cadence. +function fmtBucket(kind: PartitionSpec['kind'], d: Date): string { + const y = d.getUTCFullYear() + const m = d.getUTCMonth() + 1 + const day = d.getUTCDate() + const h = d.getUTCHours() + switch (kind) { + case 'hourly': + return `${y}-${pad(m)}-${pad(day)}T${pad(h)}` + case 'monthly': + return `${y}-${pad(m)}` + case 'weekly': { + const { isoYear, week } = isoWeekOf(d) + return `${isoYear}-W${pad(week)}` + } + default: + return `${y}-${pad(m)}-${pad(day)}` + } +} + +export function bucketFor(spec: PartitionSpec, at: Date): string { + return fmtBucket(spec.kind, zonedAsUtc(at, spec.tz ?? 'UTC')) +} + +// The zoned start date (`spec.start`, `YYYY-MM-DD`) as a UTC-substituted Date at +// 00:00, or undefined if unset/malformed. `start` is a plain date in the +// producer's tz — the backend parses it as a NaiveDate and compares by date. +function startDate(spec: PartitionSpec): Date | undefined { + if (!spec.start || !isValidStart(spec.start)) return undefined + const m = spec.start.match(/^(\d{4})-(\d{2})-(\d{2})$/)! + return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]))) +} + +// True when `at` (localized to the producer tz) falls on a date before the +// `start=` anchor — exactly the backend's `local.date_naive() < start_date` +// check (`resolve_time_partition`), which resolves such an instant to no +// partition. +export function isBeforeStart(spec: PartitionSpec, at: Date): boolean { + const start = startDate(spec) + if (!start) return false + const zoned = zonedAsUtc(at, spec.tz ?? 'UTC') + const zonedDate = Date.UTC(zoned.getUTCFullYear(), zoned.getUTCMonth(), zoned.getUTCDate()) + return zonedDate < start.getTime() +} + +// The canonical bucket of the `start=` anchor (its date at 00:00), or undefined +// if unset. Buckets sort lexicographically within a cadence, so callers can +// compare against it to drop pre-start buckets. +export function startBucketOf(spec: PartitionSpec): string | undefined { + const start = startDate(spec) + return start ? fmtBucket(spec.kind, start) : undefined +} + +// The bucket to pre-fill the picker with. Normally the current bucket (matching +// the backend's "absent partition arg -> current bucket" resolution), but when +// the current instant is before the `start=` anchor the backend would resolve +// to NO partition — so default to the first valid bucket (the start) rather +// than a pre-start one the worker would take verbatim and materialize early. +export function defaultBucket(spec: PartitionSpec, at: Date): string { + if (isBeforeStart(spec, at)) { + const start = startDate(spec) + if (start) return fmtBucket(spec.kind, start) + } + return bucketFor(spec, at) +} + +// Native input value -> canonical bucket. Only hourly differs: datetime-local +// carries a minute component the hourly bucket drops. The picked wall-clock is +// taken verbatim as the bucket (the user picks in the producer's frame), so no +// timezone conversion happens here. +export function bucketFromInputValue(spec: PartitionSpec, inputValue: string): string { + if (!inputValue) return '' + if (spec.kind === 'hourly') { + const m = inputValue.match(/^(\d{4}-\d{2}-\d{2}T\d{2})/) + return m ? m[1] : inputValue + } + return inputValue +} + +// Canonical bucket -> native input value. Only hourly differs: datetime-local +// needs a minute component the bucket omits. +export function inputValueFromBucket(spec: PartitionSpec, bucket: string): string { + if (!bucket) return '' + if (spec.kind === 'hourly') { + return /T\d{2}$/.test(bucket) ? `${bucket}:00` : bucket + } + return bucket +} + +// The last `count` buckets ending at (and including) `now`, most-recent first, +// localized to `spec.tz`. Arithmetic walks calendar units in the zoned frame, +// so it's exact across DST (no ±1 drift). Undefined for non-calendar specs +// (the caller guards on `usesCalendarPicker`). +export function recentBuckets(spec: PartitionSpec, now: Date, count: number): string[] { + const base = zonedAsUtc(now, spec.tz ?? 'UTC') + const out: string[] = [] + for (let i = 0; i < count; i++) { + const d = new Date(base) + switch (spec.kind) { + case 'hourly': + d.setUTCHours(d.getUTCHours() - i) + break + case 'weekly': + d.setUTCDate(d.getUTCDate() - 7 * i) + break + case 'monthly': + // Normalize to the 1st first so subtracting months can't roll over + // a short target month (e.g. Mar 31 − 1mo → Mar 3). + d.setUTCDate(1) + d.setUTCMonth(d.getUTCMonth() - i) + break + default: + d.setUTCDate(d.getUTCDate() - i) + } + out.push(fmtBucket(spec.kind, d)) + } + return out +} + +// How many recent buckets the "missing partitions" hint scans, per kind — a +// window that reads as "recent" for each cadence without flooding the hint. +export function recentWindow(kind: PartitionSpec['kind']): number { + switch (kind) { + case 'hourly': + return 24 + case 'weekly': + return 8 + case 'monthly': + return 6 + default: + return 14 + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts new file mode 100644 index 0000000000..a1a82870c8 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts @@ -0,0 +1,143 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { JobService, ScriptService } from '$lib/gen' +import { createPipelineAiHelpers, type PipelineDraft } from './pipelineAiHelpers' +import type { AssetGraphResponse } from './types' + +// Build a helper handle over an in-memory drafts Map, mirroring how the editor +// wires it. `getFolder` returns the bare folder name (as the route/session do). +function makeHandle( + initial: Array<[string, PipelineDraft]> = [], + runnables: Array<{ path: string }> = [] +) { + let drafts = new Map(initial) + let forgotten: string[] = [] + const handle = createPipelineAiHelpers({ + getFolder: () => 'x', + getWorkspace: () => 'w', + getResolvedGraph: () => + ({ assets: [], runnables, edges: [], triggers: [] }) as unknown as AssetGraphResponse, + getDrafts: () => drafts, + setDrafts: (next) => (drafts = next), + newDraftLocalId: () => 'id', + onForgetPath: (p) => forgotten.push(p) + }) + return { handle, drafts: () => drafts, forgotten: () => forgotten } +} + +afterEach(() => vi.restoreAllMocks()) + +const draft = (over: Partial = {}): PipelineDraft => + ({ localId: 'l', script: { content: '' } as any, ...over }) as PipelineDraft + +describe('pipeline AI direct-draft helpers', () => { + it('removeProposedNode discards the unsaved draft at a path', async () => { + const { handle, drafts, forgotten } = makeHandle([ + ['f/x/a', draft()], + ['f/x/b', draft()] + ]) + await handle.removeProposedNode('f/x/a') + expect(drafts().has('f/x/a')).toBe(false) + expect(drafts().has('f/x/b')).toBe(true) + expect(forgotten()).toContain('f/x/a') + }) + + it('removeProposedNode throws when there is no draft to discard', async () => { + const { handle } = makeHandle() + await expect(handle.removeProposedNode('f/x/missing')).rejects.toThrow() + }) + + it('getPipelineContext does not expose any pending/approval state', () => { + const { handle } = makeHandle([['f/x/a', draft()]]) + const ctx = handle.getPipelineContext() + expect(ctx).not.toHaveProperty('pendingProposals') + expect(handle).not.toHaveProperty('acceptAll') + expect(handle).not.toHaveProperty('rejectAll') + }) + + it('testNode on a deployed node never dispatches downstream subscribers', async () => { + // No draft at the path → runs the deployed version, which must carry + // `_wmill_skip_asset_dispatch` so a single-node test can't fire downstream. + const spy = vi.spyOn(JobService, 'runScriptByPath').mockResolvedValue('job-1' as any) + const { handle } = makeHandle() + await handle.testNode('f/x/deployed', { foo: 1 }) + expect(spy).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ _wmill_skip_asset_dispatch: true, foo: 1 }) + }) + ) + }) + + it('proposeNode rejects a path outside the open folder', async () => { + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ path: 'f/other/n', language: 'duckdb' as any, content: '' }) + ).rejects.toThrow(/open folder/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects content missing the pipeline annotation', async () => { + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ path: 'f/x/new', language: 'duckdb' as any, content: 'SELECT 1' }) + ).rejects.toThrow(/pipeline annotation/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects a path colliding with an existing draft', async () => { + const { handle } = makeHandle([['f/x/a', draft()]]) + await expect( + handle.proposeNode({ path: 'f/x/a', language: 'duckdb' as any, content: '-- pipeline' }) + ).rejects.toThrow(/already exists/) + }) + + it('proposeNode rejects a path colliding with an existing deployed node', async () => { + const { handle, drafts } = makeHandle([], [{ path: 'f/x/dep' }]) + await expect( + handle.proposeNode({ path: 'f/x/dep', language: 'duckdb' as any, content: '-- pipeline' }) + ).rejects.toThrow(/already exists/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects a path that is an already-deployed script when the graph has not hydrated', async () => { + // Empty graph (session preview can race open_preview), but a deployed script + // exists at the path — the backend probe must still catch it. + const spy = vi.spyOn(ScriptService, 'getScriptByPath').mockResolvedValue({} as any) + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ + path: 'f/x/deployed', + language: 'duckdb' as any, + content: '-- pipeline' + }) + ).rejects.toThrow(/already exists/) + expect(spy).toHaveBeenCalled() + expect(drafts().size).toBe(0) + }) + + it('editNode rejects a path outside the open folder', async () => { + const { handle, drafts } = makeHandle() + await expect(handle.editNode('f/other/foo', '-- pipeline')).rejects.toThrow(/open folder/) + expect(drafts().size).toBe(0) + }) + + it('editNode preserves the deployed script hash/metadata and replaces only content', async () => { + const deployed = { + hash: 'abc123', + path: 'f/x/node', + summary: 'My node', + description: 'desc', + tag: 'custom', + language: 'duckdb', + content: '-- pipeline\nSELECT 1' + } + vi.spyOn(ScriptService, 'getScriptByPath').mockResolvedValue(deployed as any) + const { handle, drafts } = makeHandle() + await handle.editNode('f/x/node', '-- pipeline\nSELECT 2') + const d = drafts().get('f/x/node') + expect(d?.script.hash).toBe('abc123') + expect(d?.script.summary).toBe('My node') + expect(d?.script.description).toBe('desc') + expect(d?.script.tag).toBe('custom') + expect(d?.script.content).toBe('-- pipeline\nSELECT 2') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts new file mode 100644 index 0000000000..959992543f --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts @@ -0,0 +1,319 @@ +import { JobService, ScriptService, type AssetKind, type Script, type ScriptLang } from '$lib/gen' +import { emptySchema, sendUserToast } from '$lib/utils' +import { inferAssets } from '$lib/infer' +import { extractWrites, type AssetWithAltAccessType } from '$lib/components/assets/lib' +import { assetUri, autoOutputAsset, type PipelineOutputKind } from './pipelineTemplates' +import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import type { AssetGraphResponse } from './types' +import type { + PipelineAIChatHelpers, + PipelineContext, + PipelineNodeSummary +} from '$lib/components/copilot/chat/pipeline/core' + +// ============================================================================ +// Shared data-pipeline AI helper layer. +// +// Both the full-page editor (/pipeline/[folder]) and the in-session preview +// (PipelineEditorView) drive the AI chat's pipeline tools through this factory, +// so the build/edit logic lives in exactly one place. Each caller injects +// accessors for its own draft Map and graph; this module owns the AI behaviour +// (build/edit/discard/test). AI edits apply directly as unsaved drafts — there +// is no separate approve/reject step. +// ============================================================================ + +/** + * An unsaved pipeline node draft. `localId` is a stable per-draft id preserved + * across renames (the page uses it to dedupe concurrent deploys). AI-built nodes + * and manually-created drafts are the same thing — an unsaved node on the canvas. + */ +export type PipelineDraft = { + localId: string + script: Script + outputAssets?: Array<{ kind: AssetKind; path: string }> +} + +export type PipelineAiHelperDeps = { + getFolder: () => string + getWorkspace: () => string | undefined + /** The draft-overlaid graph (resolveGraph output) the context summary reads. */ + getResolvedGraph: () => AssetGraphResponse + getDrafts: () => Map + setDrafts: (next: Map) => void + /** Stable id for a freshly-created draft (route page tracks deploys by it). */ + newDraftLocalId: () => string + /** Focus/select the node after it is staged (pan + open in the pane). */ + onProposeNode?: (path: string) => void + /** Throw (or switch to edit mode) when the surface can't accept AI edits. */ + ensureEditable?: () => void + /** Surface the draft overlay if it is hidden (the page's "show drafts" view). */ + onShowDrafts?: () => void + /** Forget per-path state when a draft is discarded. */ + onForgetPath?: (path: string) => void + /** Notify the caller a test run started so it can light up its run UI. */ + onRunStarted?: (jobId: string, path: string) => void +} + +export function makePipelineScript( + language: ScriptLang, + scriptPath: string, + content: string, + createdAt: string +): Script { + // Cast through unknown: a local draft only needs path/language/content/schema; + // the many readonly deployment fields on Script don't matter until createScript. + return { + hash: '', + path: scriptPath, + summary: '', + description: '', + content, + schema: emptySchema(), + is_template: false, + extra_perms: {}, + language, + kind: 'script', + created_by: '', + created_at: createdAt, + archived: false, + deleted: false, + starred: false + } as unknown as Script +} + +async function inferOutputAssets( + language: ScriptLang, + content: string +): Promise> { + try { + const inferred = await inferAssets(language, content) + if (inferred?.status === 'error') return [] + return extractWrites((inferred?.assets ?? []) as AssetWithAltAccessType[]) + } catch { + return [] + } +} + +export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIChatHelpers { + // A staged draft is always persisted into the OPEN folder's data_pipeline + // bundle, so a path outside the folder would silently land an unrelated script + // there. Both build and edit must stay scoped to the folder. + function assertInFolder(path: string) { + const folder = deps.getFolder() + if (folder && !path.startsWith(`f/${folder}/`)) { + throw new Error( + `Pipeline nodes must be in the open folder — use a path under 'f/${folder}/' (got '${path}').` + ) + } + } + + // A pipeline node IS its `// pipeline` annotation (it's what makes the deployed + // script a pipeline member). Reject content that lacks it so a staged draft + // isn't a non-member script the model can't see is broken until deploy. + function assertPipelineAnnotation(content: string) { + if (!parsePipelineAnnotations(content).inPipeline) { + throw new Error( + `Pipeline node content must declare the pipeline annotation on its own comment line ` + + `(\`// pipeline\`, or \`-- pipeline\` for SQL / \`# pipeline\` for Python).` + ) + } + } + + function buildContext(): PipelineContext { + const graph = deps.getResolvedGraph() + const drafts = deps.getDrafts() + const nodes: PipelineNodeSummary[] = graph.runnables + .filter((r) => r.usage_kind === 'script') + .map((r) => { + const draft = drafts.get(r.path) + const writes = graph.edges + .filter( + (e) => + e.runnable_kind === 'script' && + e.runnable_path === r.path && + (e.access_type === 'w' || e.access_type === 'rw') + ) + .map((e) => assetUri({ kind: e.asset_kind, path: e.asset_path })) + const reads = graph.edges + .filter( + (e) => + e.runnable_kind === 'script' && + e.runnable_path === r.path && + (e.access_type === 'r' || e.access_type === 'rw') + ) + .map((e) => assetUri({ kind: e.asset_kind, path: e.asset_path })) + const triggers = graph.triggers + .filter((t) => t.runnable_kind === 'script' && t.runnable_path === r.path) + .map((t) => + t.trigger_kind === 'asset' + ? assetUri({ kind: t.asset_kind, path: t.asset_path }) + : t.trigger_kind + ) + return { + path: r.path, + language: draft?.script.language, + unsaved: r.unsaved ?? false, + summary: draft?.script.summary || undefined, + writes: [...new Set(writes)], + reads: [...new Set(reads)], + triggers: [...new Set(triggers)] + } + }) + return { + folder: deps.getFolder(), + mode: 'edit', + nodes, + assets: graph.assets.map((a) => assetUri({ kind: a.kind, path: a.path })) + } + } + + const helpers: PipelineAIChatHelpers = { + getPipelineContext: buildContext, + getNodeBody: async (path) => { + const draft = deps.getDrafts().get(path) + if (draft) return { language: draft.script.language, content: draft.script.content } + const workspace = deps.getWorkspace() + if (!workspace) return undefined + try { + const deployed = await ScriptService.getScriptByPath({ workspace, path }) + return { language: deployed.language, content: deployed.content } + } catch { + return undefined + } + }, + proposeNode: async ({ path, language, content, outputKind }) => { + deps.ensureEditable?.() + // build_pipeline_node creates a NEW node in the OPEN folder. Reject a path + // outside the folder (it would silently stage into this folder's bundle) + // and a path that collides with an existing node (the model should use + // edit_pipeline_node instead of shadowing a deployed node as a draft). + assertInFolder(path) + assertPipelineAnnotation(content) + const drafts = deps.getDrafts() + if (drafts.has(path)) { + throw new Error( + `A draft already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + if (deps.getResolvedGraph().runnables.some((r) => r.path === path)) { + throw new Error( + `A pipeline node already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + // Authoritative new-node check: the resolved graph may not have hydrated yet + // (the session preview can race open_preview), and it only lists pipeline + // runnables — so probe the backend. ANY deployed script at this path means + // "build new" would shadow it on deploy; the model should edit instead. + const workspace = deps.getWorkspace() + if (workspace) { + let deployedExists = false + try { + await ScriptService.getScriptByPath({ workspace, path }) + deployedExists = true + } catch { + // 404 → no deployed script at this path, safe to create a new node. + } + if (deployedExists) { + throw new Error( + `A script already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + } + const inferred = await inferOutputAssets(language, content) + // Fall back to a seeded output (from the declared output_kind) when the + // body doesn't yet write anything inferable. + const seeded = + inferred[0] ?? + (outputKind + ? autoOutputAsset(outputKind as PipelineOutputKind, deps.getFolder(), language) + : undefined) + const next = new Map(drafts) + next.set(path, { + localId: deps.newDraftLocalId(), + script: makePipelineScript(language, path, content, new Date().toISOString()), + outputAssets: inferred.length > 0 ? inferred : seeded ? [seeded] : undefined + }) + deps.setDrafts(next) + deps.onShowDrafts?.() + deps.onProposeNode?.(path) + return { path } + }, + editNode: async (path, content) => { + deps.ensureEditable?.() + assertInFolder(path) + assertPipelineAnnotation(content) + const drafts = deps.getDrafts() + const existing = drafts.get(path) + // Base the edit on the existing draft's / deployed script object and replace + // ONLY the content — preserving hash, summary, description, tag, schema, and + // settings. Rebuilding a fresh script would wipe that metadata: deploying + // from the pane (auto_parent) would update the script while clearing it, and + // the route "Save all" path (no parent_hash) could hit the path-conflict + // branch on the occupied path. + let baseScript: Script + if (existing) { + baseScript = existing.script + } else { + const workspace = deps.getWorkspace() + if (!workspace) throw new Error('No workspace is selected.') + baseScript = await ScriptService.getScriptByPath({ workspace, path }) + } + const inferred = await inferOutputAssets(baseScript.language, content) + const next = new Map(drafts) + next.set(path, { + localId: existing?.localId ?? deps.newDraftLocalId(), + script: { ...baseScript, content }, + outputAssets: inferred.length > 0 ? inferred : existing?.outputAssets + }) + deps.setDrafts(next) + deps.onShowDrafts?.() + deps.onProposeNode?.(path) + }, + removeProposedNode: async (path) => { + if (!deps.getDrafts().has(path)) { + throw new Error(`No unsaved draft at '${path}' to discard.`) + } + const next = new Map(deps.getDrafts()) + next.delete(path) + deps.setDrafts(next) + deps.onForgetPath?.(path) + }, + testNode: async (path, args) => { + const workspace = deps.getWorkspace() + if (!workspace) return undefined + const draft = deps.getDrafts().get(path) + try { + let jobId: string + if (draft) { + // Un-deployed/edited body: preview-run the draft content so it can be + // tested before deploying. + jobId = await JobService.runScriptPreview({ + workspace, + requestBody: { + path, + content: draft.script.content, + language: draft.script.language, + args: args ?? {} + } + }) + } else { + // test_pipeline_node previews ONE node — never fan out to downstream + // deployed subscribers via the backend asset dispatcher (which would + // run side-effecting deployed scripts the user didn't ask for). + jobId = await JobService.runScriptByPath({ + workspace, + path, + requestBody: { ...(args ?? {}), _wmill_skip_asset_dispatch: true } + }) + } + deps.onRunStarted?.(jobId, path) + return jobId + } catch (e: any) { + sendUserToast(`Run failed: ${e?.body ?? e?.message ?? e}`, true) + return undefined + } + } + } + + return helpers +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts new file mode 100644 index 0000000000..bdecadff91 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts @@ -0,0 +1,184 @@ +import type { AssetKind, Script } from '$lib/gen' +import type { AssetWithAltAccessType } from '$lib/components/assets/lib' +import type { AssetGraphSelection } from './types' +import { + parsePipelineAnnotations, + type ColumnLineage, + type PipelineAnnotations +} from './parsePipelineAnnotations' +import type { PipelineDraft } from './pipelineAiHelpers' + +// ============================================================================ +// Externalized pipeline-editor state — the data-pipeline analogue of the flow +// editor's `flowStore` / `flowStateStore`. It owns the in-flight draft Map, the +// live editor overlays, and the current selection: the substrate the route page +// editor and the in-session preview both render through (via the shared +// ). Persistence, graph resolution, run dispatch, and deploy +// stay with the consumer; this is a plain reactive bag so a consumer can +// read/mutate it without prop plumbing. +// ============================================================================ + +const EMPTY_ANNOTATIONS: PipelineAnnotations = parsePipelineAnnotations('') + +type LiveAnnotations = { scriptPath: string | undefined; annotations: PipelineAnnotations } +type LiveBodyAssets = { + scriptPath: string | undefined + assets: AssetWithAltAccessType[] + columnLineage?: ColumnLineage[] +} +type LiveContent = { scriptPath: string | undefined; content: string } + +export class PipelineEditorState { + /** In-flight drafts keyed by script path (manual + AI-staged). */ + drafts = $state>(new Map()) + /** Draft open in the details pane (mutually exclusive with `selection`). */ + activeDraftPath = $state(undefined) + /** The persisted node/asset selected on the canvas. */ + selection = $state(undefined) + + /** Live-parsed annotations of the open script (refreshed per keystroke). */ + liveAnnotations = $state({ + scriptPath: undefined, + annotations: EMPTY_ANNOTATIONS + }) + /** Live-inferred body read/write assets of the open script. */ + liveBodyAssets = $state({ scriptPath: undefined, assets: [] }) + /** The open draft's live editor buffer. */ + liveContent = $state({ scriptPath: undefined, content: '' }) + + /** Set true once a draft bundle was restored from the DB on load — drives the + * route toolbar's one-shot "Loaded from draft" hint. Written by the editor's + * autosave hydrate when persistence is enabled. */ + loadedFromDbDraft = $state(false) + + /** Folder this state is scoped to. Used by the in-session preview (where one + * instance is reused across editor hide/show) to detect a retarget to a + * different folder and reset, so stale drafts don't bleed across folders. */ + folder = $state(undefined) + + /** True once the DB draft bundle for the current folder has been hydrated + * into this instance. Gated per-instance (not per component mount) so the + * in-session preview hydrates ONCE when its runtime is fresh and then keeps + * the in-memory drafts across editor hide/show — re-reading the DB on every + * remount would race a not-yet-flushed autosave and drop a just-staged draft. + * Reset to false on a folder retarget so the new folder re-hydrates. */ + hydratedFromDb = $state(false) + + /** Clear all in-flight state. Used when the session preview retargets a + * different pipeline folder (a same-folder remount keeps the drafts). */ + reset = () => { + this.drafts = new Map() + this.activeDraftPath = undefined + this.selection = undefined + this.clearLiveOverlays() + this.loadedFromDbDraft = false + // Force a re-hydrate from the DB draft of the newly-targeted folder. + this.hydratedFromDb = false + } + + #nextDraftLocalId = 0 + // Arrow fields so `pe.method` can be passed straight as a callback (the + // details pane takes onDraftPersist / onAnnotationsChange / … by reference). + newDraftLocalId = (): string => { + this.#nextDraftLocalId += 1 + return `pe-${this.#nextDraftLocalId}` + } + + handleAnnotationsChange = (scriptPath: string | undefined, annotations: PipelineAnnotations) => { + this.liveAnnotations = { scriptPath, annotations } + } + handleAssetsChange = ( + scriptPath: string | undefined, + assets: AssetWithAltAccessType[], + columnLineage?: ColumnLineage[] + ) => { + this.liveBodyAssets = { scriptPath, assets, columnLineage } + } + handleContentChange = (scriptPath: string | undefined, content: string) => { + this.liveContent = { scriptPath, content } + } + + clearLiveOverlays = () => { + this.liveAnnotations = { scriptPath: undefined, annotations: EMPTY_ANNOTATIONS } + this.liveBodyAssets = { scriptPath: undefined, assets: [] } + this.liveContent = { scriptPath: undefined, content: '' } + } + + /** Drop per-path editor state when a path goes away. Does NOT touch + * consumer-owned per-path state (e.g. the route page's save errors — the + * route layers that on in its own wrapper). */ + forgetPath = (path: string) => { + if (this.activeDraftPath === path) this.activeDraftPath = undefined + if (this.selection?.kind === 'runnable' && this.selection.path === path) + this.selection = undefined + if (this.liveAnnotations.scriptPath === path) + this.liveAnnotations = { scriptPath: undefined, annotations: EMPTY_ANNOTATIONS } + if (this.liveBodyAssets.scriptPath === path) + this.liveBodyAssets = { scriptPath: undefined, assets: [] } + if (this.liveContent.scriptPath === path) + this.liveContent = { scriptPath: undefined, content: '' } + } + + discardDraft = (path: string) => { + if (!this.drafts.has(path)) return + const next = new Map(this.drafts) + next.delete(path) + this.drafts = next + this.forgetPath(path) + } + + /** Commit body edits + inferred outputs back into the drafts Map on pane + * teardown (deferred a microtask so a same-batch discard doesn't resurrect the + * entry). Verbatim port of the route page's `handleDraftPersist`. */ + handleDraftPersist = ( + p: string, + snapshot: { content: string; writes: { kind: AssetKind; path: string }[]; script?: Script } + ) => { + queueMicrotask(() => { + const d = this.drafts.get(p) + if (!d) { + if (!snapshot.script) return + const next = new Map(this.drafts) + next.set(p, { + localId: this.newDraftLocalId(), + script: snapshot.script, + outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined + }) + this.drafts = next + return + } + // `?? 0` is load-bearing: an undefined `outputAssets` (a no-output draft) + // vs an empty inferred `writes` both mean "no writes". Without the + // coalesce, `undefined === 0` is false, so this never short-circuits — + // every persist re-writes the drafts Map with an equivalent object, + // re-triggering the pane's emit → graph re-derive → persist, an infinite + // microtask loop (hangs the tab without an effect-depth throw). + const writesEqual = + (d.outputAssets?.length ?? 0) === snapshot.writes.length && + (d.outputAssets ?? []).every( + (a, i) => a.kind === snapshot.writes[i]?.kind && a.path === snapshot.writes[i]?.path + ) + if (d.script.content === snapshot.content && writesEqual) return + const next = new Map(this.drafts) + next.set(p, { + ...d, + script: { ...d.script, content: snapshot.content }, + outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined + }) + this.drafts = next + }) + } + + /** The draft open in the pane, if any. */ + get activeDraft(): PipelineDraft | undefined { + return this.activeDraftPath ? this.drafts.get(this.activeDraftPath) : undefined + } + + /** Whichever script is open — the active draft, or a selected persisted script. */ + get openScriptPath(): string | undefined { + if (this.activeDraftPath) return this.activeDraftPath + if (this.selection?.kind === 'runnable' && this.selection.runnable_kind === 'script') + return this.selection.path + return undefined + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts new file mode 100644 index 0000000000..0fae3f836d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect } from 'vitest' +import { PipelineEditorState } from './pipelineEditorState.svelte' +import type { PipelineDraft } from './pipelineAiHelpers' +import type { AssetKind } from '$lib/gen' + +// handleDraftPersist defers its commit a microtask (so a same-batch discard can +// win); flush that microtask before asserting. +const flushMicrotasks = () => new Promise((resolve) => queueMicrotask(() => resolve())) + +function draft(content: string, outputAssets?: { kind: AssetKind; path: string }[]): PipelineDraft { + return { + localId: 'pe-1', + script: { path: 'f/x/n', language: 'duckdb', content } as PipelineDraft['script'], + outputAssets + } +} + +describe('PipelineEditorState.handleDraftPersist', () => { + // Regression: a no-output draft has `outputAssets: undefined`; the details pane + // infers an empty `writes: []`. Both mean "no writes", so persisting unchanged + // content+writes must be a no-op. The earlier `undefined === 0` length check made + // it false, so every persist re-wrote the drafts Map with an equivalent object — + // re-triggering the pane's emit → graph re-derive → persist, an infinite + // microtask loop that froze the tab. The Map reference must stay identical. + it('is idempotent for a no-output draft (undefined outputAssets vs empty inferred writes)', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { content: 'SELECT 1', writes: [] }) + await flushMicrotasks() + expect(pe.drafts).toBe(before) + }) + + it('re-writes the drafts Map when the content actually changes', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { content: 'SELECT 2', writes: [] }) + await flushMicrotasks() + expect(pe.drafts).not.toBe(before) + expect(pe.drafts.get('f/x/n')?.script.content).toBe('SELECT 2') + }) + + it('re-writes the drafts Map when the inferred writes actually change', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { + content: 'SELECT 1', + writes: [{ kind: 'resource' as AssetKind, path: 'f/x/out' }] + }) + await flushMicrotasks() + expect(pe.drafts).not.toBe(before) + expect(pe.drafts.get('f/x/n')?.outputAssets).toEqual([{ kind: 'resource', path: 'f/x/out' }]) + }) + + it('stays idempotent when outputAssets and inferred writes match (non-empty)', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([ + ['f/x/n', draft('SELECT 1', [{ kind: 'resource' as AssetKind, path: 'f/x/out' }])] + ]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { + content: 'SELECT 1', + writes: [{ kind: 'resource' as AssetKind, path: 'f/x/out' }] + }) + await flushMicrotasks() + expect(pe.drafts).toBe(before) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts index 50ad3e1d6e..d654085179 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts @@ -80,7 +80,15 @@ export function usePipelineHistory( kind: j.job_kind.startsWith('flow') ? 'flow' : 'script', status: j.success ? 'success' : 'failure', source: j.schedule_path ? 'schedule' : 'run', - at: j.started_at ?? j.created_at + at: j.started_at ?? j.created_at, + // Same completion-time derivation as the live poll — + // the freshness chip compares against completion, and + // `at` (start) would read a long run as older than its + // output actually is. + completedAt: + j.started_at != undefined + ? new Date(new Date(j.started_at).getTime() + j.duration_ms).toISOString() + : undefined }) } sawFullPage = rows.length === PER_PAGE diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts new file mode 100644 index 0000000000..2acdf2abfb --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest' +import { + autoOutputAsset, + compatibleOutputKinds, + generatePipelineDraft, + PIPELINE_OUTPUT_KINDS +} from './pipelineTemplates' + +// The `data_test` output kind scaffolds a *custom* (singular) data test: a +// standalone DuckDB script referenced from a materialize script's +// `-- data_test ` line. It must be a single SELECT that reads the +// freshly-materialized target through the internal `_wm_target` schema — the +// two rules the backend's self-teaching errors enforce. +describe('data_test scaffold', () => { + it('is a DuckDB-only output kind exposed in the picker', () => { + expect(compatibleOutputKinds('duckdb')).toContain('data_test') + expect(compatibleOutputKinds('python3')).not.toContain('data_test') + expect(PIPELINE_OUTPUT_KINDS.map((k) => k.id)).toContain('data_test') + }) + + it('produces no output asset (it asserts against an existing target)', () => { + expect(autoOutputAsset('data_test', 'folder', 'duckdb')).toBeUndefined() + }) + + it('scaffolds a single SELECT against `_wm_target.
`', () => { + const src = generatePipelineDraft({ + language: 'duckdb', + outputKind: 'data_test', + triggers: [] + }) + // starter body is a single SELECT against the internal target alias. + expect(src).toContain('SELECT * FROM _wm_target.your_table WHERE your_condition;') + // exactly one SQL statement (single SELECT) — count statement lines, not + // the word "SELECT" that also appears in the guidance comment. + const stmtLines = src.split('\n').filter((l) => /^\s*SELECT\b/i.test(l)) + expect(stmtLines).toHaveLength(1) + // no `-- materialize ` output annotation — a data test declares no + // asset (the word still appears in the guidance comment, which is fine). + expect(src).not.toMatch(/^--\s*materialize\s/m) + // teaches how to wire it up + the offending-rows convention. + expect(src).toContain('-- data_test ') + expect(src).toContain('offending rows') + }) + + it('seeds the table name from an upstream ducklake asset when present', () => { + const src = generatePipelineDraft({ + language: 'duckdb', + outputKind: 'data_test', + input: { kind: 'ducklake', path: 'analytics/orders' }, + triggers: [] + }) + expect(src).toContain('SELECT * FROM _wm_target.orders WHERE your_condition;') + // no ATTACH of the input — the runtime attaches the target as `_wm_target`. + expect(src).not.toContain('ATTACH') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts new file mode 100644 index 0000000000..fabbe1dca6 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' +import type { ScriptLang } from '$lib/gen' +import { + autoOutputAsset, + generatePipelineDraft, + type PipelineOutputKind +} from './pipelineTemplates' + +// The seeded draft asset (`autoOutputAsset`, stored as `outputAssets` and used +// by resolveGraph for inactive-draft node identity) must match the asset +// identity the deploy-time / wasm parser infers from the generated body. The +// parser canonicalizes any S3 URI by stripping the `s3://` prefix and all +// leading slashes (see backend `parse_asset_syntax`); if the seed carried a +// leading slash while the body wrote `s3:///key`, the preview would render a +// duplicate `/key` node and a phantom post-deploy drift. This pins the two in +// lockstep so that class of drift can't regress. + +// Mirror of the parser's S3 canonicalization for a raw `s3://…` URI. +function canonicalS3Key(uri: string): string { + const rest = uri.replace(/^s3:\/\//, '') + return rest.replace(/^\/+/, '') +} + +const S3_KINDS: PipelineOutputKind[] = ['s3_parquet', 's3_object'] +const LANGS: ScriptLang[] = ['bun', 'python3', 'duckdb'] + +describe('pipelineTemplates S3 seed/body parity', () => { + for (const language of LANGS) { + for (const outputKind of S3_KINDS) { + it(`${language} ${outputKind}: seeded asset path matches the body's S3 write URI`, () => { + const output = autoOutputAsset(outputKind, 'demo', language) + expect(output).toBeDefined() + const asset = output! + + // The seed must be a canonical slashless key so it matches the + // identity the parser infers from the generated body. + expect(asset.kind).toBe('s3object') + expect(asset.path.startsWith('/')).toBe(false) + + const body = generatePipelineDraft({ + language, + outputKind, + output: asset, + triggers: [] + }) + + // Every S3 URI the generated body emits must canonicalize back to + // the seeded asset path — the write target especially. + const uris = body.match(/s3:\/\/[^'"`)\s]+/g) ?? [] + expect(uris.length).toBeGreaterThan(0) + for (const uri of uris) { + // Runtime form must stay triple-slash (default storage); a bare + // `s3://key` would target a named storage `key` at run time. + expect(uri.startsWith('s3:///')).toBe(true) + expect(canonicalS3Key(uri)).toBe(asset.path) + } + }) + } + } +}) + +// `{partition}` substitutes to the partition IDENTITY string (e.g. `2026-07-05T23`, +// `2026-W27`, `2026-07`), which is NOT a valid DuckDB TIMESTAMP literal for any +// sub-day / non-daily grain — so a naive `WHERE ts = TIMESTAMP {partition}` +// raises a Conversion Error for hourly/weekly/monthly. The runtime injects a +// `wm_partition(ts)` macro (format from the same source that stamps the +// identity), so the scaffold teaches the one grain-agnostic filter line. +describe('pipelineTemplates materialize partition filter', () => { + const materializeBody = () => + generatePipelineDraft({ + language: 'duckdb', + outputKind: 'materialize', + output: autoOutputAsset('materialize', 'demo', 'duckdb'), + input: { kind: 'ducklake', path: 'main/orders' }, + triggers: [] + }) + + it('teaches the grain-agnostic wm_partition filter', () => { + const body = materializeBody() + expect(body).toContain(`WHERE wm_partition() = {partition}`) + // dynamic grain has no timestamp bucket — steer to the raw key. + expect(body).toContain(`WHERE = {partition}`) + }) + + it('never scaffolds the naive `TIMESTAMP {partition}` cast, nor a raw strftime format', () => { + const body = materializeBody() + // The footgun cast must never appear (comment or SQL) now that the macro + // hides the format entirely. + expect(body).not.toMatch(/TIMESTAMP\s*\{partition\}/) + // No hand-written strftime format to drift from the resolver. + expect(body).not.toContain('strftime') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index cbb5476516..51af13442d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -17,8 +17,10 @@ export type PipelineOutputKind = | 'datatable' | 'ducklake' | 'materialize' + | 'data_test' | 's3_parquet' | 's3_object' + | 'macros' export type PipelineOutputKindMeta = { id: PipelineOutputKind @@ -36,6 +38,11 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [ label: 'Materialized table', description: 'Managed DuckLake table — idempotent, versioned, tracked' }, + { + id: 'data_test', + label: 'Data test', + description: 'Custom assertion — a single SELECT returning offending rows (empty = pass)' + }, { id: 'datatable', label: 'Data table', @@ -56,6 +63,11 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [ label: 'S3 Object', description: 'Generic file (JSON/CSV/binary)' }, + { + id: 'macros', + label: 'Macro library', + description: 'Reusable DuckDB macros, callable from every script in the workspace' + }, { id: 'none', label: 'No output', @@ -77,7 +89,16 @@ const LANG_COMPATIBILITY: Record = { // single SELECT. The Python/TS `wmll.ducklake` helper currently takes a SQL // SELECT (not in-memory rows), so a polyglot managed materialize is a // separate follow-up — those langs keep the `ducklake` raw-write kind. - duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'none'], + duckdb: [ + 'materialize', + 'data_test', + 'datatable', + 'ducklake', + 's3_parquet', + 's3_object', + 'macros', + 'none' + ], postgresql: ['datatable', 'none'], mysql: ['none'], mssql: ['none'], @@ -154,16 +175,16 @@ export function autoOutputAsset( case 'ducklake': case 'materialize': return { kind: 'ducklake', path: `main/${adj}_${pick(TABLE_NOUNS)}_${slug}` } - // s3 paths carry the canonical leading slash of a default-storage - // object (`s3:///` parses to path `/`). The deploy-time - // parser stores writes in that form — a slashless seeded path would - // never match it, and the post-deploy drift check would report the - // output as lost (it isn't; the key differs by one '/'). Bodies that - // take a bare key (`{ s3: ... }`) strip the slash via `s3Key`. + // s3 outputs use the canonical slashless key. `parse_asset_syntax` + // normalizes `s3:///` (default storage) and `s3://` to the + // bare ``, so the seeded draft asset must be slashless to match the + // deploy-time inferred identity — otherwise the post-deploy drift check + // would flag the output as a phantom `/`-prefixed node. The generated + // bodies still emit the `s3:///` default-storage URI for runtime I/O. case 's3_parquet': return { kind: 's3object', - path: `/pipelines/${folder}/${adj}_${pick(DATASET_NOUNS)}_${slug}.parquet` + path: `pipelines/${folder}/${adj}_${pick(DATASET_NOUNS)}_${slug}.parquet` } case 's3_object': { // duckdb's natural output for a generic blob is CSV (one COPY TO @@ -173,9 +194,14 @@ export function autoOutputAsset( const ext = language === 'duckdb' ? 'csv' : 'json' return { kind: 's3object', - path: `/pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` + path: `pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` } } + // A macro library produces no asset — its "output" is the registry + // entries the deploy records. A custom data test produces no asset + // either — it asserts against an existing materialized target. + case 'macros': + case 'data_test': case 'none': return undefined } @@ -196,10 +222,11 @@ export function assetUri(asset: { kind: AssetKind; path: string }): string { return `${ASSET_URI_PREFIX[asset.kind]}${asset.path}` } -// Bare object key for the SDK's `{ s3: }` forms: the canonical asset -// path of a default-storage object has a leading slash, the key must not. +// Bare object key for the SDK's `{ s3: }` / `s3:///` forms. Asset +// paths are already canonical slashless keys; strip stray leading slashes +// defensively so the emitted key never starts with '/'. function s3Key(path: string): string { - return path.replace(/^\//, '') + return path.replace(/^\/+/, '') } // Splits a datatable asset path (`/
` or `/.
`) @@ -307,6 +334,17 @@ export type TemplateContext = { function header(ctx: TemplateContext): string { const { language, triggers, output, outputKind } = ctx const p = commentPrefix(language) + // A custom data test is a standalone script, not a graph node that produces + // an asset — so it gets no `// pipeline` / output annotation. Instead, tell + // the author how to wire it up (the `data_test` reference) and the two rules + // that aren't obvious: single SELECT, returning the offending rows. + if (outputKind === 'data_test') { + return [ + `${p} Custom data test — reference it from a materialize script with \`${p} data_test \`.`, + `${p} It must be a single SELECT returning the offending rows; the run fails if any row comes back.`, + '' + ].join('\n') + } const lines = triggers.map((t) => { switch (t.kind) { case 'asset': @@ -330,14 +368,24 @@ function header(ctx: TemplateContext): string { `${p} Strategy: add key=to merge (upsert), or append for insert-only; default replaces the partition` ] : [] + // Macro library: the `// macros` marker registers every CREATE MACRO below + // into the workspace registry at deploy. The hint must not start with a + // parser keyword — `Consumers` is safe. + const macrosLine = + outputKind === 'macros' + ? [ + `${p} macros`, + `${p} Consumers just call these by name; add \`${p} use \` in a consumer to force-inject the whole library` + ] + : [] // Discoverability hint — the three annotations users most often miss // when authoring their first pipeline script. Single line, real // example values (not placeholders) so users see the syntax. Docs // link is the canonical reference once they want the details. A blank // line separates it from the parsed annotations above (`// pipeline`, // `// on …`) so the editor reads as "real annotations, then a hint". - const more = `${p} More: partitioned daily, freshness 1h, retry 3, tag heavy — https://www.windmill.dev/docs/pipelines/annotations` - return [`${p} pipeline`, ...lines, ...matLine, '', more, ''].join('\n') + const more = `${p} More: partitioned daily, freshness 1h, retry 3, tag heavy — https://www.windmill.dev/docs/core_concepts/pipelines` + return [`${p} pipeline`, ...lines, ...matLine, ...macrosLine, '', more, ''].join('\n') } // Bun / Deno bodies. These share the wmill SDK surface, so we treat them @@ -372,8 +420,11 @@ function bodyTs(ctx: TemplateContext): string { if (!input) return '' switch (input.kind) { case 's3object': + // `s3:///` URI — one spelling shared with the `// on + // s3:///…` annotation form (the object literal `{ s3: }` + // is equivalent). return [ - ` const buf = await wmill.loadS3File({ s3: ${JSON.stringify(s3Key(input.path))} })`, + ` const buf = await wmill.loadS3File(${JSON.stringify(`s3:///${s3Key(input.path)}`)})`, ` const rows = JSON.parse(new TextDecoder().decode(buf))`, `` ].join('\n') @@ -399,9 +450,10 @@ function bodyTs(ctx: TemplateContext): string { switch (outputKind) { case 's3_parquet': case 's3_object': + // `s3:///` URI — see the loadS3File note above. return [ ` const payload = new TextEncoder().encode(JSON.stringify(rows))`, - ` await wmill.writeS3File({ s3: ${JSON.stringify(s3Key(output.path))} }, payload)` + ` await wmill.writeS3File(${JSON.stringify(`s3:///${s3Key(output.path)}`)}, payload)` ].join('\n') case 'datatable': { const dbName = output.path.split('/')[0] ?? 'main' @@ -455,8 +507,11 @@ function bodyPython(ctx: TemplateContext): string { if (!input) return '' switch (input.kind) { case 's3object': + // `s3:///` URI — SDK string params must be s3:// URIs + // (bare keys are rejected), and this form matches the + // `# on s3:///…` annotation spelling. return [ - ` buf = wmill.load_s3_file(${JSON.stringify(s3Key(input.path))})`, + ` buf = wmill.load_s3_file(${JSON.stringify(`s3:///${s3Key(input.path)}`)})`, ` import json; rows = json.loads(buf.decode("utf-8"))` ].join('\n') case 'datatable': @@ -479,9 +534,10 @@ function bodyPython(ctx: TemplateContext): string { switch (outputKind) { case 's3_parquet': case 's3_object': + // `s3:///` URI — see the load_s3_file note above. return [ ` import json`, - ` wmill.write_s3_file(${JSON.stringify(s3Key(output.path))}, json.dumps(rows).encode("utf-8"))` + ` wmill.write_s3_file(${JSON.stringify(`s3:///${s3Key(output.path)}`)}, json.dumps(rows).encode("utf-8"))` ].join('\n') case 'datatable': { const dbName = output.path.split('/')[0] ?? 'main' @@ -517,6 +573,22 @@ function bodyPython(ctx: TemplateContext): string { function bodyDuckdb(ctx: TemplateContext): string { const { input, output, outputKind } = ctx const dataUpload = isDataUpload(ctx.triggers) + if (outputKind === 'data_test') { + // Standalone custom data test: it reads ONLY the freshly-materialized + // target, which the runtime attaches under the internal `_wm_target` + // schema — so it emits no ATTACH / input load of its own. When the test + // was created off a ducklake asset, seed its table name; otherwise a + // clear placeholder. This exact shape (single SELECT vs `_wm_target`) is + // what the backend's self-teaching errors ask for. + const testTable = input?.kind === 'ducklake' ? catalogTableRef(input.path) : 'your_table' + return [ + '', + `-- Return the rows that VIOLATE your assertion; an empty result means the test passes.`, + `-- \`_wm_target\` is the freshly-materialized target, attached by the runtime.`, + `SELECT * FROM _wm_target.${testTable} WHERE your_condition;`, + '' + ].join('\n') + } const lines: string[] = [] if (dataUpload) { // `(s3object)` param declaration → the run form renders the S3 picker @@ -565,7 +637,7 @@ function bodyDuckdb(ctx: TemplateContext): string { if (!input) return null switch (input.kind) { case 's3object': - return `read_parquet('s3://${input.path}')` + return `read_parquet('s3:///${input.path}')` case 'datatable': // `pg` is the attached Postgres catalog (see ATTACH above). // Use a 2-part `pg.
` ref so the asset parser maps it @@ -588,7 +660,7 @@ function bodyDuckdb(ctx: TemplateContext): string { `COPY (`, ` SELECT *`, ` FROM ${fromExpr}`, - `) TO 's3://${output.path}' (FORMAT 'parquet');` + `) TO 's3:///${output.path}' (FORMAT 'parquet');` ) } break @@ -599,7 +671,7 @@ function bodyDuckdb(ctx: TemplateContext): string { `COPY (`, ` SELECT *`, ` FROM ${fromExpr}`, - `) TO 's3://${output.path}' (FORMAT 'csv', HEADER);` + `) TO 's3:///${output.path}' (FORMAT 'csv', HEADER);` ) } break @@ -617,9 +689,20 @@ function bodyDuckdb(ctx: TemplateContext): string { // the slice — the runtime wraps it into the idempotent write + // snapshot (see the `// materialize` annotation in the header). No // CREATE TABLE / INSERT, and the target is NOT attached here (the - // runtime attaches it). Add `// partitioned daily` + a `{partition}` - // filter for a partitioned table. - lines.push(`SELECT * FROM ${inSql ?? '(SELECT 1 AS placeholder)'};`) + // runtime attaches it). + // + // Partitioned? `{partition}` is the slice's identity string. For a + // time grain the runtime injects a `wm_partition(ts)` macro that + // buckets a timestamp with that same identity format, so one + // grain-agnostic line filters the source to the active slice — no + // hand-written strftime format, no `= TIMESTAMP {partition}` cast + // (which errors for hourly/weekly/monthly). Kept as a comment because + // we don't know the user's timestamp column. See partitioning docs. + lines.push( + `-- Partitioned? Filter to the active slice (uncomment, set your timestamp column):`, + `-- WHERE wm_partition() = {partition} -- dynamic grain: WHERE = {partition}`, + `SELECT * FROM ${inSql ?? '(SELECT 1 AS placeholder)'};` + ) break case 'datatable': if (output) { @@ -636,6 +719,18 @@ function bodyDuckdb(ctx: TemplateContext): string { ) } break + case 'macros': + // Library body: only CREATE [OR REPLACE] MACRO statements (plus plain + // setup). One scalar + one table example; bodies may only call macros + // defined EARLIER in the file (DuckDB bind-checks at creation). + lines.push( + `CREATE OR REPLACE MACRO safe_div(a, b, fallback := 0) AS`, + ` CASE WHEN b = 0 THEN fallback ELSE a / b END;`, + ``, + `CREATE OR REPLACE MACRO sample_rows(src, n) AS TABLE`, + ` SELECT * FROM query_table(src) LIMIT n;` + ) + break case 'none': default: // With an uploaded file but no output asset, at least surface its diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index dd4c64b225..2216d37b0f 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -9,6 +9,9 @@ const ann = (over: Partial = {}): PipelineAnnotations => ({ triggerAssets: [], nativeTriggers: [], dataTests: [], + columnLineage: [], + macros: false, + useLibs: [], ...over }) @@ -66,11 +69,14 @@ describe('resolveGraph', () => { expect(resolveGraph(input({ base }))).toEqual(base) }) - it('draft: adds an unsaved runnable + write edge from the static outputAsset', () => { + it('draft: adds an unsaved runnable + write edge from outputAssets', () => { const drafts = new Map([ [ 'f/x/d', - { script: { content: '' }, outputAsset: { kind: 's3object' as const, path: '/out.json' } } + { + script: { content: '' }, + outputAssets: [{ kind: 's3object' as const, path: '/out.json' }] + } ] ]) const r = resolveGraph(input({ drafts })) @@ -93,20 +99,48 @@ describe('resolveGraph', () => { }) }) - it('draft: outputAssets snapshot wins over the static outputAsset', () => { + it('scd2 materialize draft: writes both the base dim and its _current companion view', () => { const drafts = new Map([ [ - 'f/x/d', + 'f/x/dim', { - script: { content: '' }, - outputAsset: { kind: 's3object' as const, path: '/old.json' }, - outputAssets: [{ kind: 's3object' as const, path: '/new.json' }] + script: { + content: '-- materialize ducklake://main/dim_customers key=id history\nselect 1' + } } ] ]) const r = resolveGraph(input({ drafts })) - expect(r.edges.map((e) => e.asset_path)).toContain('/new.json') - expect(r.edges.map((e) => e.asset_path)).not.toContain('/old.json') + // Base dimension: a plain write output node. + expect(r.assets).toContainEqual({ kind: 'ducklake', path: 'main/dim_customers' }) + // Companion `_current` view: same producer, marked derived from the base. + expect(r.assets).toContainEqual({ + kind: 'ducklake', + path: 'main/dim_customers_current', + derived_from: 'main/dim_customers' + }) + for (const path of ['main/dim_customers', 'main/dim_customers_current']) { + expect(r.edges).toContainEqual({ + runnable_path: 'f/x/dim', + runnable_kind: 'script', + asset_kind: 'ducklake', + asset_path: path, + access_type: 'w', + unsaved: true + }) + } + }) + + it('non-scd2 materialize draft: writes only the base dim, no _current companion', () => { + const drafts = new Map([ + [ + 'f/x/dim', + { script: { content: '-- materialize ducklake://main/dim_customers key=id\nselect 1' } } + ] + ]) + const r = resolveGraph(input({ drafts })) + expect(r.assets).toContainEqual({ kind: 'ducklake', path: 'main/dim_customers' }) + expect(r.assets.some((a) => a.path === 'main/dim_customers_current')).toBe(false) }) it('active draft: live body writes are authoritative over the snapshot', () => { @@ -351,6 +385,55 @@ describe('resolveGraph', () => { expect(r.edges.some((e) => e.asset_path === 'main/out' && e.access_type === 'w')).toBe(true) }) + it('editing a saved scd2 producer keeps both the base and _current persisted write edges', () => { + // Deploy persists a write to both `main/dim` and `main/dim_current`. + // Opening the producer for editing must not judge the companion `_current` + // write stale — otherwise a consumer of only the view orphans mid-edit. + const base = baseGraph({ + runnables: [{ path: 'f/x/dim', usage_kind: 'script' }], + edges: [ + { + runnable_path: 'f/x/dim', + runnable_kind: 'script', + asset_kind: 'ducklake', + asset_path: 'main/dim_customers', + access_type: 'w' + }, + { + runnable_path: 'f/x/dim', + runnable_kind: 'script', + asset_kind: 'ducklake', + asset_path: 'main/dim_customers_current', + access_type: 'w' + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { + scriptPath: 'f/x/dim', + annotations: ann({ + materialize: { + targetKind: 'ducklake', + targetPath: 'main/dim_customers', + uniqueKey: 'id', + scd2: true + } + }) + }, + liveBodyAssets: { scriptPath: 'f/x/dim', assets: [] } + }) + ) + for (const path of ['main/dim_customers', 'main/dim_customers_current']) { + expect( + r.edges.some( + (e) => e.runnable_path === 'f/x/dim' && e.asset_path === path && e.access_type === 'w' + ) + ).toBe(true) + } + }) + it('selecting a saved script unchanged drops nothing (no stale removal)', () => { const base = baseGraph({ runnables: [{ path: 'f/x/prod', usage_kind: 'script' }], @@ -465,4 +548,85 @@ describe('resolveGraph', () => { missing: true }) }) + + it('macro edges: base passes through; live `// use` adds an unsaved via_use edge', () => { + const base = baseGraph({ + runnables: [ + { + path: 'f/lib/stats', + usage_kind: 'script', + macros: [{ name: 'safe_div', params: 'a, b', is_table: false }] + }, + { path: 'f/x/cons', usage_kind: 'script' } + ], + macro_edges: [ + { + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: false + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { + scriptPath: 'f/x/other', + annotations: ann({ useLibs: ['f/lib/stats'] }) + } + }) + ) + // Detection edge preserved untouched. + expect(r.macro_edges).toContainEqual({ + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: false + }) + // Live `// use` synthesizes an unsaved whole-lib edge with the lib's names. + expect(r.macro_edges).toContainEqual({ + lib_path: 'f/lib/stats', + consumer_path: 'f/x/other', + macro_names: ['safe_div'], + via_use: true, + unsaved: true + }) + }) + + it('macro edges: removing the `// use` line of an overlaid consumer retires its via_use edge', () => { + const base = baseGraph({ + macro_edges: [ + { + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: true + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { scriptPath: 'f/x/cons', annotations: ann() } + }) + ) + expect(r.macro_edges).toEqual([]) + }) + + it('macro edges: draft `// macros` library gets the ƒ badge data from its body', () => { + const drafts = new Map([ + [ + 'f/lib/new', + { + script: { + content: '// macros\nCREATE OR REPLACE MACRO dbl(a) AS a * 2;' + } + } + ] + ]) + const r = resolveGraph(input({ drafts })) + const lib = r.runnables.find((x) => x.path === 'f/lib/new') + expect(lib?.macros).toEqual([{ name: 'dbl', params: 'a', is_table: false }]) + }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index b7c82822fa..328535ef87 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -1,7 +1,8 @@ -import type { AssetGraphResponse, NativeTriggerKind } from './types' +import type { AssetGraphMacroEdge, AssetGraphResponse, NativeTriggerKind } from './types' import { mergeColumnLineage, parsePipelineAnnotations, + scd2CurrentTargetPath, type ColumnLineage, type PipelineAnnotations } from './parsePipelineAnnotations' @@ -15,7 +16,6 @@ import { /** Minimal structural shape of a pipeline draft `resolveGraph` needs. */ export type GraphDraft = { script: { content: string } - outputAsset?: { kind: AssetKind; path: string } outputAssets?: Array<{ kind: AssetKind; path: string }> } @@ -147,15 +147,94 @@ export function resolveGraph(input: ResolveGraphInput): AssetGraphResponse { return false return true }) + // Mirror the backend's skip-if-empty: no `macro_edges` key at all when + // there is nothing to show (also keeps the no-macros response shape + // byte-identical to before the feature). + const macroEdges = resolveMacroEdges(input) return { ...base, assets: acc.assets, runnables: acc.runnables, edges: acc.edges, - triggers: [...baseTriggers, ...acc.extraTriggers] + triggers: [...baseTriggers, ...acc.extraTriggers], + ...(macroEdges.length > 0 || base.macro_edges ? { macro_edges: macroEdges } : {}) } } +/** + * Macro-library → consumer edges: the deployed base edges, with `// use` + * declarations of overlaid scripts (drafts + the open buffer) taking over + * their consumer's `via_use` edges so adding/removing a `// use` line updates + * the canvas live. Detection-based edges (macro calls in the deployed body) + * are backend-owned and only refresh on redeploy. + */ +function resolveMacroEdges(input: ResolveGraphInput): AssetGraphMacroEdge[] { + const { base, drafts, liveAnnotations } = input + const libMacroNames = new Map() + for (const r of base.runnables) { + if (r.usage_kind === 'script' && r.macros?.length) { + libMacroNames.set( + r.path, + r.macros.map((m) => m.name) + ) + } + } + const useByPath = new Map() + for (const [path, d] of drafts) { + useByPath.set(path, parsePipelineAnnotations(d.script.content).useLibs) + } + if (liveAnnotations.scriptPath) { + // `?? []` — callers may hand a minimal annotations object (tests, older + // call sites) that predates the field. + useByPath.set(liveAnnotations.scriptPath, liveAnnotations.annotations.useLibs ?? []) + } + const out: AssetGraphMacroEdge[] = [] + for (const e of base.macro_edges ?? []) { + if (e.via_use && useByPath.has(e.consumer_path)) continue + out.push({ ...e }) + } + for (const [path, libs] of useByPath) { + for (const lib of libs) { + const existing = out.find((e) => e.lib_path === lib && e.consumer_path === path) + if (existing) { + // Upgrade the detection edge in place: `// use` pulls in the whole + // library, so the edge covers every macro the lib defines. + existing.via_use = true + existing.unsaved = true + existing.macro_names = [ + ...new Set([...existing.macro_names, ...(libMacroNames.get(lib) ?? [])]) + ] + } else { + out.push({ + lib_path: lib, + consumer_path: path, + macro_names: libMacroNames.get(lib) ?? [], + via_use: true, + unsaved: true + }) + } + } + } + return out +} + +// Light regex extraction of a draft macro library's definitions for the live +// node badge. The strict grammar lives in the Rust `parse_macro_library` at +// deploy; this only needs name/params/table-ness for display (nested parens +// in a default value may truncate the shown signature, never the deploy). +const MACRO_DEF_RE = + /create\s+(?:or\s+replace\s+)?(?:temp(?:orary)?\s+)?(?:macro|function)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]*)\)\s*as\s+(table\b)?/gi + +export function extractDraftMacros( + content: string +): { name: string; params: string; is_table: boolean }[] { + const out: { name: string; params: string; is_table: boolean }[] = [] + for (const m of content.matchAll(MACRO_DEF_RE)) { + out.push({ name: m[1].toLowerCase(), params: m[2].trim(), is_table: m[3] !== undefined }) + } + return out +} + type ResolveContext = { draftedPaths: Set isDrafted: (kind: string, p: string) => boolean @@ -187,9 +266,16 @@ function makeContext(input: ResolveGraphInput): ResolveContext { // lives in an annotation (not the SQL body), so neither triggerAssets // (inputs) nor the body-inferred assets cover it. Without this its // persisted write-edge is judged stale and dropped the moment the - // script is selected/edited — leaving the output asset unlinked. + // script is selected/edited — leaving the output asset unlinked. A + // managed scd2 producer persists a second write to the `_current` + // companion view, so keep that too or a consumer of only the view + // orphans while the producer is open for editing. const m = liveAnnotations.annotations.materialize - if (m) liveRefKeys.add(`${m.targetKind}:${m.targetPath}`) + if (m) { + liveRefKeys.add(`${m.targetKind}:${m.targetPath}`) + const currentPath = scd2CurrentTargetPath(m) + if (currentPath) liveRefKeys.add(`${m.targetKind}:${currentPath}`) + } } for (const a of liveBodyAssets.assets) liveRefKeys.add(`${a.kind}:${a.path}`) } @@ -252,6 +338,9 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // duplicate (which would crash svelte-flow's keyed each), so the // canvas + trigger-node labels reflect that there's pending body // editing for this path. + // `// macros` library draft: extract the definitions for the live node + // badge (regex-light; the strict parse happens at deploy). + const draftMacros = parsed.macros ? extractDraftMacros(d.script.content) : [] const baseIdx = runnables.findIndex((r) => r.usage_kind === 'script' && r.path === path) if (baseIdx === -1) { runnables.push({ @@ -265,6 +354,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, column_lineage: mergedCL.length > 0 ? mergedCL : undefined, materialize_target: materializeTarget, + macros: draftMacros.length > 0 ? draftMacros : undefined, unsaved: true }) } else { @@ -277,46 +367,58 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, column_lineage: mergedCL.length > 0 ? mergedCL : undefined, materialize_target: materializeTarget, + macros: draftMacros.length > 0 ? draftMacros : undefined, unsaved: true } } - // Output asset(s): three-tier resolution. + // Output asset(s): two-tier resolution. // 1. Active draft (the body the user is editing right now): // live body inference is authoritative — renaming a // CREATE TABLE target or writeS3File path retires the // old output node and surfaces the new one as the user // types. - // 2. Inactive draft with a captured `outputAssets` snapshot - // (taken on the last pane transition): use those, so a - // draft the user already edited keeps its renamed outputs - // after they've clicked elsewhere. - // 3. Fallback to the static `outputAsset` seeded at draft - // creation — covers fresh drafts and parser misses (e.g. - // WIN-1943: wmill.writeS3File({s3, storage}) object form - // not yet detected by the TS parser). + // 2. Inactive draft: its captured `outputAssets` (inferred at + // creation/last edit, or the seeded output for a fresh draft + // whose body doesn't yet write anything inferable). const liveForThisDraft = liveBodyAssets.scriptPath === path - const writeOuts: Array<{ kind: AssetKind; path: string }> = [] + const writeOuts: Array<{ kind: AssetKind; path: string; derivedFrom?: string }> = [] if (liveForThisDraft) { writeOuts.push(...extractWrites(liveBodyAssets.assets)) } else if (d.outputAssets) { writeOuts.push(...d.outputAssets) } - if (writeOuts.length === 0 && d.outputAsset) { - writeOuts.push(d.outputAsset) - } // `// materialize ` declares a write output via annotation, not // the SQL body, so the body-inference tiers above miss it. Add it from // the live-parsed annotations so an edited materialize script keeps its - // output edge (the loop below dedups against existing assets/edges). + // output edge (the loop below dedups against existing assets/edges). A + // managed scd2 materialize also produces the `_current` companion + // view — add it too (mirrors the deploy path) so a draft consuming only + // the view links back to this producer instead of orphaning. if (parsed.materialize) { writeOuts.push({ kind: parsed.materialize.targetKind, path: parsed.materialize.targetPath }) + const currentPath = scd2CurrentTargetPath(parsed.materialize) + if (currentPath) { + writeOuts.push({ + kind: parsed.materialize.targetKind, + path: currentPath, + derivedFrom: parsed.materialize.targetPath + }) + } } for (const out of writeOuts) { - const hasAsset = assets.some((a) => a.kind === out.kind && a.path === out.path) - if (!hasAsset) assets.push({ kind: out.kind, path: out.path }) + const existing = assets.find((a) => a.kind === out.kind && a.path === out.path) + if (!existing) { + assets.push({ + kind: out.kind, + path: out.path, + ...(out.derivedFrom ? { derived_from: out.derivedFrom } : {}) + }) + } else if (out.derivedFrom && existing.derived_from == undefined) { + existing.derived_from = out.derivedFrom + } // Dedup against edges already in the overlay (this draft's base // edges were dropped above, so this only guards against duplicate // writeOuts entries — not against the persisted version). diff --git a/frontend/src/lib/components/assets/AssetGraph/schemaContracts.test.ts b/frontend/src/lib/components/assets/AssetGraph/schemaContracts.test.ts new file mode 100644 index 0000000000..9247b23d9b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/schemaContracts.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from 'vitest' +import { + buildSchemaContractContext, + diffSchemaContracts, + mapWarningsToMarkers, + normalizeAssetPath, + referencedDucklakePaths, + type CapturedSchemaLite +} from './schemaContracts' +import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import type { AssetWithAltAccessType } from '../lib' + +// Mirrors backend/windmill-common/src/schema_contracts.rs unit tests — the two +// diffs must apply the same rules or the editor previews a different verdict +// than the save-time check returns. + +function schema(cols: [string, string][], version = 2): CapturedSchemaLite { + return { + columns: cols.map(([name, type]) => ({ name, type })), + version, + capturedAt: '2026-01-01T00:00:00Z' + } +} + +function readAsset(path: string, cols: string[]): AssetWithAltAccessType { + return { + path, + kind: 'ducklake', + access_type: 'r', + columns: Object.fromEntries(cols.map((c) => [c, 'r' as const])) + } +} + +const NO_ANN = { columnLineage: [], dataTests: [] } + +describe('diffSchemaContracts', () => { + it('warns on a missing read column, matching case-insensitively', () => { + const schemas = new Map([ + [ + 'lake/orders', + schema([ + ['Order_ID', 'BIGINT'], + ['amount_usd', 'DOUBLE'] + ]) + ] + ]) + const w = diffSchemaContracts({ + ...NO_ANN, + assets: [readAsset('lake/orders', ['order_id', 'amount'])], + schemas, + ignored: new Set() + }) + expect(w).toHaveLength(1) + expect(w[0].kind).toBe('missing_column') + expect(w[0].column).toBe('amount') + expect(w[0].schema_version).toBe(2) + }) + + it('skips unknown-column assets, "*" and reserved columns', () => { + const schemas = new Map([['lake/orders', schema([['id', 'BIGINT']])]]) + const noColumns: AssetWithAltAccessType = { + path: 'lake/orders', + kind: 'ducklake', + access_type: 'r' + } + expect( + diffSchemaContracts({ ...NO_ANN, assets: [noColumns], schemas, ignored: new Set() }) + ).toEqual([]) + expect( + diffSchemaContracts({ + ...NO_ANN, + assets: [readAsset('lake/orders', ['*', '_wm_partition', 'id'])], + schemas, + ignored: new Set() + }) + ).toEqual([]) + }) + + it('is silent for assets without a captured schema', () => { + expect( + diffSchemaContracts({ + ...NO_ANN, + assets: [readAsset('lake/unknown', ['whatever'])], + schemas: new Map(), + ignored: new Set() + }) + ).toEqual([]) + }) + + it('normalizes the {partition} token before lookup', () => { + const schemas = new Map([['lake/orders', schema([['id', 'BIGINT']])]]) + const w = diffSchemaContracts({ + ...NO_ANN, + assets: [readAsset('lake/orders/{partition}', ['gone'])], + schemas, + ignored: new Set() + }) + expect(w).toHaveLength(1) + expect(w[0].asset_path).toBe('lake/orders') + }) + + it('warns on broken // column lineage refs', () => { + const ann = parsePipelineAnnotations( + '// column total <- ducklake://lake/orders.amount\nSELECT 1;' + ) + const schemas = new Map([['lake/orders', schema([['id', 'BIGINT']])]]) + const w = diffSchemaContracts({ + assets: [], + columnLineage: ann.columnLineage, + dataTests: [], + schemas, + ignored: new Set() + }) + expect(w).toHaveLength(1) + expect(w[0].kind).toBe('missing_lineage_source') + }) + + it('flags missing relationship columns and captured-type differences', () => { + const ann = parsePipelineAnnotations( + '// materialize ducklake://lake/orders\n' + + '// data_test relationships customer_id -> ducklake://lake/customers.id\n' + + '// data_test relationships customer_id -> ducklake://lake/customers.uuid\n' + + 'SELECT 1;' + ) + const schemas = new Map([ + ['lake/customers', schema([['id', 'VARCHAR']])], + ['lake/orders', schema([['customer_id', 'BIGINT']])] + ]) + const w = diffSchemaContracts({ + assets: [], + columnLineage: ann.columnLineage, + dataTests: ann.dataTests, + materialize: ann.materialize, + schemas, + ignored: new Set() + }) + expect(w).toHaveLength(2) + expect( + w.some( + (x) => + x.kind === 'relationship_type_mismatch' && + x.expected_type === 'BIGINT' && + x.found_type === 'VARCHAR' + ) + ).toBe(true) + expect(w.some((x) => x.kind === 'missing_relationship_column' && x.column === 'uuid')).toBe( + true + ) + }) + + it('suppresses ignored assets down to one informational note', () => { + const schemas = new Map([['lake/orders', schema([['id', 'BIGINT']])]]) + const ignored = new Set(['lake/orders']) + const w = diffSchemaContracts({ + ...NO_ANN, + assets: [readAsset('lake/orders', ['a', 'b'])], + schemas, + ignored + }) + expect(w).toHaveLength(1) + expect(w[0].kind).toBe('suppressed') + expect( + diffSchemaContracts({ + ...NO_ANN, + assets: [readAsset('lake/orders', ['id'])], + schemas, + ignored + }) + ).toEqual([]) + }) +}) + +describe('referencedDucklakePaths', () => { + it('collects paths from reads, lineage, relationships and materialize', () => { + const ann = parsePipelineAnnotations( + '// materialize ducklake://lake/out\n' + + '// column total <- ducklake://lake/a.amount\n' + + '// data_test relationships k -> ducklake://lake/b.id\n' + + 'SELECT 1;' + ) + const refs = referencedDucklakePaths({ + assets: [readAsset('lake/c/{partition}', ['x'])], + columnLineage: ann.columnLineage, + dataTests: ann.dataTests, + materialize: ann.materialize + }) + expect(refs.sort()).toEqual(['lake/a', 'lake/b', 'lake/c', 'lake/out']) + }) +}) + +describe('buildSchemaContractContext', () => { + it('derives ignored assets and scd2 _current bases from graph runnables', () => { + const ctx = buildSchemaContractContext([ + { + materialize_target: { kind: 'ducklake', path: 'lake/dim' }, + materialize_strategy: 'scd2', + materialize_on_schema_change: 'ignore' + }, + { + materialize_target: { kind: 'ducklake', path: 'lake/orders' }, + materialize_strategy: 'replace' + }, + // non-ducklake and absent targets are ignored + { materialize_target: { kind: 's3object', path: 'x/y' }, materialize_strategy: 'scd2' }, + {} + ]) + expect(ctx.ignoredAssets).toEqual(['lake/dim', 'lake/dim_current']) + expect(ctx.scd2CurrentBases).toEqual({ 'lake/dim_current': 'lake/dim' }) + }) + + it('ignores _current only for scd2 producers (backend spec.scd2 gate)', () => { + const ctx = buildSchemaContractContext([ + { + materialize_target: { kind: 'ducklake', path: 'lake/t' }, + materialize_strategy: 'replace', + materialize_on_schema_change: 'ignore' + } + ]) + // a non-scd2 producer's `_current` is an unrelated asset — it must + // keep warning, exactly like the server-side check + expect(ctx.ignoredAssets).toEqual(['lake/t']) + expect(ctx.scd2CurrentBases).toEqual({}) + }) +}) + +describe('mapWarningsToMarkers', () => { + it('anchors annotation warnings to their lines and body reads to the identifier', () => { + const code = + '-- pipeline\n' + + '-- on ducklake://lake/orders\n' + + '-- column total <- ducklake://lake/orders.amount\n' + + '-- data_test relationships k -> ducklake://lake/customers.uuid\n' + + 'SELECT amount FROM dl.orders;' + const markers = mapWarningsToMarkers(code, [ + { + kind: 'missing_lineage_source', + asset_path: 'lake/orders', + column: 'amount', + message: 'm1' + }, + { + kind: 'missing_relationship_column', + asset_path: 'lake/customers', + column: 'uuid', + message: 'm2' + }, + { kind: 'missing_column', asset_path: 'lake/orders', column: 'amount', message: 'm3' }, + { kind: 'suppressed', asset_path: 'lake/orders', message: 'hidden' } + ]) + expect(markers).toHaveLength(3) + expect(markers[0].startLineNumber).toBe(3) + expect(markers[1].startLineNumber).toBe(4) + // body-read warning anchors to the first occurrence of the identifier, + // which is the annotation line mentioning `amount` (line 3) + expect(markers[2].startLineNumber).toBe(3) + // token range is tight around the identifier, not the whole line + const line3 = '-- column total <- ducklake://lake/orders.amount' + expect(markers[2].startColumn).toBe(line3.indexOf('amount') + 1) + }) + + it('falls back to the line mentioning the asset path', () => { + const code = '# pipeline\n# on ducklake://lake/orders\nprint(1)' + const markers = mapWarningsToMarkers(code, [ + { kind: 'missing_column', asset_path: 'lake/orders', column: 'zzz', message: 'm' } + ]) + expect(markers[0].startLineNumber).toBe(2) + }) +}) + +describe('normalizeAssetPath', () => { + it('strips the partition token and trailing slashes', () => { + expect(normalizeAssetPath('lake/orders/{partition}')).toBe('lake/orders') + expect(normalizeAssetPath('lake/orders_{partition}')).toBe('lake/orders_') + expect(normalizeAssetPath('lake/orders/')).toBe('lake/orders') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts b/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts new file mode 100644 index 0000000000..4a1de04e13 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts @@ -0,0 +1,440 @@ +// Client-side mirror of the save-time schema-contract check (pipelines gap +// #2b, backend/windmill-common/src/schema_contracts.rs). The backend endpoint +// (`checkSchemaContracts`) is the authoritative check run on save; this mirror +// drives the *live* editor surface (Monaco warning markers + completions) from +// the WASM parse that already runs on the open buffer, so the two must apply +// the same rules: ducklake-only, case-insensitive column names, `columns` +// absent ⇒ skip, `_wm_partition` whitelisted, `{partition}` token stripped, +// annotation-declared lineage only, asset without captured schema ⇒ silent. + +import { AssetService, ScriptService, type ContractWarning, type ScriptLang } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import type { AssetWithAltAccessType } from '../lib' +import { + parsePipelineAnnotations, + type ColumnLineage, + type DataTest, + type MaterializeSpec +} from './parsePipelineAnnotations' + +// Columns the materialize engine manages; excluded from the captured schema on +// purpose, so reads of them must not warn. +const RESERVED_COLUMNS = ['_wm_partition'] + +const PARTITION_TOKEN = '{partition}' + +export type CapturedSchemaLite = { + columns: { name: string; type: string }[] + version: number + capturedAt: string +} + +// Strip the `{partition}` token a declared URI may carry so lookups hit the +// captured path (mirrors `normalize_asset_path`). +export function normalizeAssetPath(path: string): string { + return path + .replaceAll('/' + PARTITION_TOKEN, '') + .replaceAll(PARTITION_TOKEN, '') + .replace(/\/+$/, '') +} + +function isReserved(name: string): boolean { + return RESERVED_COLUMNS.some((r) => r.toLowerCase() === name.toLowerCase()) +} + +function findColumn( + schema: CapturedSchemaLite, + name: string +): { name: string; type: string } | undefined { + const lower = name.toLowerCase() + return schema.columns.find((c) => c.name.toLowerCase() === lower) +} + +export type SchemaContractInputs = { + // Per-asset column reads/writes from the WASM parse (entries without a + // `columns` map are skipped — wildcard/unknown access). + assets: AssetWithAltAccessType[] + // Annotation-declared `// column` lineage ONLY (not merged AST-inferred + // lineage — redundant with body reads and alias-attribution can misfire). + columnLineage: ColumnLineage[] + dataTests: DataTest[] + materialize?: MaterializeSpec + // Latest captured schema per normalized ducklake path (after any + // `_current` → base-table fallback the caller resolved). + schemas: Map + // Normalized paths whose producer declares `on_schema_change=ignore`. + ignored: Set +} + +// Mirrors backend `diff_contract` — same warning kinds and suppression +// semantics, minus the human message wording (the editor renders its own). +export function diffSchemaContracts(input: SchemaContractInputs): ContractWarning[] { + const { assets, columnLineage, dataTests, materialize, schemas, ignored } = input + const warnings: ContractWarning[] = [] + + // W1 — body-read/written columns missing from the captured schema. + for (const a of assets) { + if (a.kind !== 'ducklake' || a.columns == undefined) continue + const path = normalizeAssetPath(a.path) + const schema = schemas.get(path) + if (!schema) continue + for (const col of Object.keys(a.columns)) { + if (col === '*' || isReserved(col)) continue + if (!findColumn(schema, col)) { + warnings.push({ + kind: 'missing_column', + asset_path: path, + column: col, + schema_version: schema.version, + captured_at: schema.capturedAt, + message: `column \`${col}\` of ducklake://${path} is not in its captured schema (v${schema.version}, columns: ${schema.columns.map((c) => c.name).join(', ')})` + }) + } + } + } + + // W2 — `// column` lineage source refs. + for (const cl of columnLineage) { + for (const input of cl.inputs) { + if (input.from_kind !== 'ducklake' || isReserved(input.from_column)) continue + const path = normalizeAssetPath(input.from_path) + const schema = schemas.get(path) + if (!schema) continue + if (!findColumn(schema, input.from_column)) { + warnings.push({ + kind: 'missing_lineage_source', + asset_path: path, + column: input.from_column, + schema_version: schema.version, + captured_at: schema.capturedAt, + message: `\`// column ${cl.column}\` reads \`${input.from_column}\` from ducklake://${path}, which is not in its captured schema (v${schema.version})` + }) + } + } + } + + // W3 — relationships refs: missing column, and captured-type difference + // when the consumer's own materialize target has a capture. Types still + // coerce at run time, so a difference is "differs", never "will fail". + const ownSchema = + materialize?.targetKind === 'ducklake' + ? schemas.get(normalizeAssetPath(materialize.targetPath)) + : undefined + for (const dt of dataTests) { + if (dt.type !== 'relationships' || dt.to_kind !== 'ducklake') continue + const path = normalizeAssetPath(dt.to_path) + const schema = schemas.get(path) + if (!schema) continue + const refCol = findColumn(schema, dt.to_column) + if (!refCol) { + warnings.push({ + kind: 'missing_relationship_column', + asset_path: path, + column: dt.to_column, + schema_version: schema.version, + captured_at: schema.capturedAt, + message: `\`// data_test relationships ${dt.column}\` references ducklake://${path}.${dt.to_column}, which is not in its captured schema (v${schema.version})` + }) + } else { + const ownCol = ownSchema && findColumn(ownSchema, dt.column) + if (ownCol && ownCol.type.toLowerCase() !== refCol.type.toLowerCase()) { + warnings.push({ + kind: 'relationship_type_mismatch', + asset_path: path, + column: dt.to_column, + expected_type: ownCol.type, + found_type: refCol.type, + schema_version: schema.version, + captured_at: schema.capturedAt, + message: `\`// data_test relationships ${dt.column}\` joins \`${dt.column}\` (${ownCol.type}) to ducklake://${path}.${dt.to_column} (${refCol.type}) — captured types differ` + }) + } + } + } + + // W4 — producer `on_schema_change=ignore`: drop the asset's warnings, + // leaving one informational entry per suppressed asset. + if (ignored.size > 0) { + const suppressed: string[] = [] + const kept = warnings.filter((w) => { + if (ignored.has(w.asset_path)) { + if (!suppressed.includes(w.asset_path)) suppressed.push(w.asset_path) + return false + } + return true + }) + warnings.length = 0 + warnings.push(...kept) + for (const path of suppressed) { + warnings.push({ + kind: 'suppressed', + asset_path: path, + message: `schema mismatches on ducklake://${path} suppressed by its producer's \`on_schema_change=ignore\`` + }) + } + } + + return warnings +} + +// The ducklake paths a buffer references in ways the contract check inspects — +// what the editor needs captured schemas for. +export function referencedDucklakePaths( + input: Pick +): string[] { + const paths = new Set() + for (const a of input.assets) { + if (a.kind === 'ducklake' && a.columns != undefined) paths.add(normalizeAssetPath(a.path)) + } + for (const cl of input.columnLineage) { + for (const i of cl.inputs) { + if (i.from_kind === 'ducklake') paths.add(normalizeAssetPath(i.from_path)) + } + } + for (const dt of input.dataTests) { + if (dt.type === 'relationships' && dt.to_kind === 'ducklake') + paths.add(normalizeAssetPath(dt.to_path)) + } + if (input.materialize?.targetKind === 'ducklake') + paths.add(normalizeAssetPath(input.materialize.targetPath)) + return [...paths] +} + +// --- Captured-schema cache ------------------------------------------------ + +// Short-TTL cache so per-keystroke recomputes and completion requests don't +// re-fetch. Captured schemas only change when a producer materializes, so a +// briefly stale hit is fine — the authoritative save-time check re-reads. +const SCHEMA_TTL_MS = 30_000 +const schemaCache = new Map() + +export async function fetchLatestSchema( + workspace: string, + path: string +): Promise { + const key = `${workspace}:${path}` + const hit = schemaCache.get(key) + if (hit && Date.now() - hit.at < SCHEMA_TTL_MS) return hit.value + let value: CapturedSchemaLite | undefined = undefined + try { + const versions = await AssetService.listAssetSchemas({ workspace, path }) + const latest = versions[0] + if (latest) { + value = { + columns: latest.columns, + version: latest.version, + capturedAt: latest.captured_at + } + } + } catch (e) { + console.error('failed to fetch captured asset schema', path, e) + } + schemaCache.set(key, { at: Date.now(), value }) + return value +} + +// Resolve the schema map for a set of referenced paths, applying the scd2 +// `_current` → base-table fallback when the graph identifies the view's +// producer as a managed scd2 materializer (the view is `SELECT * … WHERE +// is_current`, so columns are identical). +export async function fetchSchemasForPaths( + workspace: string, + paths: string[], + scd2CurrentBase?: (path: string) => string | undefined +): Promise> { + const out = new Map() + await Promise.all( + paths.map(async (p) => { + let schema = await fetchLatestSchema(workspace, p) + if (!schema && p.endsWith('_current')) { + const base = scd2CurrentBase?.(p) + if (base) schema = await fetchLatestSchema(workspace, base) + } + if (schema) out.set(p, schema) + }) + ) + return out +} + +// --- Pipeline-graph context --------------------------------------------------- + +// Producer-side facts the contract mirror needs but cannot derive from the +// open buffer: which assets are muted (`on_schema_change=ignore`) and which +// `_current` views map to an scd2 base table. Built by the pipeline page +// from the resolved graph; absent outside the pipeline editor (standalone +// script editor), where suppression simply doesn't apply client-side — the +// save-time server check remains authoritative either way. +export type SchemaContractGraphContext = { + // Normalized asset paths whose producer declares `on_schema_change=ignore`. + ignoredAssets: string[] + // `_current` → base for managed scd2 producers in the graph. + scd2CurrentBases: Record +} + +export function buildSchemaContractContext( + runnables: Pick< + import('./types').AssetGraphRunnableNode, + 'materialize_target' | 'materialize_strategy' | 'materialize_on_schema_change' + >[] +): SchemaContractGraphContext { + const ignoredAssets: string[] = [] + const scd2CurrentBases: Record = {} + for (const r of runnables) { + const t = r.materialize_target + if (!t || t.kind !== 'ducklake') continue + const base = normalizeAssetPath(t.path) + if (r.materialize_on_schema_change === 'ignore') { + ignoredAssets.push(base) + // The `_current` companion is the producer's own view only for scd2 — + // mirroring the backend's `spec.scd2` gate; for any other strategy a + // `_current` ref is an unrelated asset that must keep warning. + if (r.materialize_strategy === 'scd2') { + ignoredAssets.push(`${base}_current`) + } + } + if (r.materialize_strategy === 'scd2') { + scd2CurrentBases[`${base}_current`] = base + } + } + return { ignoredAssets, scd2CurrentBases } +} + +// --- Save-time surface ------------------------------------------------------ + +// Run the authoritative backend check for just-deployed content and toast the +// result. Never throws — a failed check must not taint a successful deploy. +export async function notifyContractWarnings( + workspace: string, + language: ScriptLang, + content: string +): Promise { + // Every checkable ref carries the `ducklake` token (URIs and the bare + // default-syntax shorthand alike) — skip the round-trip for the vast + // majority of saves that can't produce a warning. + if (!content.includes('ducklake')) return + try { + const { warnings } = await ScriptService.checkSchemaContracts({ + workspace, + requestBody: { language, content } + }) + const real = warnings.filter((w) => w.kind !== 'suppressed') + if (real.length === 0) return + sendUserToast( + `Schema contract: ${real.length} warning${real.length > 1 ? 's' : ''}`, + 'warning', + [], + real.map((w) => `• ${w.message}`).join('\n'), + 10000 + ) + } catch (e) { + console.error('schema-contract check failed', e) + } +} + +// End-to-end live-editor check: parse the buffer's annotations, resolve the +// captured schemas for everything it references, diff, and anchor the result +// to source positions. Cheap per keystroke — the annotation parse is a line +// scan and schema fetches hit the short-TTL cache. +export async function computeContractMarkers( + workspace: string, + code: string, + assets: AssetWithAltAccessType[], + context?: SchemaContractGraphContext +): Promise { + const ann = parsePipelineAnnotations(code) + const inputs = { + assets, + // Annotation-declared lineage only — body-inferred lineage is redundant + // with the body-read check and its alias attribution can misfire. + columnLineage: ann.columnLineage, + dataTests: ann.dataTests, + materialize: ann.materialize + } + const refs = referencedDucklakePaths(inputs) + if (refs.length === 0) return [] + const schemas = await fetchSchemasForPaths( + workspace, + refs, + context ? (p) => context.scd2CurrentBases[p] : undefined + ) + if (schemas.size === 0) return [] + const warnings = diffSchemaContracts({ + ...inputs, + schemas, + ignored: new Set(context?.ignoredAssets ?? []) + }) + return mapWarningsToMarkers(code, warnings) +} + +// --- Editor marker mapping --------------------------------------------------- + +export type ContractMarker = { + message: string + startLineNumber: number + startColumn: number + endLineNumber: number + endColumn: number +} + +// Best-effort source anchoring: annotation-family warnings anchor to their +// annotation line; body-read warnings anchor to the first occurrence of the +// column identifier; fallback is the `// on`/first line mentioning the asset. +export function mapWarningsToMarkers(code: string, warnings: ContractWarning[]): ContractMarker[] { + const lines = code.split('\n') + + function lineMatching(pred: (line: string) => boolean): number | undefined { + const idx = lines.findIndex(pred) + return idx >= 0 ? idx + 1 : undefined + } + + function tokenRange( + lineNumber: number, + token: string + ): { startColumn: number; endColumn: number } { + const line = lines[lineNumber - 1] ?? '' + const idx = line.toLowerCase().indexOf(token.toLowerCase()) + if (idx < 0) return { startColumn: 1, endColumn: line.length + 1 } + return { startColumn: idx + 1, endColumn: idx + 1 + token.length } + } + + return warnings + .filter((w) => w.kind !== 'suppressed') + .map((w) => { + let lineNumber: number | undefined + let token: string | undefined = w.column ?? undefined + switch (w.kind) { + case 'missing_lineage_source': + lineNumber = lineMatching( + (l) => /^\s*(\/\/|--|#)\s*column\s/.test(l) && !!w.column && l.includes(w.column) + ) + break + case 'missing_relationship_column': + case 'relationship_type_mismatch': + lineNumber = lineMatching( + (l) => + /^\s*(\/\/|--|#)\s*data_test\s+relationships\s/.test(l) && l.includes(w.asset_path) + ) + break + case 'missing_column': { + // first body occurrence of the column identifier + const re = new RegExp(`\\b${w.column?.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i') + lineNumber = w.column ? lineMatching((l) => re.test(l)) : undefined + break + } + } + if (lineNumber == undefined) { + // fallback: the `// on …` (or any) line mentioning the asset path + lineNumber = lineMatching((l) => l.includes(w.asset_path)) ?? 1 + token = w.asset_path + } + const range = token + ? tokenRange(lineNumber, token) + : { startColumn: 1, endColumn: (lines[lineNumber - 1]?.length ?? 0) + 1 } + return { + message: w.message, + startLineNumber: lineNumber, + endLineNumber: lineNumber, + ...range + } + }) +} diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 9276f500af..0e055abca2 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -6,6 +6,16 @@ export type GraphUsageKind = 'script' | 'flow' export interface AssetGraphAssetNode { kind: AssetKind path: string + // Fork workspaces only: 'fork' when this ducklake asset was materialized in + // the fork itself, 'deferred' when reads fall back to the parent workspace's + // current table via a defer view. Absent outside forks / for other kinds / + // when never materialized anywhere. Lockstep with Rust `GraphAssetNode`. + fork_materialization?: 'fork' | 'deferred' + // Base dimension path this node is the SCD2 `_current` companion view of + // (its producer declares `// materialize … history` on ``). Set only on + // the `_current` node; lets the canvas mark it as a derived "current view" + // rather than an unrelated table. Lockstep with Rust `GraphAssetNode`. + derived_from?: string } export interface AssetGraphRunnableNode { @@ -22,6 +32,11 @@ export interface AssetGraphRunnableNode { // Raw `// freshness ` value, e.g. "1h", "30m". Surfaced for // the badge; the runtime parses it as needed. freshness?: string + // Completion time (ISO) of the newest successful run of this pipeline + // member visible to the caller. The freshness chip compares it against + // the `// freshness` window to render fresh/stale. Absent = no + // successful run found (or none visible under job RLS). + last_success_at?: string // `// tag ` worker-tag override. Surfaced for the badge so users // can see which worker pool will pick this script up at a glance. tag?: string @@ -44,11 +59,24 @@ export interface AssetGraphRunnableNode { // Managed `// materialize` write strategy. Absent for non-materializing or // `manual` scripts. Used (with `partition_kind`) to decide whether a // produced asset's schema can evolve: only whole-table `replace` can, since - // `append`/`merge`/partitioned writes INSERT into a fixed-schema table. - materialize_strategy?: 'replace' | 'append' | 'merge' + // `append`/`merge`/`scd2`/partitioned writes INSERT into a fixed-schema + // table. `scd2` also identifies the producer of a `_current` companion + // view for the schema-contract `_current` → base-table fallback. + materialize_strategy?: 'replace' | 'append' | 'merge' | 'scd2' + // `on_schema_change=ignore` on the managed materialize — the producer's + // opt-out from downstream schema-contract warnings. Only present when set + // to `ignore` (default `warn` is absent). Threaded into the editor's + // contract mirror so it suppresses the same warnings the server check does. + materialize_on_schema_change?: string + // Macros this script provides to the workspace registry (deployed + // `// macros` library). Non-empty marks the node as a macro library; + // drives the "defines N macros" badge and the details-pane signature + // list. `params` is the verbatim parameter list. + macros?: { name: string; params: string; is_table: boolean }[] // Synthesized by the page from a local draft; the script doesn't exist // in the DB yet. Drives a dashed/lower-opacity rendering to mirror how // unsaved triggers are styled — visually distinct from persisted nodes. + // AI-built nodes are plain drafts too (no separate pending/approval state). unsaved?: boolean } @@ -112,11 +140,41 @@ export type AssetGraphTrigger = missing?: boolean } +// Macro-library → consumer edge: the consumer calls `macro_names` of +// `lib_path`'s macros (deploy-recorded detection), or pulls in the whole +// library via `// use` (`via_use`, macro_names then lists the full library). +// `unsaved: true` marks a draft's `// use` overlay. +export interface AssetGraphMacroEdge { + lib_path: string + consumer_path: string + macro_names: string[] + via_use: boolean + unsaved?: boolean +} + +// Ordering-only "must-run-after" edge: `runnable_path`'s `// data_test` +// (a `relationships` ref, or a custom test reading a pipeline asset) needs +// `asset` materialized before the tested script runs — but the tested script +// doesn't consume the asset's rows, so this is NOT a lineage edge. Resolved +// server-side to the referenced asset's in-pipeline producer; fed into the +// cascade topo-sort (buildLineageDag) so a cold cascade orders the referenced +// dimension first, and rendered dashed on the canvas (like macro edges). +export interface AssetGraphTestEdge { + producer_kind: GraphUsageKind + producer_path: string + runnable_kind: GraphUsageKind + runnable_path: string + asset_kind: AssetKind + asset_path: string +} + export interface AssetGraphResponse { assets: AssetGraphAssetNode[] runnables: AssetGraphRunnableNode[] edges: AssetGraphEdge[] triggers: AssetGraphTrigger[] + macro_edges?: AssetGraphMacroEdge[] + test_edges?: AssetGraphTestEdge[] } export type AssetGraphNodeData = diff --git a/frontend/src/lib/components/assets/workspaceMacros.ts b/frontend/src/lib/components/assets/workspaceMacros.ts new file mode 100644 index 0000000000..d25f9d1bea --- /dev/null +++ b/frontend/src/lib/components/assets/workspaceMacros.ts @@ -0,0 +1,32 @@ +import { AssetService, type ListWorkspaceMacrosResponse } from '$lib/gen' + +export type WorkspaceMacro = ListWorkspaceMacrosResponse[number] + +// Workspace macros are late-bound (the worker reads the registry per job), so +// mild staleness in editor surfaces is harmless — a short TTL keeps repeated +// editor mounts / drawer opens from refetching on every keystroke-driven +// remount while still picking up a lib deploy within seconds. +const TTL_MS = 30_000 +const cache = new Map() + +export async function listWorkspaceMacrosCached(workspace: string): Promise { + const hit = cache.get(workspace) + if (hit && Date.now() - hit.at < TTL_MS) return hit.items + const items = await AssetService.listWorkspaceMacros({ workspace }) + cache.set(workspace, { at: Date.now(), items }) + return items +} + +export function invalidateWorkspaceMacros(workspace: string) { + cache.delete(workspace) +} + +/** `name(params)` display signature, with the table-macro arrow. */ +export function macroSignature(m: WorkspaceMacro): string { + return `${m.name}(${m.params})${m.is_table ? ' → table' : ''}` +} + +/** Full `CREATE` statement for the copy button / documentation preview. */ +export function macroDefinitionSql(m: WorkspaceMacro): string { + return `CREATE OR REPLACE MACRO ${m.name}(${m.params}) AS ${m.is_table ? 'TABLE ' : ''}${m.body};` +} diff --git a/frontend/src/lib/components/common/badge/Badge.svelte b/frontend/src/lib/components/common/badge/Badge.svelte index cf7a7cad66..c57d41029c 100644 --- a/frontend/src/lib/components/common/badge/Badge.svelte +++ b/frontend/src/lib/components/common/badge/Badge.svelte @@ -90,7 +90,7 @@ const hovers: Partial> = { gray: 'hover:bg-surface-hover', - blue: 'hover:bg-blue-200 dark:hover:bg-blue-700/40', + blue: 'hover:bg-blue-100 dark:hover:bg-blue-700/60', red: 'hover:bg-red-200 dark:hover:bg-red-500/25', green: 'hover:bg-green-200 dark:hover:bg-green-500/25', yellow: 'hover:bg-yellow-200 dark:hover:bg-yellow-500/25', diff --git a/frontend/src/lib/components/common/checkbox/Checkbox.svelte b/frontend/src/lib/components/common/checkbox/Checkbox.svelte index 0c0fcef268..49e13e9519 100644 --- a/frontend/src/lib/components/common/checkbox/Checkbox.svelte +++ b/frontend/src/lib/components/common/checkbox/Checkbox.svelte @@ -4,6 +4,9 @@ interface Props { /** Controlled checked state. */ checked?: boolean + /** Tri-state display (e.g. a group header with only part of its items + * selected). Purely visual — `checked` still drives the value. */ + indeterminate?: boolean disabled?: boolean /** Native title attribute (hover hint). */ title?: string | undefined @@ -15,6 +18,7 @@ let { checked = false, + indeterminate = false, disabled = false, title = undefined, class: className = undefined, @@ -25,6 +29,7 @@ - {#if banner} - {@render banner()} - {/if} + {#snippet contentBox(tightTop = false)} +
+ {@render children?.()} +
+ {/snippet} -
- {@render children?.()} -
+ {#if banner} + +
+ {@render banner()} + {@render contentBox(bannerReserved)} +
+ {:else} + {@render contentBox()} + {/if} diff --git a/frontend/src/lib/components/common/table/AppRow.svelte b/frontend/src/lib/components/common/table/AppRow.svelte index f016fafd5c..4f2c760648 100644 --- a/frontend/src/lib/components/common/table/AppRow.svelte +++ b/frontend/src/lib/components/common/table/AppRow.svelte @@ -6,7 +6,7 @@ import DraftBadge from '$lib/components/DraftBadge.svelte' import type ShareModal from '$lib/components/ShareModal.svelte' import { AppService, type ListableApp } from '$lib/gen' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { createEventDispatcher } from 'svelte' import Button from '../button/Button.svelte' @@ -34,8 +34,7 @@ import AppDeploymentHistory from '$lib/components/apps/editor/AppDeploymentHistory.svelte' import { isDeployable } from '$lib/utils_deployable' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' - import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' - import { buildForkEditUrl } from '$lib/utils/editInFork' + import { buildForkEditUrl, editInForkAllowed, editInForkLabel } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' interface Props { @@ -170,7 +169,7 @@ {/if} - {#if !isCloudHosted() && !isRuleActive('DisableWorkspaceForking') && (!showEditButton || !app.canWrite)} + {#if !isCloudHosted() && editInForkAllowed($workspaceStore, $userWorkspaces) && (!showEditButton || !app.canWrite)}
{/if} @@ -235,10 +234,13 @@ hide: $userStore?.operator }, { - displayName: 'Edit in workspace fork', + displayName: editInForkLabel($workspaceStore, $userWorkspaces), icon: GitFork, href: buildForkEditUrl(app.raw_app ? 'raw_app' : 'app', path), - hide: $userStore?.operator || isCloudHosted() || isRuleActive('DisableWorkspaceForking') + hide: + $userStore?.operator || + isCloudHosted() || + !editInForkAllowed($workspaceStore, $userWorkspaces) }, { displayName: 'Move/Rename', diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index 89a08b412b..a19715686f 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -8,7 +8,7 @@ import DraftBadge from '$lib/components/DraftBadge.svelte' import type ShareModal from '$lib/components/ShareModal.svelte' import { FlowService, type Flow } from '$lib/gen' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { createEventDispatcher } from 'svelte' import Badge from '../badge/Badge.svelte' @@ -36,8 +36,7 @@ import FlowHistory from '$lib/components/flows/FlowHistory.svelte' import InheritedLabels from '$lib/components/InheritedLabels.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' - import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' - import { buildForkEditUrl } from '$lib/utils/editInFork' + import { buildForkEditUrl, editInForkAllowed, editInForkLabel } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' interface Props { @@ -191,7 +190,7 @@ {/if} - {#if !isCloudHosted() && !isRuleActive('DisableWorkspaceForking') && (!showEditButton || !flow.canWrite)} + {#if !isCloudHosted() && editInForkAllowed($workspaceStore, $userWorkspaces) && (!showEditButton || !flow.canWrite)}
{/if} @@ -252,10 +251,13 @@ hide: $userStore?.operator }, { - displayName: 'Edit in workspace fork', + displayName: editInForkLabel($workspaceStore, $userWorkspaces), icon: GitFork, href: buildForkEditUrl('flow', path), - hide: $userStore?.operator || isCloudHosted() || isRuleActive('DisableWorkspaceForking') + hide: + $userStore?.operator || + isCloudHosted() || + !editInForkAllowed($workspaceStore, $userWorkspaces) }, { displayName: 'Audit logs', diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 1d87d4c8d5..507986294f 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -9,7 +9,7 @@ import type ShareModal from '$lib/components/ShareModal.svelte' import { ScriptService, type Script } from '$lib/gen' - import { hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' + import { hubBaseUrlStore, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { createEventDispatcher } from 'svelte' @@ -48,8 +48,7 @@ import Tooltip from '$lib/components/Tooltip.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' import { scriptToHubUrl } from '$lib/hub' - import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' - import { buildForkEditUrl } from '$lib/utils/editInFork' + import { buildForkEditUrl, editInForkAllowed, editInForkLabel } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' interface Props { @@ -251,7 +250,7 @@ {/if} {/if} - {#if !isCloudHosted() && !isRuleActive('DisableWorkspaceForking') && (!showEditButton || !script.canWrite)} + {#if !isCloudHosted() && editInForkAllowed($workspaceStore, $userWorkspaces) && (!showEditButton || !script.canWrite)}
{/if} @@ -334,10 +333,13 @@ hide: $userStore?.operator }, { - displayName: 'Edit in workspace fork', + displayName: editInForkLabel($workspaceStore, $userWorkspaces), icon: GitFork, href: buildForkEditUrl('script', script.path), - hide: $userStore?.operator || isCloudHosted() || isRuleActive('DisableWorkspaceForking') + hide: + $userStore?.operator || + isCloudHosted() || + !editInForkAllowed($workspaceStore, $userWorkspaces) }, { displayName: 'Move/Rename', diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index fdba95ca7f..c67bcd17b5 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -44,6 +44,9 @@ import { buildSummaryMessageContent } from './compactionPrompt' import { dfs } from '$lib/components/flows/previousResults' +import { SvelteSet } from 'svelte/reactivity' +import type { UserDraftItemKind } from '$lib/gen' +import { maskKey } from '$lib/components/sessions/modifiedItemsMask' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' import { chatDraft, expanded } from './chatDraft' @@ -67,6 +70,7 @@ import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop' +import { sanitizeToolCallArguments } from './toolCallArguments' import { normalizeContextUsage } from './tokenUsage' import type { ReviewChangesOpts } from './monaco-adapter' import { @@ -88,6 +92,11 @@ import { type GlobalToolHelpers } from './global/core' import { isGlobalAiEnabled } from './global/gate' +import { + pipelineTools, + getPipelinePromptSection, + type PipelineAIChatHelpers +} from './pipeline/core' import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userScopedStorage' import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { AttachedFilesStore } from './files/attachedFiles.svelte' @@ -254,6 +263,7 @@ export class AIChatManager { skipResponsesApi = false mode = $state(AIMode.NAVIGATOR) + pipelineAiChatHelpers = $state(undefined) readonly isOpen = $derived(chatState.size > 0) savedSize = $state(0) instructions = $state('') @@ -336,6 +346,75 @@ export class AIChatManager { // session rather than the UI-active one — keeps backgrounded sessions isolated. sessionId: string | undefined = undefined + // Fired whenever the active chat id changes away from the one the consumer + // knows (a "/clear" rotation or a history switch). Session runtimes wire this + // to keep the session record's chatId aligned — the compare-page handoff + // (`from_session`) reads it, and a stale id would preselect the previous + // chat's items. Set here (not imported) to avoid a copilot→sessions cycle. + onChatRotated: ((chatId: string) => void) | undefined = undefined + + // Workspace items the CURRENT chat modified via AI tool calls, as + // `${UserDraftItemKind}:${storagePath}` keys (see modifiedItemsMask.ts). + // undefined = untracked: the global side-panel chat (never initialised) and + // loaded legacy chats with no stored mask, both of which fall back to the + // show-all bar. A SvelteSet (even empty) = tracked. Reactive so the session + // bar updates as tools record mid-turn. + modifiedItems = $state | undefined>(undefined) + + // Start tracking for a brand-new session chat (empty = "tracked, nothing yet"). + initModifiedItemsTracking() { + this.modifiedItems = new SvelteSet() + } + + // Record an item an AI tool call created/edited/deleted. No-op when untracked + // (the global singleton never initialises the set), so it stays unaffected. + recordModifiedItem(itemKind: UserDraftItemKind, storagePath: string) { + this.modifiedItems?.add(maskKey(itemKind, storagePath)) + } + + // Un-record an item whose chat-made change was discarded — without this the + // still-existing deployed item would keep reading as this chat's "Deployed" + // edit. Persisted immediately: unlike recordModifiedItem (whose persistence + // rides on the turn's saveChat), a discard can fire from the review dock + // outside any turn, and waiting would resurrect the entry on reload. + async removeModifiedItem(itemKind: UserDraftItemKind, storagePath: string) { + if (!this.modifiedItems?.delete(maskKey(itemKind, storagePath))) return + await this.#persistModifiedItems() + } + + // Move a mask entry to the path a draft actually deployed to. A draft-only + // flow/app parks at a synthetic `draft_{uuid}` storage path and deploys to + // its chosen path — without the move, the existence check at the synthetic + // path fails after reload and the deployed row vanishes from the dock. + async renameModifiedItem(itemKind: UserDraftItemKind, fromPath: string, toPath: string) { + if (fromPath === toPath) return + if (!this.modifiedItems?.delete(maskKey(itemKind, fromPath))) return + this.modifiedItems.add(maskKey(itemKind, toPath)) + await this.#persistModifiedItems() + } + + // Serialized, snapshot-at-write-time persistence: two rapid dock actions + // would otherwise race their saveChat writes, and the earlier (staler) + // snapshot could land last — dropping the later mutation until the next + // turn-end save. + #maskPersistQueue: Promise = Promise.resolve() + #persistModifiedItems(): Promise { + this.#maskPersistQueue = this.#maskPersistQueue.then(() => + this.historyManager + .saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) + // Swallow (and log) a failed write so it can't wedge the queue as a + // rejected link — the next persist snapshots the full current set, so + // a lost write self-heals on the next mutation or turn-end save. + .catch((e) => console.error('Failed to persist modified-items mask', e)) + ) + return this.#maskPersistQueue + } + // Workspace AI skills (name + description) advertised in the GLOBAL system // prompt and surfaced as slash commands in session chat. Loaded // asynchronously when entering GLOBAL mode; the system message is rebuilt @@ -485,7 +564,10 @@ export class AIChatManager { this.compacting = true try { const raw = await getNonStreamingCompletion( - [...prefix, { role: 'user', content: getCompactionSummaryPrompt() }], + [ + ...sanitizeToolCallArguments(prefix), + { role: 'user', content: getCompactionSummaryPrompt() } + ], abortController ) const formatted = formatCompactSummary(raw ?? '') @@ -652,15 +734,13 @@ export class AIChatManager { await this.historyManager.saveChat( this.displayMessages, this.messages, - this.contextUsage + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined ) sendUserToast('Conversation compacted.') break case 'empty': - sendUserToast( - 'Compaction produced an empty summary — conversation left unchanged.', - true - ) + sendUserToast('Compaction produced an empty summary — conversation left unchanged.', true) break case 'error': sendUserToast('Failed to compact the conversation.', true) @@ -958,28 +1038,7 @@ export class AIChatManager { this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] this.helpers = {} } else if (mode === AIMode.GLOBAL) { - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(mode), { - previewTools: this.isSessionChat, - skills: this.globalSkills - }) - this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) - this.helpers = { - ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), - testActiveFlow: async (args?: Record) => - this.flowAiChatHelpers?.testFlow(args), - attachedFiles: this.attachedFiles, - getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', - setUserInstructions: (instructions: string) => { - const prompts = getUserCustomPrompts() - if (instructions.trim()) { - prompts[AIMode.GLOBAL] = instructions - } else { - delete prompts[AIMode.GLOBAL] - } - setUserCustomPrompts(prompts) - this.rebuildGlobalSystemMessage() - } - } satisfies GlobalToolHelpers + this.configureGlobalMode() void this.refreshGlobalSkills() } else if (mode === AIMode.APP) { const customPrompt = getCombinedCustomPrompt(mode) @@ -992,6 +1051,43 @@ export class AIChatManager { // Fetch the workspace's AI skills and, if GLOBAL mode is still active, rebuild // the system message so the next chat-loop iteration advertises them. Ignore // stale resolves so workspace changes cannot overwrite newer skills. + // Build the global-mode system message, tools, and helpers, layering on the + // pipeline surface when a /pipeline editor has registered helpers. Centralized + // so changeMode, refreshGlobalSkills, and setPipelineHelpers stay consistent — + // each rebuild would otherwise drop the pipeline augmentation the others added. + private configureGlobalMode = () => { + const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { + previewTools: this.isSessionChat, + skills: this.globalSkills + }) + const baseHelpers: GlobalToolHelpers = { + ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), + testActiveFlow: async (args?: Record) => this.flowAiChatHelpers?.testFlow(args), + attachedFiles: this.attachedFiles, + getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', + setUserInstructions: (instructions: string) => { + const prompts = getUserCustomPrompts() + if (instructions.trim()) { + prompts[AIMode.GLOBAL] = instructions + } else { + delete prompts[AIMode.GLOBAL] + } + setUserCustomPrompts(prompts) + this.rebuildGlobalSystemMessage() + } + } + const pipeline = this.pipelineAiChatHelpers + if (pipeline) { + systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) + this.tools = [...globalToolsFor({ sessionPreview: this.isSessionChat }), ...pipelineTools] + this.helpers = { ...baseHelpers, pipeline } + } else { + this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) + this.helpers = baseHelpers + } + this.systemMessage = systemMessage + } + refreshGlobalSkills = async (workspace = get(workspaceStore) ?? '') => { const refreshId = ++this.globalSkillsRefreshId const skills = await loadWorkspaceSkills(workspace) @@ -1000,10 +1096,7 @@ export class AIChatManager { } this.globalSkills = skills if (this.mode === AIMode.GLOBAL) { - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { - previewTools: this.isSessionChat, - skills - }) + this.configureGlobalMode() } } @@ -1014,10 +1107,18 @@ export class AIChatManager { if (this.mode !== AIMode.GLOBAL) { return } - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { + const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { previewTools: this.isSessionChat, skills: this.globalSkills }) + // Preserve the active pipeline-editor augmentation that configureGlobalMode + // adds — otherwise update_user_instructions (which calls this) would drop the + // /pipeline/ context + direct-draft/materialize guidance mid-session. + const pipeline = this.pipelineAiChatHelpers + if (pipeline) { + systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) + } + this.systemMessage = systemMessage } private expandGlobalSkillCommand = (instructions: string): string => { @@ -1612,7 +1713,12 @@ export class AIChatManager { const projectedContextTokens = this.contextTokens + this.estimateMessagesTokens([userMessage]) this.messages.push(userMessage) - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) this.currentReply = '' this.currentReasoning = '' @@ -1648,7 +1754,12 @@ export class AIChatManager { this.contextUsage = Math.max(0, this.contextUsage - freed) } } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } } // Rollback anchors for restoreUnsentTurn: captured after compaction so @@ -1734,7 +1845,10 @@ export class AIChatManager { }, requestConfirmation: this.requestConfirmation, shouldAutoAcceptToolConfirmations: () => this.autoAcceptToolConfirmationsActive, - requestUserQuestion: this.requestUserQuestion + requestUserQuestion: this.requestUserQuestion, + onItemModified: (kind, path) => this.recordModifiedItem(kind, path), + onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), + onItemDiscarded: (kind, path) => void this.removeModifiedItem(kind, path) } } @@ -1770,7 +1884,12 @@ export class AIChatManager { this.contextUsage = result?.lastIterationUsage ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion : undefined - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) // Still counts as the saved first turn — skipping the hook here would // permanently miss it (the next turn isn't "first" anymore). if (isFirstUserTurn && this.afterFirstTurnSaved) { @@ -1800,7 +1919,12 @@ export class AIChatManager { // user message on reload. Remove it instead. this.historyManager.deletePastChat(this.historyManager.getCurrentChatId()) } else { - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } if (!wasAborted) { sendUserToast('The model returned no response — your message was restored to the input.') @@ -1819,7 +1943,12 @@ export class AIChatManager { if (this.autoAcceptEditsActive) { this.acceptPendingFlowEdits() } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) // Only this branch is a clean send: the queued-message flush below // auto-sends the next message after it (set after saveChat so a // persistence failure falls through to the restore path instead). @@ -1843,7 +1972,12 @@ export class AIChatManager { // compaction on the next send instead of failing the same way again. this.contextUsage = undefined try { - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } catch (saveErr) { console.error('Failed to persist partial chat after error', saveErr) } @@ -1972,15 +2106,25 @@ export class AIChatManager { // Drop any message queued in this conversation so it can't auto-send into // the fresh chat or linger as a card across the switch. this.queuedMessage = '' - await this.historyManager.save(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.save( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) this.displayMessages = [] this.messages = [] this.contextUsage = undefined + // The mask belongs to the conversation just saved — the fresh chat starts + // its own (empty) tracking; carrying entries over would claim the previous + // conversation's edits for the new one. Untracked chats stay untracked. + if (this.modifiedItems) this.modifiedItems = new SvelteSet() // In an AI session, linked files are session-scoped: they persist across conversations // (cleared only when the session is deleted). The ephemeral global side-panel chat has no // session, so "New chat" must clear them — otherwise the next, unrelated conversation // would still get the previous file roster and could read/search it. if (!this.isSessionChat) this.attachedFiles.clear() + this.onChatRotated?.(this.historyManager.getCurrentChatId()) } loadPastChat = async (id: string) => { @@ -1995,7 +2139,16 @@ export class AIChatManager { this.displayMessages = chat.displayMessages this.messages = chat.actualMessages this.contextUsage = normalizeContextUsage(chat.contextUsage) + // Seed the modified-items mask from the stored chat. A stored array + // (even empty) → tracked; a legacy chat with no field stays untracked + // (undefined) so the session bar falls back to showing all drafts. The + // global side-panel chat never tracks, so leave it untouched there. + if (this.isSessionChat) { + const stored = this.historyManager.getModifiedItems(id) + this.modifiedItems = stored !== undefined ? new SvelteSet(stored) : undefined + } this.#automaticScroll = true + this.onChatRotated?.(id) } } @@ -2132,7 +2285,7 @@ export class AIChatManager { moduleState && !moduleState.previewSuccess ? getStringError(moduleState.previewResult) : undefined, - getCode: () => module.value.type === 'rawscript' ? module.value.content : '', + getCode: () => (module.value.type === 'rawscript' ? module.value.content : ''), lang: module.value.language, path: module.id, ...editorRelated @@ -2176,6 +2329,28 @@ export class AIChatManager { } } + // Registered by the /pipeline editor while it is mounted. Rebuilds the global + // tool set so the pipeline tools appear (and disappear on unregister). Pipeline + // AI edits apply directly as drafts, so there is nothing to auto-accept. + // Returns a cleanup that tears the registration back down. + setPipelineHelpers = (pipelineHelpers: PipelineAIChatHelpers) => { + this.pipelineAiChatHelpers = pipelineHelpers + untrack(() => { + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + }) + + return () => { + this.pipelineAiChatHelpers = undefined + untrack(() => { + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + }) + } + } + /** * Refresh cached datatables from the app helpers (async) * Creates one context element per table (not per datatable) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index d23cbf96a1..a415377a7b 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -693,7 +693,9 @@ describe('AIChatManager context compaction', () => { expect(manager.messages[0]).toMatchObject({ role: 'user', content: 'c'.repeat(400) }) // Mid-turn, the report is debited by the freed estimate (visible in the // compaction-time save) so a rolled-back turn keeps a consistent value - expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000) + // 4th arg: the modified-items mask rides on every save (undefined here — + // this bare manager never initialised tracking). + expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000, undefined) // At commit, the no-report turn clears the stored value; the readable // number falls back to estimating the now-tiny compacted history expect(manager.contextUsage).toBeUndefined() diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 9173e4c441..0db0751293 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -25,6 +25,12 @@ interface ChatSchema extends IDBSchema { // New writes store the plain reported token count; chats persisted by // earlier versions may still hold the legacy anchor object. contextUsage?: PersistedContextUsage + // Workspace items this chat modified via AI tool calls, as + // `${UserDraftItemKind}:${storagePath}` keys. Persisted out-of-band from + // the message arrays so it survives compaction. Absent (undefined) on + // chats predating this feature → consumers fall back to showing all + // workspace drafts; a defined array (even empty) means "tracked". + modifiedItems?: string[] } } } @@ -80,6 +86,29 @@ export function __resetLegacyChatClaimForTesting(): void { legacyChatClaim = undefined } +// Read a chat's modified-items mask by chatId WITHOUT mounting an AIChatManager, +// for the standalone /forks/compare route. Returns undefined for a legacy chat +// (no field) so the page falls back to selecting all items; a defined array +// (even empty) narrows the preselection. Opens a throwaway user-scoped handle; +// the `get` is O(1) on the `id` keyPath. +export async function readChatModifiedItems(chatId: string): Promise { + const dbh = userScopedDb(DB_NAME, { + version: 1, + upgrade: createChatStore, + migrate: migrateLegacyChatDb + }) + try { + const db = await dbh.whenReady() + const chat = await db?.get('chats', chatId) + return chat?.modifiedItems + } catch (err) { + console.error('Could not read chat modified items', err) + return undefined + } finally { + dbh.close() + } +} + export default class HistoryManager { // Per-instance handle to the shared per-user DB lifecycle. There is one // HistoryManager per AIChatManager (the singleton + one per session runtime), @@ -100,6 +129,7 @@ export default class HistoryManager { lastModified: number sessionId?: string contextUsage?: PersistedContextUsage + modifiedItems?: string[] } > = $state({}) @@ -173,10 +203,15 @@ export default class HistoryManager { return Object.values(this.savedChats) } + getModifiedItems(id: string): string[] | undefined { + return this.savedChats[id]?.modifiedItems + } + async saveChat( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], - contextUsage?: number + contextUsage?: number, + modifiedItems?: string[] ) { if (displayMessages.length > 0) { // Compaction replaces the original first message with a summary boundary. @@ -203,7 +238,18 @@ export default class HistoryManager { id: this.currentChatId, lastModified: Date.now(), ...(this.sessionId ? { sessionId: this.sessionId } : {}), - ...(contextUsage !== undefined ? { contextUsage } : {}) + ...(contextUsage !== undefined ? { contextUsage } : {}), + // Only persist when the caller passes a defined array. Loaded legacy + // chats keep their accumulator undefined, so we never retroactively + // stamp them with [] (which would flip them to the filtered view). + // But since `put` replaces the whole record, a caller that omits the + // argument must not ERASE a tracked chat's stored mask — fall back to + // the previously saved field. + ...(modifiedItems !== undefined + ? { modifiedItems } + : this.savedChats[this.currentChatId]?.modifiedItems !== undefined + ? { modifiedItems: this.savedChats[this.currentChatId].modifiedItems } + : {}) } this.savedChats = { ...this.savedChats, @@ -218,9 +264,10 @@ export default class HistoryManager { async save( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], - contextUsage?: number + contextUsage?: number, + modifiedItems?: string[] ) { - await this.saveChat(displayMessages, messages, contextUsage) + await this.saveChat(displayMessages, messages, contextUsage, modifiedItems) this.currentChatId = createLongHash() } diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index 38fb7ecf0d..540dfaf72e 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -141,3 +141,30 @@ describe('HistoryManager title across compaction', () => { expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('original first question') }) }) + +describe('HistoryManager modified-items mask persistence', () => { + const msgs = [{ role: 'user', content: 'hello', index: 0 }] as DisplayMessage[] + + it('a save without the argument preserves a previously stored mask', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[], undefined, ['script:u/a/x']) + expect(hm.getModifiedItems(id)).toEqual(['script:u/a/x']) + + // e.g. manual compaction re-saving the transcript: the whole record is + // rewritten, but the tracked mask must survive. + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[]) + expect(hm.getModifiedItems(id)).toEqual(['script:u/a/x']) + }) + + it('never retroactively stamps an untracked chat', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[]) + expect(hm.getModifiedItems(id)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.test.ts b/frontend/src/lib/components/copilot/chat/anthropic.test.ts new file mode 100644 index 0000000000..9c9a4d0bb0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/anthropic.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import { convertOpenAIToAnthropicMessages } from './anthropic' + +// anthropic.ts pulls in the chat client/registry layer at import time; the +// converter under test is pure, so stub those side-effecting modules away. +vi.mock('../lib', () => ({ + getProviderAndCompletionConfig: vi.fn(), + workspaceAIClients: {} +})) + +vi.mock('../reasoningRegistry', () => ({ + applyReasoningToConfig: vi.fn() +})) + +vi.mock('./shared', () => ({ + processToolCall: vi.fn() +})) + +describe('convertOpenAIToAnthropicMessages', () => { + it('replays a captured assistant turn verbatim, skips the standalone text copy, and leaves the turn untouched', () => { + const anthropicContent = [ + { type: 'thinking', thinking: 'first', signature: 'sig-1' }, + { + type: 'server_tool_use', + id: 'srv_1', + name: 'web_search', + input: { query: 'nist password length' } + }, + { + type: 'web_search_tool_result', + tool_use_id: 'srv_1', + content: [{ type: 'web_search_result', title: 'NIST', url: 'https://nist.gov' }] + }, + { type: 'thinking', thinking: 'second', signature: 'sig-2' }, + { type: 'tool_use', id: 'tool_1', name: 'list_resources', input: {} } + ] + // Snapshot to assert the stored content is never mutated by the converter. + const anthropicContentSnapshot = JSON.parse(JSON.stringify(anthropicContent)) + + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'find the nist length then list resources' }, + // Standalone text the streamer emits before the tool-call message. + { role: 'assistant', content: 'Let me search the web.' }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool_1', + type: 'function', + function: { name: 'list_resources', arguments: '{}' } + } + ], + _anthropicContent: anthropicContent + } as any, + { role: 'tool', tool_call_id: 'tool_1', content: 'resource A, resource B' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + // user + the verbatim assistant turn + the tool result; the standalone text is dropped. + expect(out).toHaveLength(3) + expect(out[0]).toEqual({ role: 'user', content: 'find the nist length then list resources' }) + + // Assistant turn replayed in original order, no reordering or dropped blocks. + expect(out[1].role).toBe('assistant') + expect((out[1].content as any[]).map((b) => b.type)).toEqual([ + 'thinking', + 'server_tool_use', + 'web_search_tool_result', + 'thinking', + 'tool_use' + ]) + // The verbatim turn must stay byte-identical — no cache_control injected into it, + // or a thinking-block signature would no longer validate. + expect(out[1].content).toEqual(anthropicContentSnapshot) + expect((out[1].content as any[]).some((b) => 'cache_control' in b)).toBe(false) + expect(anthropicContent).toEqual(anthropicContentSnapshot) + + // The cache breakpoint lands on the trailing tool result, not the assistant turn. + expect(out[2]).toEqual({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tool_1', + content: 'resource A, resource B', + cache_control: { type: 'ephemeral' } + } + ] + }) + }) + + it('converts a plain text assistant turn (no tools) and caches the trailing text block', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: 'hi there' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(2) + expect(out[0]).toEqual({ role: 'user', content: 'hello' }) + expect(out[1].role).toBe('assistant') + expect(out[1].content).toEqual([ + { type: 'text', text: 'hi there', cache_control: { type: 'ephemeral' } } + ]) + }) + + it('falls back to _anthropicThinkingBlocks for turns persisted before _anthropicContent', () => { + const thinkingBlocks = [{ type: 'thinking', thinking: 'reasoning', signature: 'sig-old' }] + + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'do a thing' }, + // The standalone text persisted alongside an old-style turn must NOT be skipped + // (the fallback reconstruction relies on it for the assistant text). + { role: 'assistant', content: 'working on it' }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool_old', + type: 'function', + function: { name: 'list_resources', arguments: '{}' } + } + ], + _anthropicThinkingBlocks: thinkingBlocks + } as any + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(3) + expect(out[1]).toEqual({ role: 'assistant', content: 'working on it' }) + const content = out[2].content as any[] + // Thinking block re-injected first, then the tool_use. + expect(content.map((b) => b.type)).toEqual(['thinking', 'tool_use']) + expect(content[0]).toEqual(thinkingBlocks[0]) + expect(content[1]).toMatchObject({ type: 'tool_use', id: 'tool_old', name: 'list_resources' }) + }) + + it('caches a trailing tool result even when the prior turn used no captured content', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'q' }, + { + role: 'assistant', + tool_calls: [ + { id: 't1', type: 'function', function: { name: 'list_resources', arguments: '{}' } } + ] + } as any, + { role: 'tool', tool_call_id: 't1', content: 'done' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + const last = out[out.length - 1] + expect(last.role).toBe('user') + expect((last.content as any[])[0]).toMatchObject({ + type: 'tool_result', + tool_use_id: 't1', + cache_control: { type: 'ephemeral' } + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 48b068039f..b6e7e57a17 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -286,15 +286,15 @@ export async function parseAnthropicCompletion( role: 'assistant', tool_calls: toolCallsToProcess } - // Preserve thinking blocks (with signatures) so the next request keeps the - // reasoning chain — Anthropic requires this when thinking is combined with tool - // use. They are re-injected by convertOpenAIToAnthropicMessages. - const thinkingBlocks = finalMessage.content.filter( - (b) => b.type === 'thinking' || b.type === 'redacted_thinking' - ) - if (thinkingBlocks.length > 0) { - ;(assistantWithTools as any)._anthropicThinkingBlocks = thinkingBlocks - } + // Preserve the assistant turn verbatim (thinking/redacted_thinking with their + // signatures, server_tool_use + web_search_tool_result, text and tool_use) in + // original order. Anthropic binds each thinking block's signature to the blocks + // that precede it in the latest assistant message, so when this turn is replayed + // to continue past its tool call it must be byte-identical: reordering thinking to + // the front or dropping the web-search blocks invalidates a later block's + // signature and the request 400s with "thinking blocks ... cannot be modified". + // convertOpenAIToAnthropicMessages replays this content as-is. + ;(assistantWithTools as any)._anthropicContent = finalMessage.content messages.push(assistantWithTools) addedMessages.push(assistantWithTools) @@ -323,7 +323,33 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage let system: TextBlockParam[] | undefined const anthropicMessages: MessageParam[] = [] - for (const message of messages) { + // A streamed assistant turn that ends in tool calls is persisted as one or more + // standalone text messages followed by the tool-call message carrying + // _anthropicContent. That text is already inside _anthropicContent (replayed verbatim + // below), so drop the standalone copies — otherwise the text is duplicated and + // emitted ahead of the turn's thinking blocks. The scan stops at the preceding + // user/tool message, so only the current turn's own text is skipped. + const skipStandaloneText = new Set() + for (let i = 0; i < messages.length; i++) { + if (!(messages[i] as any)._anthropicContent) continue + for (let j = i - 1; j >= 0; j--) { + const m = messages[j] + if ( + m.role === 'assistant' && + typeof m.content === 'string' && + !m.tool_calls && + !(m as any)._anthropicContent + ) { + skipStandaloneText.add(j) + } else { + break + } + } + } + + for (let i = 0; i < messages.length; i++) { + const message = messages[i] + if (skipStandaloneText.has(i)) continue if (message.role === 'system') { const systemText = typeof message.content === 'string' ? message.content : JSON.stringify(message.content) @@ -345,10 +371,19 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage typeof message.content === 'string' ? message.content : JSON.stringify(message.content) }) } else if (message.role === 'assistant') { + // Replay a captured assistant turn verbatim so its thinking-block signatures + // stay valid (see the _anthropicContent note where the streamed turn is stored). + const anthropicContent = (message as any)._anthropicContent + if (Array.isArray(anthropicContent) && anthropicContent.length > 0) { + anthropicMessages.push({ role: 'assistant', content: anthropicContent as any }) + continue + } + const content: any[] = [] - // Re-inject preserved thinking blocks first (Anthropic requires thinking to - // precede tool_use in the same assistant turn when thinking is enabled). + // Fallback for sessions persisted before _anthropicContent existed: re-inject + // the preserved thinking blocks first (Anthropic requires thinking to precede + // tool_use in the same assistant turn when thinking is enabled). const thinkingBlocks = (message as any)._anthropicThinkingBlocks if (Array.isArray(thinkingBlocks) && thinkingBlocks.length > 0) { content.push(...thinkingBlocks) @@ -404,26 +439,29 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage } } - // Add cache_control to the last message content blocks + // Cache the conversation prefix: put an ephemeral breakpoint on the last content + // block of the last message. Each continuation only appends a tool result plus the + // next turn, so everything up to here is read from cache — which is what keeps + // replaying assistant turns verbatim (web-search results included) affordable. + // cache_control is valid on text/tool_use/tool_result blocks, but a thinking or + // redacted_thinking block must never be modified, so skip the breakpoint there. if (anthropicMessages.length > 0) { const lastMessage = anthropicMessages[anthropicMessages.length - 1] - if (Array.isArray(lastMessage.content)) { - // Add cache_control to the last content block - if (lastMessage.content.length > 0) { - const lastBlock = lastMessage.content[lastMessage.content.length - 1] - if (lastBlock.type === 'text') { - lastBlock.cache_control = { type: 'ephemeral' } - } - } - } else if (typeof lastMessage.content === 'string') { - // Convert string content to array format with cache_control + if (typeof lastMessage.content === 'string') { lastMessage.content = [ - { - type: 'text', - text: lastMessage.content, - cache_control: { type: 'ephemeral' } - } + { type: 'text', text: lastMessage.content, cache_control: { type: 'ephemeral' } } ] + } else if (Array.isArray(lastMessage.content) && lastMessage.content.length > 0) { + const lastIndex = lastMessage.content.length - 1 + const lastBlock = lastMessage.content[lastIndex] as any + if (lastBlock.type !== 'thinking' && lastBlock.type !== 'redacted_thinking') { + // Clone the block instead of mutating in place: the array may be a verbatim + // _anthropicContent turn that must stay unaltered for later requests. + lastMessage.content = [ + ...lastMessage.content.slice(0, lastIndex), + { ...lastBlock, cache_control: { type: 'ephemeral' } } + ] + } } } diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 9c1863ae54..93acdfbd86 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -399,3 +399,42 @@ describe('truncateToToolPairedPrefix', () => { expect(truncateToToolPairedPrefix(msgs)).toEqual(msgs) }) }) + +describe('runChatLoop history sanitization', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.providerSupportsWebSearch.mockReturnValue(false) + mocks.resolveRequestReasoning.mockReturnValue(undefined) + mocks.getOpenAIResponsesCompletion.mockResolvedValue({}) + mocks.parseOpenAIResponsesCompletion.mockResolvedValue({ + shouldContinue: false, + tokenUsage + }) + }) + + it('replaces unparseable historical tool_call arguments before sending, without mutating the stored history', async () => { + const config = createConfig({ workspace: `workspace-${randomUUID()}` }) + const poisoned: ChatCompletionMessageParam = { + role: 'assistant', + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { name: 'patch_app_file', arguments: '{"path": "u/x", "old_string": "trunc' } + } + ] + } + config.messages.push(poisoned, { + role: 'tool', + tool_call_id: 'call_1', + content: 'Error while calling tool' + }) + + await runChatLoop(config) + + const sent = mocks.getOpenAIResponsesCompletion.mock.calls[0][0] as any[] + const sentAssistant = sent.find((m) => m.role === 'assistant' && m.tool_calls) + expect(sentAssistant.tool_calls[0].function.arguments).toBe('{}') + expect((poisoned as any).tool_calls[0].function.arguments).toContain('trunc') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index 0e5e4923d7..edf5be4d07 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -8,8 +8,10 @@ import type { import { getCompletion, parseOpenAICompletion, providerSupportsWebSearch } from '../lib' import { resolveRequestReasoning, type ReasoningProviderModel } from '../reasoningRegistry' import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' +import { usesAnthropicMessagesApi } from '../modelConfig' import { getOpenAIResponsesCompletion, parseOpenAIResponsesCompletion } from './openai-responses' import type { Tool, ToolCallbacks } from './shared' +import { sanitizeToolCallArguments } from './toolCallArguments' import { addChatTokenUsage, emptyChatTokenUsage, type ChatTokenUsage } from './tokenUsage' export interface ChatClients { @@ -257,7 +259,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise t.def) diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index 0309aeaf91..c7ede4c0b5 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.716.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.716.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts index 21e6735ab1..3e0b8fcf69 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts @@ -1,6 +1,6 @@ import { z } from "zod" -export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -20,7 +20,7 @@ export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "i }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task"), "type": z.literal("aiagent"), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type") -export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => diff --git a/frontend/src/lib/components/copilot/chat/flow/providerKind.test.ts b/frontend/src/lib/components/copilot/chat/flow/providerKind.test.ts new file mode 100644 index 0000000000..ab520762ec --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/flow/providerKind.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest' +import { flowModulesSchema } from './openFlowZod.gen' + +// Guards against the generated copilot flow Zod schema (openFlowZod.gen.ts) +// drifting from the AIProviderKind enum in openflow.openapi.yaml. A missing +// provider kind here silently rejects AI-generated flow edits for that provider +// in the copilot flow-editing path (validateFlowModules -> flowModulesSchema). +function aiAgentModuleWithProviderKind(kind: string) { + return { + id: 'agent', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + provider: { + type: 'static', + value: { kind, resource: '$res:u/admin/foundry', model: 'gpt-4o' } + }, + user_message: { type: 'static', value: 'hello' }, + output_type: { type: 'static', value: 'text' } + } + } + } +} + +describe('copilot flow module validation - AI agent provider kind', () => { + it('accepts azure_foundry (and the existing azure_openai baseline)', () => { + expect( + flowModulesSchema.safeParse([aiAgentModuleWithProviderKind('azure_openai')]).success + ).toBe(true) + expect( + flowModulesSchema.safeParse([aiAgentModuleWithProviderKind('azure_foundry')]).success + ).toBe(true) + }) + + it('still rejects an unknown provider kind (enum is actually enforced)', () => { + expect( + flowModulesSchema.safeParse([aiAgentModuleWithProviderKind('not_a_real_provider')]).success + ).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index a3b9c22499..0f30076bda 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -36,7 +36,7 @@ const { backendDrafts, serverTimestamps, failingWrites, failingReads } = vi.hois // concurrent writer advancing the row; otherwise empty, so the conflict // branch in `updateDraft` stays inert for every pre-existing test. serverTimestamps: new Map(), - // Keys whose `updateDraft` / `getDraftForUser` throw a non-404 (network/5xx); + // Keys whose `updateDraft` / draft reads throw a non-404 (network/5xx); // only set by the error-handling tests, empty otherwise. failingWrites: new Set(), failingReads: new Set() @@ -132,13 +132,17 @@ vi.mock('$lib/gen', async () => { existsSchedule: vi.fn(async () => false), getSchedule: vi.fn(async () => { throw new Error('getSchedule mock not configured') - }) + }), + createSchedule: vi.fn(async () => 'created'), + updateSchedule: vi.fn(async () => 'updated') }), HttpTriggerService: wrapService(actual.HttpTriggerService, { existsHttpTrigger: vi.fn(async () => false), getHttpTrigger: vi.fn(async () => { throw new Error('getHttpTrigger mock not configured') - }) + }), + createHttpTrigger: vi.fn(async () => 'created'), + updateHttpTrigger: vi.fn(async () => 'updated') }), AppService: wrapService(actual.AppService, { existsApp: vi.fn(async () => false), @@ -156,7 +160,9 @@ vi.mock('$lib/gen', async () => { existsResource: vi.fn(async () => false), getResource: vi.fn(async () => { throw new Error('getResource mock not configured') - }) + }), + createResource: vi.fn(async () => 'created'), + updateResource: vi.fn(async () => 'updated') }), VariableService: wrapService(actual.VariableService, { existsVariable: vi.fn(async () => false), @@ -190,14 +196,27 @@ vi.mock('$lib/gen', async () => { return { status: 'saved', current_timestamp: '2026-06-15T00:00:00Z' } }), getDraftForUser: vi.fn(async ({ kind, path }: any) => { + // The real endpoint rejects drawer kinds up front (drafts for + // schedule/trigger/resource/variable are private to their owner) — + // mirror it so a caller regressing to this route for those kinds + // fails in tests the same way it does against the backend. + if (!['script', 'flow', 'app', 'raw_app'].includes(kind)) + throw Object.assign(new Error('drafts for this item kind are private to their owner'), { + status: 404 + }) const key = `${kind}:${path}` if (failingReads.has(key)) throw Object.assign(new Error('server error'), { status: 500 }) - // 404-shaped (status) like the real ApiError, so the adapter's - // narrowed catch treats it as "no draft" rather than re-throwing. if (!backendDrafts.has(key)) throw Object.assign(new Error('no draft for that owner at that path'), { status: 404 }) return { value: backendDrafts.get(key), created_at: '2026-06-15T00:00:00Z' } }), + getOwnDraft: vi.fn(async ({ kind, path }: any) => { + const key = `${kind}:${path}` + if (failingReads.has(key)) throw Object.assign(new Error('server error'), { status: 500 }) + // The real endpoint returns 200 with null when the user has no draft. + if (!backendDrafts.has(key)) return null + return { value: backendDrafts.get(key), created_at: '2026-06-15T00:00:00Z' } + }), listDrafts: vi.fn(async () => Array.from(backendDrafts.entries()).map(([key, value]) => { const idx = key.indexOf(':') @@ -1163,6 +1182,170 @@ describe('global AI tools', () => { expect(draft).not.toHaveProperty('override') }) + // Schedule drafts (like all drawer kinds) are private to their owner, so the + // cross-user draft route 404s on them. Reading them back must go through the + // own-draft route, else a freshly written schedule draft is listed but can + // never be read or deployed. + it('reads and deploys a schedule draft written by the chat', async () => { + await callGlobalTool('write_schedule', { + path: 'u/admin/test_schedule_greet', + schedule: '0 0 9 * * *', + timezone: 'UTC', + script_path: 'f/scripts/greet', + is_flow: false, + args: {} + }) + + const readRaw = await callGlobalTool('read_workspace_item', { + type: 'schedule', + path: 'u/admin/test_schedule_greet' + }) + expect(JSON.parse(readRaw)).toMatchObject({ + type: 'schedule', + path: 'u/admin/test_schedule_greet', + isDraft: true + }) + + await callGlobalTool('deploy_workspace_item', { + type: 'schedule', + path: 'u/admin/test_schedule_greet' + }) + expect(ScheduleService.createSchedule).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: expect.objectContaining({ + path: 'u/admin/test_schedule_greet', + schedule: '0 0 9 * * *', + script_path: 'f/scripts/greet' + }) + }) + // The draft is consumed by the deploy. + expect( + getBackendDraft('trigger_schedule', 'u/admin/test_schedule_greet', { + workspace: WORKSPACE + }) + ).toBeUndefined() + }) + + // Same private-owner read path as schedules, for the trigger drawer kinds. + it('reads and deploys a trigger draft written by the chat', async () => { + await callGlobalTool('write_trigger', { + kind: 'http', + config: { + path: 'u/admin/fresh_route', + script_path: 'f/scripts/handler', + is_flow: false, + route_path: 'api/fresh', + http_method: 'get', + authentication_method: 'none', + is_static_website: false + } + }) + + const readRaw = await callGlobalTool('read_workspace_item', { + type: 'trigger', + trigger_kind: 'http', + path: 'u/admin/fresh_route' + }) + expect(JSON.parse(readRaw)).toMatchObject({ + type: 'trigger', + triggerKind: 'http', + path: 'u/admin/fresh_route', + isDraft: true + }) + + await callGlobalTool('deploy_workspace_item', { + type: 'trigger', + trigger_kind: 'http', + path: 'u/admin/fresh_route' + }) + expect(HttpTriggerService.createHttpTrigger).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: expect.objectContaining({ + path: 'u/admin/fresh_route', + route_path: 'api/fresh', + script_path: 'f/scripts/handler' + }) + }) + expect( + getBackendDraft('trigger_http', 'u/admin/fresh_route', { workspace: WORKSPACE }) + ).toBeUndefined() + }) + + // Same private-owner read path as schedules, for the resource drawer kind. + it('reads and deploys a resource draft written by the chat', async () => { + await callGlobalTool('write_resource', { + path: 'u/admin/fresh_db', + value: { host: 'db.example.com', port: 5432 }, + resource_type: 'postgresql', + description: 'fresh database' + }) + + const readRaw = await callGlobalTool('read_workspace_item', { + type: 'resource', + path: 'u/admin/fresh_db' + }) + expect(JSON.parse(readRaw)).toMatchObject({ + type: 'resource', + path: 'u/admin/fresh_db', + isDraft: true + }) + + await callGlobalTool('deploy_workspace_item', { + type: 'resource', + path: 'u/admin/fresh_db' + }) + expect(ResourceService.createResource).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: expect.objectContaining({ + path: 'u/admin/fresh_db', + resource_type: 'postgresql', + value: { host: 'db.example.com', port: 5432 } + }) + }) + expect( + getBackendDraft('resource', 'u/admin/fresh_db', { workspace: WORKSPACE }) + ).toBeUndefined() + }) + + // Same private-owner read path as schedules, for the variable drawer kind. + // Secret variables deploy through the ephemeral in-memory value instead + // (see the ephemeral-value tests above); this pins the plain-value cycle. + it('reads and deploys a non-secret variable draft written by the chat', async () => { + await callGlobalTool('write_variable', { + path: 'u/admin/fresh_config', + value: 'plain-value', + is_secret: false, + description: 'fresh config' + }) + + const readRaw = await callGlobalTool('read_workspace_item', { + type: 'variable', + path: 'u/admin/fresh_config' + }) + expect(JSON.parse(readRaw)).toMatchObject({ + type: 'variable', + path: 'u/admin/fresh_config', + isDraft: true + }) + + await callGlobalTool('deploy_workspace_item', { + type: 'variable', + path: 'u/admin/fresh_config' + }) + expect(VariableService.createVariable).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: expect.objectContaining({ + path: 'u/admin/fresh_config', + value: 'plain-value', + is_secret: false, + description: 'fresh config' + }) + }) + expect( + getBackendDraft('variable', 'u/admin/fresh_config', { workspace: WORKSPACE }) + ).toBeUndefined() + }) + it('requires trigger_kind when discarding a trigger draft', async () => { await expect( callGlobalTool('discard_local_draft', { @@ -3086,9 +3269,7 @@ describe('folder tools', () => { }) it('create_folder surfaces a backend error (e.g. name conflict)', async () => { - vi.mocked(FolderService.createFolder).mockRejectedValueOnce( - new Error('Folder already exists') - ) + vi.mocked(FolderService.createFolder).mockRejectedValueOnce(new Error('Folder already exists')) const raw = await callGlobalTool('create_folder', { name: 'taken' }) const parsed = JSON.parse(raw) expect(parsed.success).toBe(false) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b85e271632..24045280d0 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -56,6 +56,7 @@ import { createInlineScriptSession } from '../flow/inlineScriptsUtils' import { getDatatableSdkReference, getFlowPrompt, + getPipelinePrompt, getRawAppPrompt, getResourcePrompt, getScriptPrompt @@ -113,6 +114,7 @@ import { getEphemeralSecretVariableDraftValue, getGlobalDraft, getGlobalDraftStoragePath, + itemKindFor, listGlobalDrafts, persistGlobalDraft, readGlobalDraftValue, @@ -138,7 +140,9 @@ const INSTRUCTION_SUBJECTS = [ ] as const satisfies readonly WorkspaceItemType[] // `datatable` is not a workspace item type, but the model can request the // datatable SDK reference (the wmill.datatable() runnable API) the same way. -const INSTRUCTION_SUBJECTS_EXTRA = ['datatable'] as const +// `pipeline` likewise isn't an item type — a data pipeline is a set of +// annotated scripts in a folder, so it gets authoring guidance, not a CRUD type. +const INSTRUCTION_SUBJECTS_EXTRA = ['datatable', 'pipeline'] as const const ALL_INSTRUCTION_SUBJECTS = [...INSTRUCTION_SUBJECTS, ...INSTRUCTION_SUBJECTS_EXTRA] as const const MAX_LIST_LIMIT = 100 type ActiveGlobalEditorType = Extract @@ -171,7 +175,7 @@ const scriptLangSchema = z.enum($ScriptLang.enum) const getInstructionsSchema = z.object({ subject: instructionSubjectSchema.describe( - 'What to get authoring instructions for: a workspace item type (script, flow, resource, app) or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' + 'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' ), language: scriptLangSchema .optional() @@ -206,7 +210,9 @@ const updateUserInstructionsSchema = z.object({ .string() .min(1) .optional() - .describe("Required when operation is 'append': the instruction to add. Ignored for 'replace'."), + .describe( + "Required when operation is 'append': the instruction to add. Ignored for 'replace'." + ), old_string: z .string() .min(1) @@ -678,11 +684,13 @@ const deleteAppRunnableSchema = z.object({ const openPreviewSchema = z.object({ kind: z - .enum(['script', 'flow', 'raw_app']) + .enum(['script', 'flow', 'raw_app', 'pipeline']) .describe( - 'Item kind to preview. Use "raw_app" for code-based apps (created via init_app). The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' + 'Item kind to preview. Use "raw_app" for code-based apps (created via init_app). Use "pipeline" to show the data-pipeline graph for a folder — here `path` is the folder name, not an item path. The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' ), - path: z.string().describe('Workspace path of the item to preview.') + path: z + .string() + .describe('Workspace path of the item to preview, or the folder name when kind is "pipeline".') }) const getPreviewStatusSchema = z.object({}) @@ -825,6 +833,7 @@ Rules: - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource. - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. +- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow. - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. @@ -833,6 +842,7 @@ Rules: previewTools ? ` - After writing or substantially editing a script / flow / app draft, show it via open_preview(kind, path) so the user sees the editor and live preview right next to the chat. First check whether it is already shown: if unsure, call get_preview_status. Only call open_preview (or offer to) when no preview is open or it is showing a different item — don't re-open a preview already showing the item you just edited. +- Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node — this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node — use ONLY those to add or change pipeline nodes, never write_script for a pipeline node — they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). - get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected.` : '' @@ -1608,6 +1618,10 @@ Datatables are workspace-scoped managed PostgreSQL databases. In chat, explore a ${getDatatableSdkReference(lang)}` } +function getPipelineInstructions(): string { + return getPipelinePrompt() +} + function getInstructions(subject: InstructionSubject, language?: ScriptLang): string { switch (subject) { case 'script': @@ -1620,6 +1634,8 @@ function getInstructions(subject: InstructionSubject, language?: ScriptLang): st return getAppInstructions() case 'datatable': return getDatatableInstructions(language) + case 'pipeline': + return getPipelineInstructions() } } @@ -1672,7 +1688,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( getInstructionsSchema, 'get_instructions', - 'Get authoring guidance for scripts, flows, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' + 'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' ), fn: async ({ args, toolId, toolCallbacks }) => { const parsed = getInstructionsSchema.parse(args) @@ -2490,7 +2506,7 @@ function activeFlowTestFromCtx( export type OpenPreviewHandler = (req: { sessionId: string | undefined - kind: 'script' | 'flow' | 'raw_app' + kind: 'script' | 'flow' | 'raw_app' | 'pipeline' path: string }) => string @@ -2501,7 +2517,7 @@ export function setOpenPreviewHandler(handler: OpenPreviewHandler | undefined): } function openSessionPreview( - args: { kind: 'script' | 'flow' | 'raw_app'; path: string }, + args: { kind: 'script' | 'flow' | 'raw_app' | 'pipeline'; path: string }, sessionId: string | undefined ) { if (!openPreviewHandler) { @@ -2764,6 +2780,7 @@ function finishAppDraftWrite( ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const { content, message } = onSaved() ctx.toolCallbacks.setToolStatus(ctx.toolId, { content, result: 'Saved as draft' }) return JSON.stringify({ success: true, message }, null, 2) @@ -2776,6 +2793,7 @@ function finishDraftWrite( ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const stored = result.item const verb = existed ? 'Updated' : 'Created' // Don't echo the flow value back: the model just sent it in the write call, @@ -3883,6 +3901,16 @@ async function discardLocalDraft( await deleteGlobalDraft(workspace, type, path, triggerKind) + // The chat's touch on the item is undone — drop it from the mask so a + // pre-existing deployed item doesn't keep reading as this chat's edit. + const discardedKind = itemKindFor(type, triggerKind) + if (discardedKind) { + toolCallbacks.onItemDiscarded?.( + discardedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind) + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Discarded ${type} "${path}" draft`, result: 'Draft discarded' @@ -4222,6 +4250,9 @@ async function deployDraft( }) let actions: ToolDisplayAction[] | undefined + // Where the deploy actually lands — the app branch can resolve a different + // target from the draft's own path fields; the mask rename below must track it. + let deployedPath = path if (type === 'script' || type === 'flow') { // Promote the full persisted draft via the shared deploy module — the same @@ -4414,6 +4445,7 @@ async function deployDraft( throw e } } + deployedPath = targetPath if (await AppService.existsApp({ workspace, path: targetPath })) { // Omit custom_path on update for now. The backend preserves it when absent, while // sending it requires admin privileges; this chat deploy path does not yet mirror @@ -4463,6 +4495,18 @@ async function deployDraft( await deleteGlobalDraft(workspace, type, path, triggerKind, { preserveLiveDraft: true }) + // Move the chat's mask entry to the deployed path: a draft-only item's + // synthetic storage key never exists deployed, so the entry would otherwise + // stop matching anything after the draft is gone. + const deployedKind = itemKindFor(type, triggerKind) + if (deployedKind) { + toolCallbacks.onItemDeployed?.( + deployedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind), + deployedPath + ) + } + // Reload the session preview if it's open on the deployed item. Map the // deploy type to the preview kind — a raw app deploys under 'app' but the // preview addresses it as 'raw_app'; non-previewable types map to undefined. @@ -4533,6 +4577,17 @@ async function deleteWorkspaceItem( await deleteGlobalDraft(workspace, type, path, triggerKind) + // Record the deletion in the chat's modified-items mask. In a fork this leaves a + // reviewable "removed" diff vs the parent that stays scoped to this chat. Keyed + // by the same (itemKind, storagePath) as writes so it joins the draft/fork lists. + const deletedKind = itemKindFor(type, triggerKind) + if (deletedKind) { + toolCallbacks.onItemModified?.( + deletedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind) + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Deleted ${type} "${path}"`, result: 'Deleted' diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index 87ca767904..342717f26e 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -1,7 +1,5 @@ import type { Flow, NewSchedule, NewScript } from '$lib/gen/types.gen' import { DraftService } from '$lib/gen' -import { get } from 'svelte/store' -import { userStore } from '$lib/stores' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { UserDraft, type UserDraftEntry, type UserDraftItemKind } from '$lib/userDraft.svelte' @@ -108,7 +106,7 @@ function clearEphemeralSecretVariableDraftValues(workspace: string): void { secretVariableDraftValues.delete(workspace) } -function itemKindFor( +export function itemKindFor( type: WorkspaceItemType, triggerKind?: TriggerKind ): UserDraftItemKind | undefined { @@ -336,29 +334,26 @@ function getGlobalDraftSlot( } // Current user's persisted draft value (+ records the sync baseline so a later -// save detects external conflicts). undefined on 404 (no draft at that path). +// save detects external conflicts). undefined when no draft exists at that path. +// Uses `getOwnDraft` (not `getDraftForUser`): the latter rejects drawer kinds +// (schedule/trigger/resource/variable drafts are private to their owner), which +// would make those drafts write-only here — listed but never readable/deployable. +// Errors (403/500/network) MUST propagate: swallowing one would make the write +// merge fall through to the deployed item instead of the user's in-progress +// draft, silently overwriting their draft-only changes. async function fetchBackendDraftValue( workspace: string, itemKind: UserDraftItemKind, storagePath: string ): Promise { - try { - const resp = await DraftService.getDraftForUser({ - workspace, - kind: itemKind as any, - path: storagePath, - username: get(userStore)?.username - }) - UserDraftDbSyncer.recordRemoteSync({ workspace, itemKind, path: storagePath }, resp.created_at) - return resp.value ?? undefined - } catch (e) { - // 404 = no draft for this owner at that path (the intended empty case). - // Anything else (403/500/network) MUST propagate: swallowing it would make - // the write merge fall through to the deployed item instead of the user's - // in-progress draft, silently overwriting their draft-only changes. - if ((e as { status?: number } | null | undefined)?.status === 404) return undefined - throw e - } + const resp = await DraftService.getOwnDraft({ + workspace, + kind: itemKind, + path: storagePath + }) + if (!resp) return undefined + UserDraftDbSyncer.recordRemoteSync({ workspace, itemKind, path: storagePath }, resp.created_at) + return resp.value ?? undefined } // Draft VALUE for a write merge: cell-if-present (the user's freshest in-tab @@ -377,10 +372,25 @@ export async function readGlobalDraftValue( return (await fetchBackendDraftValue(workspace, itemKind, storagePath)) as V | undefined } +// `itemKind` + `storagePath` are the canonical identity of the persisted draft +// (NOT item.path, which is the friendly display path). Callers use them to record +// the chat's modified-items mask. export type DraftPersistResult = - | { status: 'saved'; item: WorkspaceItem } - | { status: 'conflict'; item: WorkspaceItem; serverTimestamp?: string } - | { status: 'error'; item: WorkspaceItem; message: string } + | { status: 'saved'; item: WorkspaceItem; itemKind: UserDraftItemKind; storagePath: string } + | { + status: 'conflict' + item: WorkspaceItem + itemKind: UserDraftItemKind + storagePath: string + serverTimestamp?: string + } + | { + status: 'error' + item: WorkspaceItem + itemKind: UserDraftItemKind + storagePath: string + message: string + } // Persist a built draft value. `UserDraft.seed` reflects it into an open editor's // cell WITHOUT a double-POST (no-ops if no cell; its seedNextWrite suppresses the @@ -417,14 +427,20 @@ export async function persistGlobalDraft( // the chat "saved" while the DB-backed source of truth was never updated. const saveState = UserDraftDbSyncer.getState({ workspace, itemKind, path: storagePath }) if (saveState.state === 'failed') { - return { status: 'error', item, message: saveState.failureMessage ?? 'Draft save failed' } + return { + status: 'error', + item, + itemKind, + storagePath, + message: saveState.failureMessage ?? 'Draft save failed' + } } const conflict = opts.force ? undefined : UserDraftDbSyncer.getConflict({ workspace, itemKind, path: storagePath }).conflict return conflict - ? { status: 'conflict', item, serverTimestamp: conflict.serverTimestamp } - : { status: 'saved', item } + ? { status: 'conflict', item, itemKind, storagePath, serverTimestamp: conflict.serverTimestamp } + : { status: 'saved', item, itemKind, storagePath } } export async function getGlobalDraft( diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts new file mode 100644 index 0000000000..4f3e31986d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect, vi } from 'vitest' + +// `../shared` transitively pulls in the monaco editor (and its CSS), which the +// node test environment can't load — mirror the sibling chat tests' stub. +vi.mock('monaco-editor', () => ({ editor: {} })) + +import { + pipelineTools, + getPipelinePromptSection, + type PipelineAIChatHelpers, + type PipelineContext +} from './core' +import type { ToolCallbacks } from '../shared' + +function toolByName(name: string) { + const tool = pipelineTools.find((t) => t.def.function.name === name) + if (!tool) throw new Error(`tool ${name} not found`) + return tool +} + +function noopCallbacks(): ToolCallbacks { + return { setToolStatus: () => {}, removeToolStatus: () => {} } +} + +const sampleContext: PipelineContext = { + folder: 'analytics', + mode: 'edit', + nodes: [ + { + path: 'f/analytics/orders', + language: 'bun', + unsaved: true, + writes: ['ducklake://main/orders'], + reads: [], + triggers: ['schedule'] + } + ], + assets: ['ducklake://main/orders'] +} + +function makeHelpers(overrides: Partial = {}): { + helpers: { pipeline: PipelineAIChatHelpers } + calls: Record +} { + const calls: Record = {} + const record = + (name: string, ret?: any) => + (...args: any[]) => { + ;(calls[name] ??= []).push(args) + return ret + } + const pipeline: PipelineAIChatHelpers = { + getPipelineContext: () => sampleContext, + getNodeBody: async (path: string) => { + calls.getNodeBody = [...(calls.getNodeBody ?? []), [path]] + return { language: 'bun', content: 'export async function main() { return 1 }' } + }, + proposeNode: async (input) => { + calls.proposeNode = [...(calls.proposeNode ?? []), [input]] + return { path: input.path } + }, + editNode: async (path, content) => { + calls.editNode = [...(calls.editNode ?? []), [path, content]] + }, + removeProposedNode: record('removeProposedNode'), + testNode: async () => 'job-123', + ...overrides + } + return { helpers: { pipeline }, calls } +} + +describe('pipeline tools', () => { + it('exposes the expected tool surface', () => { + expect(pipelineTools.map((t) => t.def.function.name).sort()).toEqual([ + 'build_pipeline_node', + 'edit_pipeline_node', + 'get_pipeline_graph', + 'read_pipeline_node', + 'remove_pipeline_node', + 'test_pipeline_node' + ]) + }) + + it('get_pipeline_graph returns the live context as JSON', async () => { + const { helpers } = makeHelpers() + const out = await toolByName('get_pipeline_graph').fn({ + args: {}, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(JSON.parse(out)).toMatchObject({ folder: 'analytics' }) + }) + + it('build_pipeline_node forwards to proposeNode and does not deploy', async () => { + const { helpers, calls } = makeHelpers() + const out = await toolByName('build_pipeline_node').fn({ + args: { + path: 'f/analytics/clean', + language: 'bun', + content: '// pipeline\nexport async function main() {}', + output_kind: 'ducklake' + }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(calls.proposeNode?.[0]?.[0]).toMatchObject({ + path: 'f/analytics/clean', + language: 'bun', + outputKind: 'ducklake' + }) + expect(out).toContain('not deployed') + }) + + it('edit_pipeline_node reads then applies an exact find/replace', async () => { + const { helpers, calls } = makeHelpers({ + getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\nconst y = 2\n' }) + }) + await toolByName('edit_pipeline_node').fn({ + args: { path: 'f/analytics/orders', old_string: 'const x = 1', new_string: 'const x = 42' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(calls.editNode?.[0]?.[1]).toContain('const x = 42') + }) + + it('edit_pipeline_node surfaces a clear error when old_string is absent', async () => { + const { helpers } = makeHelpers({ + getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\n' }) + }) + await expect( + toolByName('edit_pipeline_node').fn({ + args: { path: 'f/analytics/orders', old_string: 'NOT THERE', new_string: 'x' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + ).rejects.toThrow(/was not found/) + }) + + it('mutation tools fail clearly when no pipeline editor is registered', async () => { + await expect( + toolByName('build_pipeline_node').fn({ + args: { path: 'f/a/b', language: 'bun', content: 'x' }, + workspace: 'w', + helpers: {}, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + ).rejects.toThrow(/No pipeline editor is open/) + }) + + it('test_pipeline_node requires confirmation', () => { + expect(toolByName('test_pipeline_node').requiresConfirmation).toBe(true) + }) +}) + +describe('getPipelinePromptSection', () => { + it('names the active folder and the direct-draft workflow', () => { + const section = getPipelinePromptSection(sampleContext) + expect(section).toContain('/pipeline/analytics') + expect(section).toContain('build_pipeline_node') + expect(section).toContain('directly as unsaved drafts') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts new file mode 100644 index 0000000000..86fe9eb634 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -0,0 +1,328 @@ +import { z } from 'zod' +import { $ScriptLang } from '$lib/gen/schemas.gen' +import type { ScriptLang } from '$lib/gen' +import { createToolDef, executeTestRun, findAndReplace, type Tool } from '../shared' +import type { PipelineOutputKind } from '$lib/components/assets/AssetGraph/pipelineTemplates' + +// ============================================================================ +// Pipeline AI chat tools. +// +// These tools extend the GLOBAL chat mode when the user is on a /pipeline/ +// editor (the page registers `PipelineAIChatHelpers` on the AIChatManager). They +// let the model read the live pipeline graph and BUILD/EDIT pipeline nodes +// (scripts annotated with `// pipeline`). Mutations don't deploy: they apply +// directly as an unsaved DRAFT on the canvas — the same way the flow/script +// editor applies AI edits — which the user then deploys. There is no separate +// approve/reject step: the draft IS the change. +// +// The pipeline tools are added on top of the full global tool set, so docs +// search, datatable SQL, and workspace-item tools are already available +// alongside them — this file only carries the pipeline-graph-specific surface. +// ============================================================================ + +/** Compact, model-facing summary of one node in the pipeline graph. */ +export type PipelineNodeSummary = { + path: string + language?: ScriptLang + /** Has an unsaved local edit (draft) not yet deployed. */ + unsaved: boolean + summary?: string + /** Asset URIs this node writes (its outputs). */ + writes: string[] + /** Asset URIs this node reads (its inputs). */ + reads: string[] + /** Declared `// on ` execution-DAG bindings (asset URIs or native kinds). */ + triggers: string[] +} + +/** Compact, model-facing snapshot of the whole pipeline graph. */ +export type PipelineContext = { + folder: string + mode: 'view' | 'edit' + nodes: PipelineNodeSummary[] + /** All storage assets referenced by the graph, as URIs. */ + assets: string[] +} + +/** + * Bridge the pipeline page registers on the AIChatManager. Reads expose the live + * graph; writes apply directly as unsaved drafts (never deploy). Kept intentionally + * small — the page owns the draft Map and canvas rendering. + */ +export interface PipelineAIChatHelpers { + getPipelineContext: () => PipelineContext + /** Read a node's source (the in-flight draft body if one exists, else deployed). */ + getNodeBody: (path: string) => Promise<{ language: ScriptLang; content: string } | undefined> + /** Create a brand-new pipeline node as an unsaved draft on the canvas. */ + proposeNode: (input: { + path: string + language: ScriptLang + content: string + outputKind?: PipelineOutputKind + }) => Promise<{ path: string }> + /** Replace an existing node's body, applied as an unsaved draft. */ + editNode: (path: string, content: string) => Promise + /** Discard the unsaved draft at a path (undo a build_pipeline_node). */ + removeProposedNode: (path: string) => Promise + /** Preview-run a node (draft body preferred). Returns the started job id. */ + testNode: (path: string, args?: Record) => Promise +} + +/** Helper bag the pipeline tools receive from the manager in global mode. */ +export type PipelineToolHelpers = { pipeline?: PipelineAIChatHelpers } + +function requirePipeline(helpers: PipelineToolHelpers): PipelineAIChatHelpers { + if (!helpers?.pipeline) { + throw new Error( + 'No pipeline editor is open. Pipeline tools only work on a /pipeline/ page in edit mode.' + ) + } + return helpers.pipeline +} + +const scriptLangSchema = z.enum($ScriptLang.enum) + +const outputKindSchema = z + .enum(['none', 'datatable', 'ducklake', 'materialize', 's3_parquet', 's3_object']) + .describe( + 'Kind of output asset this node materializes, used to seed the output edge on the canvas before the body is parsed: "materialize"/"ducklake" → a DuckLake table, "datatable" → a Postgres data table, "s3_parquet"/"s3_object" → an S3 file, "none" → side-effect only. Defaults to none.' + ) + +// ---------------------------------------------------------------------------- +// Read tools +// ---------------------------------------------------------------------------- + +const getPipelineGraphSchema = z.object({}) + +const getPipelineGraphToolDef = createToolDef( + getPipelineGraphSchema, + 'get_pipeline_graph', + "Read the live pipeline graph for the open /pipeline/ editor: its nodes (scripts), each node's language, asset reads/writes, declared triggers, and whether it has an unsaved draft edit. Call this before building or editing nodes so you reuse existing assets/paths and understand the current DAG." +) + +const readPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the pipeline node (script) to read.') +}) + +const readPipelineNodeToolDef = createToolDef( + readPipelineNodeSchema, + 'read_pipeline_node', + 'Read the full source of one pipeline node (its in-flight draft body if it has unsaved edits, otherwise the deployed body). Use before edit_pipeline_node so edits target the exact current text.' +) + +// ---------------------------------------------------------------------------- +// Mutation tools (apply directly as unsaved drafts; never deploy) +// ---------------------------------------------------------------------------- + +const buildPipelineNodeSchema = z.object({ + path: z + .string() + .describe( + "Workspace path for the new node, e.g. f//. Use the open pipeline's folder. Must not collide with an existing node." + ), + language: scriptLangSchema.describe( + 'Script language. SQL-shaped data work uses duckdb (DuckLake/S3) or postgresql (data tables); bun/python3 for general transforms.' + ), + content: z + .string() + .describe( + "Full script source. Start it with the `pipeline` annotation as a top-of-file comment in the LANGUAGE'S comment syntax — `-- pipeline` for SQL (duckdb/postgresql), `# pipeline` for python3/bash, `// pipeline` for bun/TS — to mark it a pipeline member; declare inputs the same way (e.g. `-- on `), and write outputs via the wmill SDK / SQL so the lineage edges are inferred. A `// pipeline` line in a SQL node is a syntax error. Read existing node bodies first to match conventions." + ), + output_kind: outputKindSchema.optional() +}) + +const buildPipelineNodeToolDef = createToolDef( + buildPipelineNodeSchema, + 'build_pipeline_node', + 'Build a NEW pipeline node. It is applied directly as an unsaved draft on the canvas (a dashed node wired by its parsed asset reads/writes) — it does NOT deploy; the user deploys it. Prefer this over editing for new scripts.', + { strict: false } +) + +const editPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node to edit.'), + old_string: z.string().min(1).describe("Exact text to find in the node's current source."), + new_string: z.string().describe('Replacement text.'), + replace_all: z + .boolean() + .optional() + .default(false) + .describe( + 'When true, replace every exact match. When false, old_string must match exactly once.' + ) +}) + +const editPipelineNodeToolDef = createToolDef( + editPipelineNodeSchema, + 'edit_pipeline_node', + 'Edit an existing pipeline node by exact find/replace. The result is applied directly as an unsaved draft (does NOT deploy). Call read_pipeline_node first to get the exact current text.', + { strict: false } +) + +const removePipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node whose unsaved draft should be discarded.') +}) + +const removePipelineNodeToolDef = createToolDef( + removePipelineNodeSchema, + 'remove_pipeline_node', + 'Discard the unsaved draft at a path (undo a build_pipeline_node). Only affects the in-flight draft — to delete a deployed node, ask the user to do it on the canvas.' +) + +const testPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node to preview-run.'), + args: z + .record(z.string(), z.any()) + .nullable() + .optional() + .describe('Arguments to pass to the script. Omit or pass null when none are needed.') +}) + +const testPipelineNodeToolDef = createToolDef( + testPipelineNodeSchema, + 'test_pipeline_node', + 'Preview-run one pipeline node (using its draft body when unsaved) and return the result/logs, without deploying. Requires user confirmation before it runs.', + { strict: false } +) + +// ---------------------------------------------------------------------------- +// Tool set +// ---------------------------------------------------------------------------- + +export const pipelineTools: Tool[] = [ + { + def: getPipelineGraphToolDef, + fn: async ({ helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + toolCallbacks.setToolStatus(toolId, { content: 'Reading pipeline graph...' }) + const ctx = pipeline.getPipelineContext() + toolCallbacks.setToolStatus(toolId, { + content: `Read pipeline graph (${ctx.nodes.length} node${ctx.nodes.length === 1 ? '' : 's'})`, + result: 'Success' + }) + return JSON.stringify(ctx, null, 2) + } + }, + { + def: readPipelineNodeToolDef, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path } = readPipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Reading node '${path}'...` }) + const node = await pipeline.getNodeBody(path) + if (!node) { + return `No pipeline node found at '${path}'. Call get_pipeline_graph to list the available nodes.` + } + toolCallbacks.setToolStatus(toolId, { content: `Read node '${path}'`, result: 'Success' }) + return JSON.stringify({ path, language: node.language, content: node.content }) + } + }, + { + def: buildPipelineNodeToolDef, + streamArguments: true, + showDetails: true, + showFade: true, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, language, content, output_kind } = buildPipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Building node '${path}'...` }) + await pipeline.proposeNode({ + path, + language: language as ScriptLang, + content, + outputKind: output_kind as PipelineOutputKind | undefined + }) + toolCallbacks.setToolStatus(toolId, { + content: `Added draft node '${path}'`, + result: 'Success' + }) + return `Pipeline node '${path}' added as an unsaved draft on the canvas. It is not deployed — the user deploys it.` + } + }, + { + def: editPipelineNodeToolDef, + streamArguments: true, + showDetails: true, + showFade: true, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, old_string, new_string, replace_all } = editPipelineNodeSchema.parse(args) + const node = await pipeline.getNodeBody(path) + if (!node) { + return `No pipeline node found at '${path}'. Call get_pipeline_graph to list the available nodes.` + } + toolCallbacks.setToolStatus(toolId, { content: `Editing node '${path}'...` }) + const updated = findAndReplace( + node.content, + old_string, + new_string, + replace_all ?? false, + 'node source' + ) + await pipeline.editNode(path, updated) + toolCallbacks.setToolStatus(toolId, { + content: `Edited draft '${path}'`, + result: 'Success' + }) + return `Pipeline node '${path}' updated as an unsaved draft on the canvas (not deployed).` + } + }, + { + def: removePipelineNodeToolDef, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path } = removePipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Discarding draft '${path}'...` }) + await pipeline.removeProposedNode(path) + toolCallbacks.setToolStatus(toolId, { + content: `Discarded draft '${path}'`, + result: 'Success' + }) + return `Discarded the unsaved draft at '${path}'.` + } + }, + { + def: testPipelineNodeToolDef, + requiresConfirmation: true, + confirmationMessage: 'Run pipeline node', + showDetails: true, + autoCollapseDetails: false, + fn: async ({ args, workspace, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, args: runArgs } = testPipelineNodeSchema.parse(args) + return executeTestRun({ + jobStarter: async () => { + const jobId = await pipeline.testNode(path, runArgs ?? undefined) + if (!jobId) { + throw new Error(`Could not start a run for node '${path}'.`) + } + return jobId + }, + workspace, + toolCallbacks, + toolId, + startMessage: `Starting run of '${path}'...`, + contextName: 'script' + }) + } + } +] + +/** + * Pipeline-specific guidance appended to the global system prompt when a + * /pipeline editor is open. Describes the annotation model and the direct-draft + * workflow so the model uses the pipeline tools rather than the generic + * write_script draft tools. + */ +export function getPipelinePromptSection(ctx: PipelineContext): string { + return ` + +Data Pipeline editor (ACTIVE): +- The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. +- Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. +- Build new nodes with build_pipeline_node and edit existing ones with edit_pipeline_node. These apply directly as unsaved drafts on the canvas (like the flow/script editor applies AI edits) — they DO NOT deploy. There is no separate Accept/Reject step. Prefer these over the generic write_script/edit_script draft tools while a pipeline is open. +- Reuse existing asset paths from the graph when wiring a downstream node to an upstream one (read the upstream's write asset, then \`// on\` that same URI). +- Only deploy when the user explicitly asks; the user deploys drafts from the canvas.` +} diff --git a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte index 06bcb63d98..dda1892baa 100644 --- a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte @@ -1,6 +1,11 @@ {#if showSvg} -
- - {@html svg} +
+
+
+
+ + {@html svg} +
+ + + {#snippet settings()} +
+
+ {/snippet} +
+ {#if expanded} +
+ + {@html svg} +
+ {/if} +
+ {:else} 
{code}
diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index b958283972..c454f15fe6 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -3,6 +3,7 @@ import type { ChatCompletionMessageFunctionToolCall, ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import type { UserDraftItemKind } from '$lib/gen' /** * Special module IDs used throughout the flow system @@ -759,6 +760,15 @@ export interface ToolCallbacks { toolId: string, question: UserQuestionDisplay ) => Promise + /** Records a workspace item the tool call created/edited/deleted, by its + * canonical (itemKind, storagePath). Session chats wire this to accumulate the + * chat's modified-items mask; the global side-panel chat omits it (no-op). */ + onItemModified?: (itemKind: UserDraftItemKind, storagePath: string) => void + /** A tool deployed a draft: the mask entry moves from the draft's storage path + * to the deployed path (they differ for synthetic draft-only storage keys). */ + onItemDeployed?: (itemKind: UserDraftItemKind, storagePath: string, deployedPath: string) => void + /** A tool discarded a draft: the chat's touch on the item is undone. */ + onItemDiscarded?: (itemKind: UserDraftItemKind, storagePath: string) => void } export function createToolDef( @@ -950,7 +960,11 @@ export async function buildSchemaForTool( // OPEN AI models don't support strict mode well with schema with complex properties, so we disable it const model = getCurrentModel() - if (model.provider === 'openai' || model.provider === 'azure_openai') { + if ( + model.provider === 'openai' || + model.provider === 'azure_openai' || + model.provider === 'azure_foundry' + ) { toolDef.function.strict = false } return true diff --git a/frontend/src/lib/components/copilot/chat/toolCallArguments.test.ts b/frontend/src/lib/components/copilot/chat/toolCallArguments.test.ts new file mode 100644 index 0000000000..53246bc50e --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/toolCallArguments.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import { hasValidToolCallArguments, sanitizeToolCallArguments } from './toolCallArguments' + +describe('hasValidToolCallArguments', () => { + it('accepts empty or valid JSON arguments', () => { + expect(hasValidToolCallArguments(undefined)).toBe(true) + expect(hasValidToolCallArguments('')).toBe(true) + expect(hasValidToolCallArguments('{}')).toBe(true) + expect(hasValidToolCallArguments('{"path": "u/admin/app", "content": "x"}')).toBe(true) + }) + + it('rejects arguments truncated mid-stream', () => { + expect(hasValidToolCallArguments('{"path": "u/admin/app", "old_string": "setMess')).toBe(false) + expect(hasValidToolCallArguments('{"path": "u/admin/app"')).toBe(false) + }) +}) + +describe('sanitizeToolCallArguments', () => { + const poisoned: ChatCompletionMessageParam = { + role: 'assistant', + tool_calls: [ + { + id: 'call_bad', + type: 'function', + function: { name: 'patch_app_file', arguments: '{"path": "u/x", "old_string": "trunc' } + }, + { + id: 'call_ok', + type: 'function', + function: { name: 'read_file', arguments: '{"file": "a.txt"}' } + } + ] + } + + it('replaces only unparseable arguments with {}', () => { + const [sanitized] = sanitizeToolCallArguments([poisoned]) as any[] + expect(sanitized.tool_calls[0].function.arguments).toBe('{}') + expect(sanitized.tool_calls[1].function.arguments).toBe('{"file": "a.txt"}') + // The stored history object is not mutated + expect((poisoned as any).tool_calls[0].function.arguments).toContain('trunc') + }) + + it('rewrites empty arguments to {} so replayed history stays parseable', () => { + const emptyArgs: ChatCompletionMessageParam = { + role: 'assistant', + tool_calls: [{ id: 'c1', type: 'function', function: { name: 'list_files', arguments: '' } }] + } + const [sanitized] = sanitizeToolCallArguments([emptyArgs]) as any[] + expect(sanitized.tool_calls[0].function.arguments).toBe('{}') + }) + + it('returns untouched messages by reference', () => { + const user: ChatCompletionMessageParam = { role: 'user', content: 'hi' } + const validAssistant: ChatCompletionMessageParam = { + role: 'assistant', + tool_calls: [{ id: 'c1', type: 'function', function: { name: 'read_file', arguments: '{}' } }] + } + const result = sanitizeToolCallArguments([user, validAssistant]) + expect(result[0]).toBe(user) + expect(result[1]).toBe(validAssistant) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/toolCallArguments.ts b/frontend/src/lib/components/copilot/chat/toolCallArguments.ts new file mode 100644 index 0000000000..997186a611 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/toolCallArguments.ts @@ -0,0 +1,58 @@ +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' + +/** + * A tool call whose `arguments` string is not valid JSON (typically a stream + * cut mid-arguments) must never reach a provider: the whole request is + * rejected, and once persisted in the session history every follow-up request + * fails the same way. + * + * Empty arguments count as valid: some providers stream '' for no-arg tool + * calls, and flagging those would wrongly report the tool as not executed. + * Callers persisting arguments must normalize '' to '{}' themselves. + */ +export function hasValidToolCallArguments(args: string | undefined): boolean { + if (!args) { + return true + } + try { + JSON.parse(args) + return true + } catch { + return false + } +} + +// '' is valid per hasValidToolCallArguments but JSON.parse('') still throws +// provider-side, so replaying history additionally requires non-empty args. +function isReplayableArguments(args: string | undefined): boolean { + return !!args && hasValidToolCallArguments(args) +} + +/** + * Replaces unparseable or empty assistant tool_call arguments with '{}' in the + * outgoing copy of the history, so a session whose persisted history contains + * a truncated tool call recovers instead of failing every request. The paired + * tool result already tells the model the call failed. + */ +export function sanitizeToolCallArguments( + messages: ChatCompletionMessageParam[] +): ChatCompletionMessageParam[] { + return messages.map((m) => { + if ( + m.role !== 'assistant' || + !m.tool_calls?.some( + (t) => t.type === 'function' && !isReplayableArguments(t.function.arguments) + ) + ) { + return m + } + return { + ...m, + tool_calls: m.tool_calls.map((t) => + t.type === 'function' && !isReplayableArguments(t.function.arguments) + ? { ...t, function: { ...t.function, arguments: '{}' } } + : t + ) + } + }) +} diff --git a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts new file mode 100644 index 0000000000..36e667f57d --- /dev/null +++ b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' + +vi.mock('monaco-editor', () => ({ + editor: {} +})) + +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: () => () => undefined } +})) + +vi.mock('$lib/components/flows/flowTree', () => ({ + findModuleInModules: () => undefined +})) + +vi.mock('$lib/gen', () => ({ + OpenAPI: {}, + ResourceService: {}, + ScriptService: {}, + FlowService: {}, + JobService: {}, + ScheduleService: {}, + HttpTriggerService: {}, + WebsocketTriggerService: {}, + KafkaTriggerService: {}, + NatsTriggerService: {}, + PostgresTriggerService: {}, + MqttTriggerService: {}, + SqsTriggerService: {}, + GcpTriggerService: {}, + AzureTriggerService: {} +})) + +vi.mock('$lib/utils', () => ({ + emptyString: (value: string | undefined | null) => !value, + generateRandomString: () => 'generated_id' +})) + +vi.mock('$lib/scripts', () => ({ + scriptLangToEditorLang: (language: string) => language +})) + +vi.mock('$lib/aiStore', () => ({ + getCurrentModel: () => undefined, + getMetadataModel: () => undefined, + copilotInfo: { + subscribe: (run: (value: unknown) => void) => { + run({}) + return () => undefined + } + } +})) + +vi.mock('@leeoniya/ufuzzy', () => ({ + default: class { + search() { + return [[], [], []] + } + } +})) + +function streamOf(chunks: unknown[]): any { + return (async function* () { + for (const chunk of chunks) { + yield chunk + } + })() +} + +function toolCallChunk(delta: Record) { + return { choices: [{ delta: { tool_calls: [{ index: 0, ...delta }] } }] } +} + +function createCallbacks() { + return { + onNewToken: vi.fn(), + onMessageEnd: vi.fn(), + setToolStatus: vi.fn(), + removeToolStatus: vi.fn() + } +} + +function createTool(fn = vi.fn().mockResolvedValue('tool ok')) { + return { + def: { + type: 'function' as const, + function: { name: 'patch_app_file', parameters: { type: 'object' } } + }, + fn + } +} + +describe('parseOpenAICompletion tool call arguments', () => { + it('does not execute nor persist a tool call whose streamed arguments are truncated', async () => { + const { parseOpenAICompletion } = await import('./lib') + const fn = vi.fn() + const callbacks = createCallbacks() + const messages: ChatCompletionMessageParam[] = [] + const addedMessages: ChatCompletionMessageParam[] = [] + + const result = await parseOpenAICompletion( + streamOf([ + toolCallChunk({ + id: 'call_1', + function: { name: 'patch_app_file', arguments: '{"path": "u/admin/app", ' } + }), + // The stream ends mid-arguments (e.g. output token limit or dropped connection) + toolCallChunk({ function: { arguments: '"old_string": "setMessages(prev' } }) + ]), + callbacks, + messages, + addedMessages, + [createTool(fn)] as any, + {}, + undefined, + { workspace: 'test' } + ) + + expect(fn).not.toHaveBeenCalled() + expect(result.shouldContinue).toBe(true) + + const assistant = messages.find((m) => m.role === 'assistant') as any + // Persisting the truncated arguments string would make every follow-up + // request fail provider-side JSON parsing, bricking the session. + expect(assistant.tool_calls[0].function.arguments).toBe('{}') + + const toolResult = messages.find((m) => m.role === 'tool') as any + expect(toolResult.tool_call_id).toBe('call_1') + expect(toolResult.content).toContain('NOT executed') + expect(callbacks.setToolStatus).toHaveBeenCalledWith( + 'call_1', + expect.objectContaining({ error: expect.stringContaining('invalid or truncated') }) + ) + expect(addedMessages).toEqual(messages) + }) + + it('executes a tool call with valid streamed arguments and keeps them verbatim', async () => { + const { parseOpenAICompletion } = await import('./lib') + const fn = vi.fn().mockResolvedValue('tool ok') + const messages: ChatCompletionMessageParam[] = [] + + const result = await parseOpenAICompletion( + streamOf([ + toolCallChunk({ + id: 'call_1', + function: { name: 'patch_app_file', arguments: '{"path": ' } + }), + toolCallChunk({ function: { arguments: '"u/admin/app"}' } }) + ]), + createCallbacks(), + messages, + [], + [createTool(fn)] as any, + {}, + undefined, + { workspace: 'test' } + ) + + expect(fn).toHaveBeenCalledWith(expect.objectContaining({ args: { path: 'u/admin/app' } })) + expect(result.shouldContinue).toBe(true) + + const assistant = messages.find((m) => m.role === 'assistant') as any + expect(assistant.tool_calls[0].function.arguments).toBe('{"path": "u/admin/app"}') + const toolResult = messages.find((m) => m.role === 'tool') as any + expect(toolResult.content).toBe('tool ok') + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index ffa4e5db24..323b552cca 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -14,10 +14,11 @@ import Anthropic from '@anthropic-ai/sdk' import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' -import { requiresMaxCompletionTokens } from './modelConfig' +import { requiresMaxCompletionTokens, usesAnthropicMessagesApi } from './modelConfig' import { applyReasoningToConfig } from './reasoningRegistry' import { formatResourceTypes } from './utils' import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' +import { hasValidToolCallArguments } from './chat/toolCallArguments' import { getNonStreamingOpenAIResponsesCompletion, getOpenAIResponsesCompletionStream @@ -62,22 +63,10 @@ export const AI_PROVIDERS: Record = { label: 'OpenAI', defaultModels: OPENAI_MODELS }, - azure_openai: { - label: 'Azure OpenAI', - defaultModels: OPENAI_MODELS - }, anthropic: { label: 'Anthropic', defaultModels: ['claude-sonnet-4-6', 'claude-3-5-haiku-latest'] }, - mistral: { - label: 'Mistral', - defaultModels: ['codestral-latest'] - }, - deepseek: { - label: 'DeepSeek', - defaultModels: ['deepseek-v4-pro', 'deepseek-chat', 'deepseek-reasoner'] - }, googleai: { label: 'Google AI', defaultModels: [ @@ -89,6 +78,29 @@ export const AI_PROVIDERS: Record = { 'gemini-3.1-flash-lite' ] }, + azure_openai: { + label: 'Azure OpenAI', + defaultModels: OPENAI_MODELS + }, + azure_foundry: { + label: 'Azure AI Foundry', + defaultModels: [ + 'gpt-4o', + 'gpt-4o-mini', + 'DeepSeek-R1', + 'Llama-3.3-70B-Instruct', + 'Phi-4', + 'Mistral-Large-2411' + ] + }, + mistral: { + label: 'Mistral', + defaultModels: ['codestral-latest'] + }, + deepseek: { + label: 'DeepSeek', + defaultModels: ['deepseek-v4-pro', 'deepseek-chat', 'deepseek-reasoner'] + }, groq: { label: 'Groq', defaultModels: ['llama-3.3-70b-versatile', 'llama-3.1-8b-instant'] @@ -267,7 +279,10 @@ export async function fetchAvailableModels( export function getModelMaxTokens(provider: AIProvider, model: string) { if (model.includes('gpt-5')) { return 128000 - } else if ((provider === 'azure_openai' || provider === 'openai') && model.startsWith('o')) { + } else if ( + (provider === 'azure_openai' || provider === 'openai' || provider === 'azure_foundry') && + model.startsWith('o') + ) { return 100000 } else if ( model.includes('claude-sonnet') || @@ -287,7 +302,6 @@ export function getModelMaxTokens(provider: AIProvider, model: string) { return 8192 } - function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[] @@ -302,7 +316,9 @@ function getModelSpecificConfig( } const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens if ( - (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && + (modelProvider.provider === 'openai' || + modelProvider.provider === 'azure_openai' || + modelProvider.provider === 'azure_foundry') && requiresMaxCompletionTokens(modelProvider.model) ) { return { @@ -352,6 +368,7 @@ const DEFAULT_COMPLETION_CONFIG: ChatCompletionCreateParams = { export const PROVIDER_COMPLETION_CONFIG_MAP: Record = { openai: DEFAULT_COMPLETION_CONFIG, azure_openai: DEFAULT_COMPLETION_CONFIG, + azure_foundry: DEFAULT_COMPLETION_CONFIG, groq: DEFAULT_COMPLETION_CONFIG, openrouter: DEFAULT_COMPLETION_CONFIG, togetherai: DEFAULT_COMPLETION_CONFIG, @@ -449,15 +466,19 @@ export async function testKey({ throw new Error('Missing a model to test') } - // Use Anthropic SDK for Anthropic provider - if (aiProvider === 'anthropic') { + // Providers served through the Anthropic Messages API (native Anthropic, and + // Claude deployments on Azure Foundry) must use the Anthropic SDK path rather + // than OpenAI chat completions. Mirrors the chat loop's routing so the test + // key exercises the same request shape the chat actually sends. + if (usesAnthropicMessagesApi(aiProvider, modelToTest)) { await testAnthropicKey({ apiKey, workspace, resourcePath, model: modelToTest, abortController, - messages + messages, + aiProvider }) return } @@ -479,7 +500,8 @@ async function testAnthropicKey({ resourcePath, model, abortController, - messages + messages, + aiProvider }: { apiKey?: string workspace?: string @@ -487,11 +509,15 @@ async function testAnthropicKey({ model: string abortController: AbortController messages: ChatCompletionMessageParam[] + aiProvider: AIProvider }) { const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) + // X-Provider must be the real provider (e.g. azure_foundry) so the backend + // resolves the right credentials and Anthropic URL; the SDK headers tell it to + // route through the Anthropic Messages API. const headers: Record = { - 'X-Provider': 'anthropic', + 'X-Provider': aiProvider, 'anthropic-version': '2023-06-01', 'X-Anthropic-SDK': 'true' } @@ -906,7 +932,10 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const completionConfig = applyReasoningToConfig( - (provider === 'openai' || provider === 'azure_openai' || provider === 'googleai') && + (provider === 'openai' || + provider === 'azure_openai' || + provider === 'azure_foundry' || + provider === 'googleai') && config.stream ? { ...config, @@ -1094,11 +1123,14 @@ export async function parseOpenAICompletion( } if (toolCalls.length > 0) { + const invalidToolCallIds = new Set( + toolCalls.filter((t) => !hasValidToolCallArguments(t.function.arguments)).map((t) => t.id) + ) const normalizedToolCalls = toolCalls.map((t) => ({ ...t, function: { ...t.function, - arguments: t.function.arguments || '{}' + arguments: invalidToolCallIds.has(t.id) ? '{}' : t.function.arguments || '{}' } })) const toAdd = buildAssistantToolCallMessage({ @@ -1113,6 +1145,22 @@ export async function parseOpenAICompletion( messages.push(toAdd) addedMessages.push(toAdd) for (const toolCall of toolCalls) { + if (invalidToolCallIds.has(toolCall.id)) { + callbacks.setToolStatus(toolCall.id, { + isLoading: false, + isStreamingArguments: false, + error: 'Tool call arguments were invalid or truncated' + }) + const messageToAdd = { + role: 'tool' as const, + tool_call_id: toolCall.id, + content: + 'The tool call arguments were invalid or truncated JSON, so the tool was NOT executed. Retry the call; if the arguments were long, split the work into several smaller calls.' + } + messages.push(messageToAdd) + addedMessages.push(messageToAdd) + continue + } const messageToAdd = await processToolCall({ tools, toolCall, diff --git a/frontend/src/lib/components/copilot/modelConfig.test.ts b/frontend/src/lib/components/copilot/modelConfig.test.ts new file mode 100644 index 0000000000..1788d9c365 --- /dev/null +++ b/frontend/src/lib/components/copilot/modelConfig.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { usesAnthropicMessagesApi } from './modelConfig' + +describe('usesAnthropicMessagesApi', () => { + it('routes the native Anthropic provider through the Messages API', () => { + expect(usesAnthropicMessagesApi('anthropic', 'claude-sonnet-5')).toBe(true) + }) + + it('routes Azure Foundry Claude deployments through the Messages API', () => { + expect(usesAnthropicMessagesApi('azure_foundry', 'claude-sonnet-5')).toBe(true) + expect(usesAnthropicMessagesApi('azure_foundry', 'Claude-Opus-4-8')).toBe(true) + }) + + it('keeps other Azure Foundry models on the OpenAI-compatible path', () => { + expect(usesAnthropicMessagesApi('azure_foundry', 'gpt-4o')).toBe(false) + expect(usesAnthropicMessagesApi('azure_foundry', 'DeepSeek-R1')).toBe(false) + }) + + it('does not affect other providers', () => { + expect(usesAnthropicMessagesApi('openai', 'gpt-4o')).toBe(false) + expect(usesAnthropicMessagesApi('azure_openai', 'gpt-4o')).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index 4ab08ce02c..af0dd942b2 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -1,3 +1,17 @@ +import type { AIProvider } from '$lib/gen' + +// Azure AI Foundry fronts multiple model families under one resource. Claude +// deployments are served only through the Anthropic Messages API, so the chat must +// route them like the native Anthropic provider (Anthropic SDK, message format) +// rather than the OpenAI-compatible surface used for the rest of Foundry's catalog. +// Mirrors the backend `AIProvider::is_anthropic_model`. +export function usesAnthropicMessagesApi(provider: AIProvider, model: string): boolean { + return ( + provider === 'anthropic' || + (provider === 'azure_foundry' && model.toLowerCase().startsWith('claude')) + ) +} + // gpt-5+ and o-series reasoning models reject the legacy `max_tokens` field on // the OpenAI/Azure Chat Completions API and require `max_completion_tokens` // instead. The check strips any provider prefix (e.g. OpenRouter's "openai/o3") diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts index 42f0006a1d..2b27ac9d7b 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts @@ -243,6 +243,36 @@ describe('supportsReasoning (static registry)', () => { }) }) +describe('Azure AI Foundry reasoning follows the model family', () => { + it('treats Foundry Claude deployments like the Anthropic provider', () => { + // Live-verified: Foundry Claude accepts the adaptive-thinking effort ladder. + expect(supportsReasoning('azure_foundry', 'claude-sonnet-5')).toBe(true) + expect(supportsReasoning('azure_foundry', 'claude-opus-4-8')).toBe(true) + expect(getReasoningCapability('azure_foundry', 'claude-opus-4-8').levels).toEqual([ + 'low', + 'medium', + 'high', + 'xhigh', + 'max' + ]) + // Off is achieved by omission (Foundry rejects effort 'none'), like Anthropic. + expect(getReasoningCapability('azure_foundry', 'claude-sonnet-5').canDisable).toBe(true) + expect( + resolveRequestReasoning({ + provider: 'azure_foundry', + model: 'claude-sonnet-5', + reasoning: REASONING_OFF + }) + ).toBeUndefined() + }) + + it('treats Foundry OpenAI deployments like the OpenAI provider', () => { + expect(supportsReasoning('azure_foundry', 'gpt-5.1')).toBe(true) + expect(supportsReasoning('azure_foundry', 'gpt-4o')).toBe(false) + expect(supportsReasoning('azure_foundry', 'DeepSeek-R1')).toBe(false) + }) +}) + describe('resolveEffectiveReasoning', () => { it('defaults capable models to high when unset', () => { expect(resolveEffectiveReasoning({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe( diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index e1ce2467f2..519c1a4021 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -1,4 +1,5 @@ import type { AIProvider, AIProviderModel } from '$lib/gen' +import { usesAnthropicMessagesApi } from './modelConfig' /** * Reasoning effort is provider/model-specific. We never normalize a single @@ -38,6 +39,20 @@ function baseModelId(model: string): string { return normalized.split('/').pop() ?? normalized } +/** + * Azure AI Foundry hosts multiple model families under one provider, so reasoning + * support follows the underlying model rather than the provider: Claude deployments + * reason like the native Anthropic provider (adaptive thinking + `output_config.effort`), + * everything else (gpt-5 / o-series) like OpenAI. Resolving to the owning family here + * lets the rest of the registry keep its per-family logic unchanged. + */ +function reasoningProviderFamily(provider: AIProvider, model: string): AIProvider { + if (provider === 'azure_foundry') { + return usesAnthropicMessagesApi(provider, model) ? 'anthropic' : 'openai' + } + return provider +} + /** * Suggested effort levels per provider, sourced from each provider SDK's own * vocabulary. @@ -143,14 +158,16 @@ function anthropicReasoningLevels(model: string): ReasoningEffort[] { function supportsReasoningStatic(provider: AIProvider, model: string): boolean { const m = model.toLowerCase() const base = baseModelId(model) - switch (provider) { + switch (reasoningProviderFamily(provider, model)) { case 'anthropic': // Bedrock serves the same Claude models under prefixed ids // (e.g. `us.anthropic.claude-opus-4-6-v1`), so match on the full string. case 'aws_bedrock': // 4.6+ only: Opus 4.5 rejects adaptive thinking (and, on Bedrock, // the whole output_config surface) — live-verified hard 400. - return /claude-opus-4-(6|7|8)/.test(m) || /claude-sonnet-4-6/.test(m) || m.includes('fable') + return ( + /claude-opus-4-(6|7|8)/.test(m) || /claude-sonnet-(4-6|5)/.test(m) || m.includes('fable') + ) case 'openai': case 'azure_openai': return base.startsWith('gpt-5') || /^o\d/.test(base) @@ -204,16 +221,17 @@ export function getReasoningCapability(provider: AIProvider, model: string): Rea if (!supported) { return { supported: false, levels: [], canDisable: false } } + const family = reasoningProviderFamily(provider, bareModel) const levels = - provider === 'anthropic' || provider === 'aws_bedrock' + family === 'anthropic' || family === 'aws_bedrock' ? anthropicReasoningLevels(bareModel) - : provider === 'googleai' + : family === 'googleai' ? geminiReasoningLevels(bareModel) - : provider === 'openai' || provider === 'azure_openai' + : family === 'openai' || family === 'azure_openai' ? openaiReasoningLevels(bareModel) - : provider === 'openrouter' + : family === 'openrouter' ? openrouterReasoningLevels(bareModel) - : (PROVIDER_REASONING_LEVELS[provider] ?? ['low', 'medium', 'high']) + : (PROVIDER_REASONING_LEVELS[family] ?? ['low', 'medium', 'high']) return { supported, levels, canDisable: canDisableReasoning(provider, bareModel) } } @@ -226,7 +244,7 @@ export function getReasoningCapability(provider: AIProvider, model: string): Rea function canDisableReasoning(provider: AIProvider, model: string): boolean { const m = model.toLowerCase() const base = baseModelId(model) - switch (provider) { + switch (reasoningProviderFamily(provider, model)) { case 'anthropic': case 'aws_bedrock': // Claude 4.6+ only think when asked, so omission is a real off — @@ -301,8 +319,8 @@ export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' * model that reasons *by default* — omitting the field would silently keep * the default-on behavior. Undefined means omission is the correct off. */ -function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { - switch (provider) { +export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { + switch (reasoningProviderFamily(provider, model)) { case 'googleai': // Gemini 2.5/3 think by default (dynamic budget / level). The backend // proxy maps 'none' to off on Flash, or the floor on Pro (only diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte new file mode 100644 index 0000000000..4d8adb88e9 --- /dev/null +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -0,0 +1,582 @@ + + +
+ + + {#if $open && active} +
+ {#if showDoc} + +
+
+
+ +
+
+
+

{active.label}

+ {#if active.badge} + + {active.badge.label} + + {/if} +
+

{active.tagline}

+
+
+ +

{active.description}

+ +
    + {#each active.bullets as bullet (bullet)} +
  • + + {bullet} +
    • + {/each} +
+ + +
+ {/if} + + +
+ {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} +
+ +
+ + {option.label} + + {#if option.badge} + + {option.badge.label} + + {/if} + {/snippet} + {#each allOptions as option (option.key)} + {@const ac = accentClasses[option.accent]} + {@const rowClass = + 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} + {#if option.variants} + + {#if $wacSubOpen} +
+ {#each option.variants ?? [] as variant (variant.label)} + {@const VariantIcon = variant.icon} + + {/each} +
+ {/if} + {:else} + + {/if} + {/each} + + +
+ + {#if $importSubOpen} +
+ {#each importActions as action (action.label)} + + {/each} +
+ {/if} + + {#if !showDoc} + + {/if} +
+
+ {/if} +
+ + + + importDrawer?.closeDrawer?.()}> + + + + {#snippet content()} +
+ {#key importType} + {#await import('$lib/components/SimpleEditor.svelte')} + + {:then Module} + + {/await} + {/key} +
+ {/snippet} + + {#snippet actions()} + + {/snippet} + + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 256b2277f5..49a0dc62ee 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -13,6 +13,7 @@ type ListableRawApp } from '$lib/gen' import { resource } from 'runed' + import { getDraftItems } from '$lib/workspaceDrafts.svelte' import { userStore, workspaceStore } from '$lib/stores' import type uFuzzy from '@leeoniya/ufuzzy' import { @@ -73,23 +74,44 @@ type TableApp = TableItem type TableRawApp = TableItem - // Folders with ≥1 pipeline script (auto_kind='pipeline'). Used by - // TreeView to surface a "Pipeline" entry inside those folders. Cheap - // thanks to the partial index on script.auto_kind. + // Folders that are data pipelines, surfaced as their own "Pipeline" entry + // (the member scripts are folded into it, not listed individually). Two + // sources: deployed pipelines (folders with ≥1 `auto_kind='pipeline'` script, + // cheap via the partial index) AND bundle-phase pipelines that only exist as a + // `data_pipeline` draft so far — so a pipeline shows up the moment its first + // node is drafted, before anything is deployed. let pipelineFoldersRes = resource( () => $workspaceStore, async (ws) => { if (!ws) return new Set() + const folders = new Set() try { - const rows = await AssetService.listPipelineFolders({ workspace: ws }) - return new Set(rows.map((r) => r.folder)) + for (const r of await AssetService.listPipelineFolders({ workspace: ws })) + folders.add(r.folder) } catch { - // Decorative tree entry — degrade to "no pipelines" on failure. - return new Set() + // Decorative entry — degrade gracefully on failure. } + try { + for (const d of await getDraftItems(ws)) { + if (d.kind !== 'data_pipeline') continue + const m = d.path.match(/^f\/([^/]+)\/data_pipeline$/) + if (m) folders.add(m[1]) + } + } catch { + // Drafts unavailable — show deployed pipelines only. + } + return folders } ) - let pipelineFolders = $derived(pipelineFoldersRes.current ?? new Set()) + // Folders of pipeline-member scripts present in the current listing (captured + // in loadScripts before they're filtered out). Unioned in so a folder whose + // only pipeline node is a never-deployed `// pipeline` script draft — not in + // listPipelineFolders (deployed-only) nor a `data_pipeline` bundle — still gets + // a pipeline entry instead of vanishing. + let pipelineMemberFolders = $state(new Set()) + let pipelineFolders = $derived( + new Set([...(pipelineFoldersRes.current ?? []), ...pipelineMemberFolders]) + ) let scripts: TableScript[] | undefined = $state() let flows: TableFlow[] | undefined = $state() @@ -115,12 +137,26 @@ withoutDescription: true }) - scripts = loadedScripts.map((script: Script) => { - return { - canWrite: canWrite(script.path, script.extra_perms, $userStore) && !$userStore?.operator, - ...script - } - }) + // Pipeline-member scripts (`auto_kind='pipeline'`) are represented by their + // pipeline's entry, not listed individually — but capture their folders so + // the pipeline entry still surfaces (incl. a members-only / draft-only folder). + const memberFolders = new Set() + scripts = loadedScripts + .filter((script: Script) => { + if (script.auto_kind === 'pipeline') { + const m = script.path.match(/^f\/([^/]+)\//) + if (m) memberFolders.add(m[1]) + return false + } + return true + }) + .map((script: Script) => { + return { + canWrite: canWrite(script.path, script.extra_perms, $userStore) && !$userStore?.operator, + ...script + } + }) + pipelineMemberFolders = memberFolders loading = false } @@ -239,13 +275,32 @@ const INCLUDE_WITHOUT_MAIN_SETTING_NAME = 'includeWithoutMain' let treeView = $state(getLocalSetting(TREE_VIEW_SETTING_NAME) == 'true') let filterUserFoldersType: 'only f/*' | 'u/username and f/*' | undefined = $derived( - $userStore?.is_super_admin && $userStore.username.includes('@') + $userStore?.non_member ? 'only f/*' : $userStore?.is_admin || $userStore?.is_super_admin ? 'u/username and f/*' : undefined ) let filterUserFolders = $state(getLocalSetting(FILTER_USER_FOLDER_SETTING_NAME) == 'true') + + // Pipeline entries are rendered independently of the item list, so apply the + // same gates the items get — otherwise a pipeline would still show under the + // Flows/Apps tabs, in the archived view, under a label filter, or outside a + // selected owner. Pipelines are script-based units always at `f/`, so + // kind=script and the user-folder toggle always include them; kind=flow/app, + // archived, a label filter (pipelines carry no labels), and a non-matching + // owner exclude them. + let visiblePipelineFolders = $derived.by(() => { + if (archived) return new Set() + if (itemKind !== 'all' && itemKind !== 'script') return new Set() + if (labelFilter != undefined) return new Set() + if (ownerFilter == undefined) return pipelineFolders + return new Set( + [...pipelineFolders].filter( + (f) => `f/${f}` === ownerFilter || `f/${f}`.startsWith(ownerFilter + '/') + ) + ) + }) let includeWithoutMain = $state( getLocalSetting(INCLUDE_WITHOUT_MAIN_SETTING_NAME) ? getLocalSetting(INCLUDE_WITHOUT_MAIN_SETTING_NAME) == 'true' @@ -506,7 +561,8 @@ if (menuItem) { if (e.key === 'ArrowUp' || e.key === 'ArrowDown') { const menu = menuItem.closest('[role="menu"]') - if (menu) { + // menus marked data-arrow-loop keep melt's cyclic wrap instead of exiting + if (menu && !menu.hasAttribute('data-arrow-loop')) { const items = Array.from(menu.querySelectorAll('[role="menuitem"]')) const idx = items.indexOf(menuItem) const isFirst = idx === 0 @@ -825,14 +881,17 @@ {#each new Array(6) as _} {/each} - {:else if filteredItems.length === 0} + {:else if filteredItems.length === 0 && (filter !== '' || visiblePipelineFolders.size === 0)} + {:else if treeView} loadScripts(includeWithoutMain)} on:flowChanged={loadFlows} @@ -849,7 +908,7 @@ {:else}
{#if filter === ''} - {#each [...pipelineFolders].sort() as folder (folder)} + {#each [...visiblePipelineFolders].sort() as folder (folder)} i && 'folderName' in i + // Hidden while searching: pipelines aren't part of the text filter (the list + // view hides their rows on a query too), so a folder matching the search + // shouldn't surface an unrelated Pipeline row. let hasPipeline = $derived( - depth === 0 && isFolderItem(item) && (pipelineFolders?.has(item.folderName) ?? false) + depth === 0 && + !isSearching && + isFolderItem(item) && + (pipelineFolders?.has(item.folderName) ?? false) ) const isFolder = isFolderItem diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index 59adf1aaa9..ef5cf25d58 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -24,7 +24,42 @@ let groupedItems: ReturnType | 'loading' = $state('loading') $effect(() => { items - untrack(() => (groupedItems = groupItems(items))) + pipelineFolders + isSearching + untrack(() => { + const grouped = groupItems(items) + // Ensure every pipeline folder is present at the top level so its + // "Pipeline" entry shows even when it has no listed items — a bundle-phase + // pipeline (only a draft so far) or a folder whose only scripts are + // pipeline members (folded into the pipeline, hidden from the list). + // Skip while searching: pipelines aren't part of the text filter (list view + // hides them on `filter !== ''`), so injecting them would surface unrelated + // folders in the results. + if (!isSearching) { + const present = new Set( + grouped + .filter((g) => 'folderName' in g) + .map((g) => (g as { folderName: string }).folderName) + ) + // Insert each missing pipeline folder among the existing folders in name + // order — `groupItems` already sorts user groups first then folders + // alphabetically, so inserting before the first greater-named folder + // keeps that ordering (rather than prepending out of order). + for (const folderName of [...(pipelineFolders ?? [])] + .filter((f) => !present.has(f)) + .sort()) { + const item = { folderName, items: [] } + const idx = grouped.findIndex( + (g) => + 'folderName' in g && + (g as { folderName: string }).folderName.localeCompare(folderName) > 0 + ) + if (idx < 0) grouped.push(item) + else grouped.splice(idx, 0, item) + } + } + groupedItems = grouped + }) }) diff --git a/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte b/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte index c297b1a32f..f7607c5d6c 100644 --- a/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte @@ -5,10 +5,6 @@ smtpSettings.smtp_host && smtpSettings.smtp_host.trim() !== '' && smtpSettings.smtp_port && - smtpSettings.smtp_username && - smtpSettings.smtp_username.trim() !== '' && - smtpSettings.smtp_password && - smtpSettings.smtp_password.trim() !== '' && smtpSettings.smtp_from && smtpSettings.smtp_from.trim() !== '' ) diff --git a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte index 2d7bed8333..eaf9be651b 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte @@ -9,7 +9,13 @@ import { UserDraft } from '$lib/userDraft.svelte' import { discardDraftAfterDeploy } from '$lib/userDraftToast' import { rawAppToHubUrl } from '$lib/hub' - import { enterpriseLicense, hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' + import { + enterpriseLicense, + hubBaseUrlStore, + userStore, + userWorkspaces, + workspaceStore + } from '$lib/stores' import YAML from 'yaml' import { Bug, @@ -70,8 +76,7 @@ import { AIBtnClasses } from '../copilot/chat/AIButtonStyle' import { stripRawAppDiffNoise } from './utils' import type { RawAppData } from './dataTableRefUtils' - import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' - import { buildForkEditUrl } from '$lib/utils/editInFork' + import { buildForkEditUrl, editInForkAllowed, editInForkLabel } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' // async function hash(message) { @@ -890,10 +895,10 @@ window.open(`/apps/add?template=${appPath}`) } }, - ...(!isCloudHosted() && !isRuleActive('DisableWorkspaceForking') + ...(!isCloudHosted() && editInForkAllowed($workspaceStore, $userWorkspaces) ? [ { - label: 'Edit in workspace fork', + label: editInForkLabel($workspaceStore, $userWorkspaces), onClick: () => { window.open(buildForkEditUrl('raw_app', appPath)) } diff --git a/frontend/src/lib/components/runs/UpstreamSnapshotsPanel.svelte b/frontend/src/lib/components/runs/UpstreamSnapshotsPanel.svelte new file mode 100644 index 0000000000..ee265d5874 --- /dev/null +++ b/frontend/src/lib/components/runs/UpstreamSnapshotsPanel.svelte @@ -0,0 +1,82 @@ + + +{#if snapshots.length > 0} +
+

+ Upstream snapshots + + Version of each upstream asset when this run was dispatched. Recorded for debugging only — + the run reads the latest data, not these versions. To inspect what this run saw, query the + asset with the copied AT (VERSION => n) clause. For partitioned + assets, the partition shown is the slice whose write produced that snapshot — the snapshot itself + covers the whole table. + +

+
+
+ + + + + + + + + {#each snapshots as s (s.asset)} + + + + + + {/each} + +
AssetSnapshotTime travel
+ {s.asset} + + @ {s.snapshot_id} + {#if s.partition} + + · partition {s.partition} + {/if} + + +
+ + +{/if} diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 19e71530e6..3e631a55e9 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -505,12 +505,16 @@ time: new Date(x.edited_at).getTime(), search_id: x.path })), - ...scripts.map((x) => ({ - ...x, - type: 'script' as 'script', - time: new Date(x.created_at).getTime(), - search_id: x.path - })), + // Pipeline-member scripts (`auto_kind='pipeline'`) are reached through + // their pipeline, not searched individually. + ...scripts + .filter((x) => x.auto_kind !== 'pipeline') + .map((x) => ({ + ...x, + type: 'script' as 'script', + time: new Date(x.created_at).getTime(), + search_id: x.path + })), ...apps.map((x) => ({ ...x, type: 'app' as 'app', diff --git a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte deleted file mode 100644 index 0716398131..0000000000 --- a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte +++ /dev/null @@ -1,118 +0,0 @@ - - - buildEditUrl(d as unknown as WorkspaceItemDiff, workspaceId)} -> - {#snippet titleExtra()} -
- - {ws?.name ?? workspaceId} -
- {/snippet} - diff --git a/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte b/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte deleted file mode 100644 index e03fc5a49c..0000000000 --- a/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte +++ /dev/null @@ -1,125 +0,0 @@ - - - buildEditUrl(d as unknown as WorkspaceItemDiff, forkWorkspaceId)} -> - {#snippet titleExtra()} -
- - {forkWs?.name ?? forkWorkspaceId} - - {parentWs?.name ?? parentWorkspaceId} - {#if comparison} - - {comparison.summary.total_diffs} item{comparison.summary.total_diffs !== 1 ? 's' : ''} - - {#if comparison.summary.conflicts > 0} - - - {comparison.summary.conflicts} conflict{comparison.summary.conflicts !== 1 ? 's' : ''} - - {/if} - {/if} -
- {/snippet} - diff --git a/frontend/src/lib/components/sessions/PipelineEditorView.svelte b/frontend/src/lib/components/sessions/PipelineEditorView.svelte new file mode 100644 index 0000000000..0e3be64e36 --- /dev/null +++ b/frontend/src/lib/components/sessions/PipelineEditorView.svelte @@ -0,0 +1,391 @@ + + +
+
+ + f/{path} + · data pipeline +
+
+ + {#if graphRes.loading && !graphRes.current && pe.drafts.size === 0} +
+ + Loading pipeline… +
+ {:else if graphRes.error && pe.drafts.size === 0} +
+ Failed to load pipeline: {graphRes.error.message} +
+ {:else} + + runNode(path, args)} + canRunByPath + onTestStateChange={(running) => { + const openPath = pe.openScriptPath + if (running && openPath) { + activeRunnable = { kind: 'script', path: openPath } + activeRunnables.arm(`script:${openPath}`) + activeRunnableJobId = undefined + } else if (!running && activeRunnable?.path === openPath) { + // Only clear the hint for the script the pane just finished — a + // canvas per-node run of a different script keeps its own hint. + activeRunnable = undefined + activeRunnableJobId = undefined + } + }} + onRunCompleted={() => { + activeRunnable = undefined + activeRunnableJobId = undefined + }} + onSelect={handleCanvasSelect} + onDraftSaved={afterSaved} + onPersistedSaved={afterSaved} + onScriptRemoved={async (removedPath) => { + pe.forgetPath(removedPath) + await graphRes.refetch() + }} + onScriptRenamed={async (oldPath, newPath) => { + // Repoint the selection so the canvas follows the renamed node instead + // of staying on the now-gone old path until an unrelated refetch. + if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) { + pe.selection = { ...pe.selection, path: newPath } + } + await graphRes.refetch() + }} + onDiscard={() => { + if (pe.activeDraftPath) pe.discardDraft(pe.activeDraftPath) + }} + onClose={() => { + pe.selection = undefined + pe.activeDraftPath = undefined + pe.clearLiveOverlays() + }} + /> + {/if} +
+
+ + + graphRes.refetch()} +/> diff --git a/frontend/src/lib/components/sessions/SessionChangesBar.svelte b/frontend/src/lib/components/sessions/SessionChangesBar.svelte new file mode 100644 index 0000000000..9f59d1f320 --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionChangesBar.svelte @@ -0,0 +1,264 @@ + + +{#snippet dock()} + +
+ {#if dockCounts.draft > 0} + diffDrawer?.open()}> + {dockCounts.draft} draft{dockCounts.draft === 1 ? '' : 's'} + + {/if} + {#if dockCounts.deployed > 0} + diffDrawer?.open()}> + {dockCounts.deployed} deployed + + {/if} + {#if deletionOnly && compareHref} + + + Review deletions on compare page + + + {/if} +
+{/snippet} + +{#if committedId && isUnavailable} + +
+
+ +
+ The {committedIsFork ? 'fork' : 'workspace'} has been archived or deleted + + Move this session to another workspace, or discard it. + {committedId} + +
+
+
+ onMove?.(workspaceId)} + onCreateFork={async (fork) => { + await onCreateForkAndMove?.(fork) + }} + createForkCaption="Created immediately and the session moved into it." + > + {#snippet trigger()} + + {/snippet} + + +
+
+{:else if showBar && committedId} + +
+
+ + Edits +
+ {@render dock()} +
+{/if} + + +{#if committedId && !isUnavailable} + + void runtime?.manager.renameModifiedItem(item.draftKind, item.path, item.displayPath)} + onItemDiscarded={(item) => void runtime?.manager.removeModifiedItem(item.draftKind, item.path)} + /> +{/if} diff --git a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte new file mode 100644 index 0000000000..1a6692048c --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte @@ -0,0 +1,107 @@ + + + + {#snippet titleExtra()} +
+ {#if isFork} + + + {ws?.name ?? workspaceId} + + + + {parentWs?.name ?? parentWorkspaceId} + + {:else} + + + {ws?.name ?? workspaceId} + + {/if} +
+ {/snippet} + diff --git a/frontend/src/lib/components/sessions/SessionDraftBar.svelte b/frontend/src/lib/components/sessions/SessionDraftBar.svelte deleted file mode 100644 index d23070151a..0000000000 --- a/frontend/src/lib/components/sessions/SessionDraftBar.svelte +++ /dev/null @@ -1,77 +0,0 @@ - - -{#if committedId && count > 0} -
-
- - - {count} draft{count === 1 ? '' : 's'} - {#snippet text()} - Tracks all unsaved draft changes in this workspace — including edits made outside this - chat (e.g. in the editor), not only changes made by the assistant. - {/snippet} - -
-
- drawer?.open()} /> - -
-
- - -{/if} diff --git a/frontend/src/lib/components/sessions/SessionForkBar.svelte b/frontend/src/lib/components/sessions/SessionForkBar.svelte deleted file mode 100644 index 18d36a755c..0000000000 --- a/frontend/src/lib/components/sessions/SessionForkBar.svelte +++ /dev/null @@ -1,209 +0,0 @@ - - -{#if committedId && isUnavailable} - -
-
- -
- The {committedIsFork ? 'fork' : 'workspace'} has been archived or deleted - - Move this session to another workspace, or discard it. - {committedId} - -
-
-
- onMove?.(workspaceId)} - onCreateFork={async (fork) => { - await onCreateForkAndMove?.(fork) - }} - createForkCaption="Created immediately and the session moved into it." - > - {#snippet trigger()} - - {/snippet} - - -
-
-{:else if forksAllowed && isFork && sessionWorkspace && parentWorkspace && parentWorkspaceId && committedId} - {@const StatusIcon = - forkStatus === 'ahead' - ? GitPullRequestArrow - : forkStatus === 'diverged' - ? GitCompareArrows - : GitFork} - {@const statusColor = - forkStatus === 'ahead' - ? 'text-blue-500' - : forkStatus === 'diverged' - ? 'text-amber-500' - : 'text-secondary'} - {@const statusTitle = - forkStatus === 'ahead' - ? 'Ahead of parent' - : forkStatus === 'diverged' - ? 'Diverged from parent' - : forkStatus === 'in_sync' - ? 'In sync with parent' - : 'Fork'} -
-
- - - - - {sessionWorkspace.name} - - - - {parentWorkspace.name} - -
-
- diffDrawer?.open()} - /> - -
-
- - -{/if} diff --git a/frontend/src/lib/components/sessions/SessionItemNotFound.svelte b/frontend/src/lib/components/sessions/SessionItemNotFound.svelte index dc1a088c53..77c5173813 100644 --- a/frontend/src/lib/components/sessions/SessionItemNotFound.svelte +++ b/frontend/src/lib/components/sessions/SessionItemNotFound.svelte @@ -3,7 +3,11 @@ import type { WorkspaceItem, WorkspaceItemKind } from '$lib/components/workspacePicker' import type { SessionTarget } from './sessionState.svelte' - const KIND_NOT_FOUND_LABEL: Record = { + // `pipeline` targets never hit this component (they aren't slot-loaded, so they + // can't 404 through SessionEditorTarget) — exclude it from the kinds here. + type NotFoundKind = Exclude + + const KIND_NOT_FOUND_LABEL: Record = { flow: 'Flow', script: 'Script', raw_app: 'Raw app' @@ -14,7 +18,7 @@ path, onNavigate }: { - kind: SessionTarget['kind'] + kind: NotFoundKind path: string onNavigate?: (item: WorkspaceItem) => void } = $props() diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index a3273ddceb..0342fd53e1 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -21,7 +21,6 @@ import { slide } from 'svelte/transition' import { createSession, - deriveForkStatus, deleteSessionsForWorkspace, isForkSession, reconcileAfterWorkspaceChange, @@ -48,19 +47,11 @@ import DropdownV2 from '$lib/components/DropdownV2.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' import { userWorkspaces, workspaceStore } from '$lib/stores' + import { workspaceIsFork } from '$lib/utils/workspaceHierarchy' import { WorkspaceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { currentWorkspaceRootId, workspaceRootId } from './sessionScope.svelte' - // Look up the cached fork comparison for a session through its runtime - // (if any). The deriveForkStatus helper handles the "no runtime yet" - // and "comparison not loaded" cases by returning undefined; we render - // a neutral fork icon in that interim, then upgrade to the proper - // status icon once the comparison lands. - function forkStatusFor(session: Session) { - return deriveForkStatus(session, $userWorkspaces, getRuntime(session.id)?.forkComparison.val) - } - function isForkFor(session: Session): boolean { return isForkSession(session, $userWorkspaces) } @@ -123,8 +114,8 @@ } // Flat list passing the archive + scope filters. Grouping for display happens - // in `sessionGroups`; this flat view drives the runtime / fork-comparison - // effects, the unread total, and keyboard navigation. + // in `sessionGroups`; this flat view drives the runtime effect, the unread + // total, and keyboard navigation. const visibleSessions = $derived( sessionState.sessions.filter((s) => { if (s.transient) return false @@ -210,23 +201,6 @@ } }) - // Pre-fetch the fork comparison for every visible fork session so the - // sidebar icons reflect the right ahead/diverged state without - // requiring the user to click into each session. Cheap enough at - // typical session counts; falls back to a plain dot until the - // fetch lands. - $effect(() => { - if (sectionCollapsed.val) return - for (const session of visibleSessions) { - if (!session.workspace_id) continue - const ws = $userWorkspaces.find((w) => w.id === session.workspace_id) - if (!ws?.parent_workspace_id) continue - const rt = getRuntime(session.id) - if (!rt) continue - void rt.ensureForkComparison(ws.parent_workspace_id, session.workspace_id) - } - }) - function isUnavailableFork(session: Session): boolean { return !!session.workspace_id && !$userWorkspaces.find((w) => w.id === session.workspace_id) } @@ -240,10 +214,6 @@ if (!isUnavailableFork(session)) { syncWorkspaceTo(session.workspace_id) } - // Refresh the fork diff count — users typically click back into a - // session after editing items elsewhere in the SPA, where neither - // the visibility-change nor the AI-loading signal would fire. - void getRuntime(session.id)?.refreshForkComparison() await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) if (restoreFocus) { // goto() resets focus to — put it back on the active session button @@ -287,9 +257,12 @@ // Fork workspace tied to `pendingDelete`, if any, and still accessible. const pendingDeleteForkId = $derived.by(() => { const wsId = pendingDelete?.workspace_id - if (!wsId || !wsId.startsWith('wm-fork-')) return undefined + if (!wsId) return undefined const ws = $userWorkspaces.find((w) => w.id === wsId) - if (!ws || !ws.parent_workspace_id) return undefined + // Fork = prefix OR parent (so an orphaned wm-fork- fork still qualifies); exclude persistent + // dev workspaces, which are not ephemeral session forks. + if (!ws || !workspaceIsFork(wsId, $userWorkspaces)) return undefined + if (ws.is_dev_workspace) return undefined return wsId }) @@ -432,7 +405,7 @@ {session.summary ?? 'Untitled session'} {#if draft || unread > 0} diff --git a/frontend/src/lib/components/sessions/SessionStatusDot.svelte b/frontend/src/lib/components/sessions/SessionStatusDot.svelte index b47816ab19..35028c80cc 100644 --- a/frontend/src/lib/components/sessions/SessionStatusDot.svelte +++ b/frontend/src/lib/components/sessions/SessionStatusDot.svelte @@ -3,19 +3,16 @@ AlertCircle, AlertTriangle, Building, - GitCompareArrows, GitFork, - GitPullRequestArrow, GitPullRequestClosed } from 'lucide-svelte' import type { SessionChatStatus } from './sessionRuntime.svelte' - import type { ForkStatus } from './sessionState.svelte' let { status, isFork, - forkStatus - }: { status: SessionChatStatus; isFork: boolean; forkStatus?: ForkStatus } = $props() + unavailable = false + }: { status: SessionChatStatus; isFork: boolean; unavailable?: boolean } = $props() const statusTooltip: Record = { idle: 'No chat activity', @@ -26,13 +23,6 @@ error: 'Last message had an error' } - const forkTooltip: Record = { - in_sync: 'Fork — in sync with parent', - ahead: 'Fork — ahead of parent', - diverged: 'Fork — diverged from parent', - unavailable: 'Fork — no longer available' - } - // Live chat signals take precedence over the persistent kind/fork // indicator: streaming, needs-confirmation, and error are time-critical // and warrant briefly hijacking the icon slot. @@ -41,7 +31,11 @@ ) const persistentTitle = $derived( - isFork ? (forkStatus ? forkTooltip[forkStatus] : 'Fork session') : 'Root workspace session' + isFork + ? unavailable + ? 'Fork — no longer available' + : 'Fork session' + : 'Root workspace session' ) const title = $derived(liveOverride ? statusTooltip[status] : persistentTitle) @@ -59,11 +53,7 @@ {:else if status === 'error'} {:else if isFork} - {#if forkStatus === 'ahead'} - - {:else if forkStatus === 'diverged'} - - {:else if forkStatus === 'unavailable'} + {#if unavailable} {:else} diff --git a/frontend/src/lib/components/sessions/SessionWorkspaceBar.svelte b/frontend/src/lib/components/sessions/SessionWorkspaceBar.svelte index 310d43e5be..07f7f08329 100644 --- a/frontend/src/lib/components/sessions/SessionWorkspaceBar.svelte +++ b/frontend/src/lib/components/sessions/SessionWorkspaceBar.svelte @@ -8,6 +8,7 @@ type Session } from './sessionState.svelte' import WorkspaceFamilyPicker from './WorkspaceFamilyPicker.svelte' + import { Badge } from '$lib/components/common' import { Building, ChevronDown, GitFork } from 'lucide-svelte' let { session }: { session: Session } = $props() @@ -65,6 +66,9 @@ {pendingFork?.name ?? currentWs?.name ?? effectiveId ?? 'Pick workspace'} + {#if !pendingFork && currentWs?.is_dev_workspace} + dev + {/if} {#if pendingFork} (new) {/if} diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index 027089d4ae..ee8cbed6e0 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -8,6 +8,7 @@ import DropdownV2 from '$lib/components/DropdownV2.svelte' import { AIChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte' import { userWorkspaces, workspaceStore } from '$lib/stores' + import { workspaceIsFork } from '$lib/utils/workspaceHierarchy' import { WorkspaceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import Toggle from '$lib/components/Toggle.svelte' @@ -27,9 +28,9 @@ import FlowEditorView from './FlowEditorView.svelte' import ScriptEditorView from './ScriptEditorView.svelte' import RawAppEditorView from './RawAppEditorView.svelte' + import PipelineEditorView from './PipelineEditorView.svelte' import SessionWorkspaceBar from './SessionWorkspaceBar.svelte' - import SessionForkBar from './SessionForkBar.svelte' - import SessionDraftBar from './SessionDraftBar.svelte' + import SessionChangesBar from './SessionChangesBar.svelte' import { createSession, deleteSessionsForWorkspace, @@ -91,10 +92,13 @@ // the fork lingers as an orphan whose only purpose was this session. const sessionForkId = $derived.by(() => { const wsId = session?.workspace_id - if (!wsId || !wsId.startsWith('wm-fork-')) return undefined + if (!wsId) return undefined const ws = $userWorkspaces.find((w) => w.id === wsId) - // Don't offer the option if the fork is gone or not user-accessible. - if (!ws || !ws.parent_workspace_id) return undefined + // Don't offer the option if the fork is gone/not user-accessible or isn't a fork (prefix OR + // parent, so an orphaned wm-fork- fork still qualifies). + if (!ws || !workspaceIsFork(wsId, $userWorkspaces)) return undefined + // A persistent dev workspace is not an ephemeral session fork — never offer to delete it. + if (ws.is_dev_workspace) return undefined return wsId }) @@ -230,7 +234,7 @@ // True when the session committed to a workspace that's no longer in // the user's list (deleted / archived / access revoked). The chat is - // disabled and SessionForkBar shows a move/discard banner. + // disabled and SessionChangesBar shows a move/discard banner. const isUnavailable = $derived( !!session?.workspace_id && !$userWorkspaces.find((w) => w.id === session!.workspace_id) ) @@ -261,23 +265,24 @@ {@const hasTarget = session.target?.kind === 'flow' || session.target?.kind === 'script' || - session.target?.kind === 'raw_app'} + session.target?.kind === 'raw_app' || + session.target?.kind === 'pipeline'} {@const hasEditor = mountEditor && hasTarget && editorVisible} {#snippet inputPreface()} {#if !hasFirstUserMessage} {/if} - +
{#if session.archived && !isUnavailable}
{/if} - moveAndActivate(workspaceId)} onCreateForkAndMove={(fork) => createForkAndMove(fork)} onArchive={() => archiveAndReset()} onDelete={() => (deleteConfirmOpen = true)} /> -
{/snippet} @@ -469,6 +473,13 @@ onNavigate={pickEditorTarget} isActiveSession={sessionState.currentSessionId === sessionId} /> + {:else if session.target.kind === 'pipeline'} + {/if} diff --git a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte index c4099edd91..50c6653b0a 100644 --- a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte @@ -1,40 +1,19 @@ - - searchableText(d)} + items={displayEntries} + bind:filteredItems={searchedEntries} + f={(e: DisplayEntry) => searchableText(e)} /> -{#snippet renderTreeNode(node: TreeNode, depth: number)} + +{#snippet rowBadge(item: DeployItem)} + {#if badgeOf(item) === 'draft'} + {#if model.staleOf(item.key)} + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+ Started from an older deployed version. A newer version was deployed after this draft + began. Review the latest deploy before deploying. +
+ {/snippet} + + {/if} + + {item.draftOnly ? 'Draft only' : 'Draft'} + + {:else} + + + + + + + {/if} +{/snippet} + + +{#snippet deployFailed(item: DeployItem)} + {@const s = model.statusOf(item.key)} + {#if s?.status === 'failed'} + + Failed + + + {/if} +{/snippet} + +{#snippet renderTreeNode(node: TreeNode, depth: number)} {#if node.type === 'folder'} {@const isUserScope = node.isScope && node.name.startsWith('u/')} {@const fkey = node.key} @@ -411,36 +613,62 @@ {@const isHl = fkey === highlightedKey}
(folderOpen[fkey] = (e.currentTarget as HTMLDetailsElement).open)} + ontoggle={(e) => { + // Record real user toggles only; skip the echo fired when the `open` + // attribute is driven by state (search force-open, expandApp). + const domOpen = (e.currentTarget as HTMLDetailsElement).open + if (domOpen !== isFolderOpen(fkey)) folderOpen[fkey] = domOpen + }} class="select-none" > {#if node.app} - {@const appSummary = summaries[node.app.summaryKey] ?? node.app.summary} - + {@const appItem = segmentItems.find((it) => it.key === node.app?.summaryKey)} + setHoverHighlight(fkey)} + onclick={(e) => { + e.preventDefault() + if (appItem) revealDiff(appItem, fkey) + }} title={node.fullPath} - class="flex items-center gap-1.5 px-3 py-1.5 cursor-pointer hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl + class="flex items-center gap-2 pl-3 pr-1 py-2 rounded-md cursor-pointer hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl ? 'bg-surface-hover' : ''}" style="padding-left: {depth * 12 + 8}px" > - {appSummary ?? node.name} + {node.app.summary ?? node.name} - - + {#if appItem} + {@render rowBadge(staged[appItem.key] ?? appItem)} + {/if} + + {:else} setHoverHighlight(fkey)} - class="flex items-center gap-1.5 px-3 py-1.5 cursor-pointer text-xs font-normal font-mono text-secondary hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl + class="flex items-center gap-2 pl-3 pr-1 py-2 rounded-md cursor-pointer text-xs font-normal font-mono text-secondary hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl ? 'bg-surface-hover' : ''}" style="padding-left: {depth * 12 + 8}px" @@ -459,14 +687,13 @@ {/if} {node.name} - - + + + + {/if}
-
- { - highlightedKey = key - scrollToDiff(d) - }} - onmouseenter={() => setHoverHighlight(key)} + +
+ {#if isSynthetic(d)} + void revealSynthetic(d, key)} + onmouseenter={() => setHoverHighlight(key)} + > + {#snippet extras()} + + {/snippet} + + {:else} + revealDiff(d, key)} + onmouseenter={() => setHoverHighlight(key)} + > + {#snippet extras()} + {@render rowBadge(staged[d.key] ?? d)} + {#if d.deployKind === 'raw_app'} + + { + e.stopPropagation() + expandApp(d) + }} + > + {#if loadedDiffs[d.key]?.state === 'loading'} + + {:else} + + {/if} + + {:else} + + {/if} + {/snippet} + + {/if} +
+ {/if} +{/snippet} + + +{#snippet diffBlock(item: DeployItem)} + {@const loaded = loadedDiffs[item.key]} + {#if !mountedRows[item.key]} +
+ + Diff loads on scroll… +
+ {:else if !loaded || loaded.state === 'loading'} +
+ + Loading diff… +
+ {:else if loaded.state === 'error'} +
{loaded.error}
+ {:else if item.deployKind === 'raw_app'} +
+ {#each rawAppItems(item, loaded) as sub (displayKey(sub))} + +
+
+ {sub.path} +
+ {#if sub.kind === 'raw_app_file'} + + {:else} + {@const runnable = sub as RawAppRunnableItem} + + {/if} +
+ {/each} +
+ {:else} + {/if} {/snippet} @@ -534,178 +869,215 @@ /> {/snippet} - {/snippet} -
- {#if diffs.length > 0} - - {/if} -
-
- {#if loading && diffs.length === 0} -
- - Loading comparison... +
+
+ {#if model.items.length > 0} + + {/if} +
+
+ {#if model.loading && model.items.length === 0} +
+ + Loading changes... +
+ {:else if model.error} +
{model.error}
+ {:else if model.items.length === 0} +
No changes.
+ {:else if orderedItems.length === 0} +
No files match.
+ {:else} +
+ {#each orderedItems as d (d.key)} + + {@const view = staged[d.key] ?? d} + {@const action = actionFor(view)} + {@const editUrl = editUrlFor?.(d)} + {@const status = model.statusOf(d.key)} +
- - -
- {#if editUrl} - + +
+ {#if editUrl} + + {d.displayPath} + + {:else} +
+ {d.displayPath} +
+ {/if} +
+
+ {@render rowBadge(view)} + {#if status?.status === 'failed'} + {@render deployFailed(d)} + {:else if action.op !== 'none'} + {#if action.secondary?.length} + + {/if} +
+ + {#if staged[d.key]} + +
+ +
+ {/if} +
+ {/if} +
+
+
+ + {#if view.done} +
- {dpath} - + Deployed — no pending changes. +
{:else} -
- {dpath} +
+ {@render diffBlock(view)}
{/if}
-
- {#if d.ahead && d.ahead > 0} - {d.ahead} ahead - {/if} - {#if d.behind && d.behind > 0} - {d.behind} behind - {/if} - - - {status} - -
- -
- {#if !mountedRows[key]} - -
- - Diff loads on scroll… -
- {:else if d.kind === 'raw_app_file'} - - {@const rawFile = d as RawAppFileItem} - - {:else if 'appPath' in d} - - {@const runnable = d as RawAppRunnableItem} - - {:else if !loaded || loaded.state === 'loading'} -
- - Loading diff… -
- {:else if loaded.state === 'error'} -
{loaded.error}
- {:else if loaded.state === 'ready'} - - {/if}
-
- {/each} - - {/if} - + {/each} + + + + {/if} + + + + + {#if model.items.length > 0 && compareSessionHref} +
+ + Deploy all / request review on compare page + + +
+ {/if} + +