diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index 3468449753..9fa13bca58 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -139,8 +139,8 @@ jobs: run: | cargo build --release -p windmill_duckdb_ffi_internal New-Item -ItemType Directory -Path ..\target\debug -Force - Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\ - # duckdb is bundled (~2GB of build artifacts); the DLL is the only + Copy-Item target\release\windmill_duckdb_runner.exe ..\target\debug\ + # duckdb is bundled (~2GB of build artifacts); the runner is the only # thing we need from this excluded-crate target dir. Remove-Item -Recurse -Force -ErrorAction SilentlyContinue target Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 73f955131e..b01e99a696 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -83,9 +83,16 @@ jobs: chmod +x "$HOME/.local/bin/wmill" echo "$HOME/.local/bin" >> $GITHUB_PATH working-directory: / - - name: Install PowerShell, mold and clang + - name: Install PowerShell, mold, clang and the latest pg_dump + # PGDG's `postgresql-client` is its newest major, Ubuntu's is not: the `postgres` service + # image floats, and pg_dump refuses a server newer than itself. `/usr/bin/pg_dump` picks + # the version of the runner's own preinstalled cluster, so the newest bin dir goes first. run: | - sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev postgresql-client + PG_BIN="/usr/lib/postgresql/$(ls /usr/lib/postgresql | sort -n | tail -1)/bin" + echo "$PG_BIN" >> "$GITHUB_PATH" + "$PG_BIN/pg_dump" --version working-directory: / - uses: actions-rust-lang/setup-rust-toolchain@v1 with: diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index dcb223b534..f037e7eb75 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -83,17 +83,17 @@ jobs: # Extract arm64 binary (include deps/ for hard link resolution) mkdir -p /tmp/arm64 crane export --platform linux/arm64 ${{ steps.meta-ee-public.outputs.tags }} - \ - | tar -xf - -C /tmp/arm64 windmill/target/release/ usr/src/app/libwindmill_duckdb_ffi_internal.so + | tar -xf - -C /tmp/arm64 windmill/target/release/ usr/src/app/windmill_duckdb_runner cp /tmp/arm64/windmill/target/release/windmill extracted/windmill-ee-arm64-rhel9 - cp /tmp/arm64/usr/src/app/libwindmill_duckdb_ffi_internal.so extracted/libwindmill_duckdb_ffi_internal-arm64.so + cp /tmp/arm64/usr/src/app/windmill_duckdb_runner extracted/windmill_duckdb_runner-arm64 rm -rf /tmp/arm64 # Extract amd64 binary mkdir -p /tmp/amd64 crane export --platform linux/amd64 ${{ steps.meta-ee-public.outputs.tags }} - \ - | tar -xf - -C /tmp/amd64 windmill/target/release/ usr/src/app/libwindmill_duckdb_ffi_internal.so + | tar -xf - -C /tmp/amd64 windmill/target/release/ usr/src/app/windmill_duckdb_runner cp /tmp/amd64/windmill/target/release/windmill extracted/windmill-ee-amd64-rhel9 - cp /tmp/amd64/usr/src/app/libwindmill_duckdb_ffi_internal.so extracted/libwindmill_duckdb_ffi_internal-amd64.so + cp /tmp/amd64/usr/src/app/windmill_duckdb_runner extracted/windmill_duckdb_runner-amd64 rm -rf /tmp/amd64 - uses: actions/upload-artifact@v4 @@ -109,9 +109,9 @@ jobs: - uses: actions/upload-artifact@v4 with: name: RHEL9-arm64 dynamic libraries build - path: extracted/libwindmill_duckdb_ffi_internal-arm64.so + path: extracted/windmill_duckdb_runner-arm64 - uses: actions/upload-artifact@v4 with: name: RHEL9-amd64 dynamic libraries build - path: extracted/libwindmill_duckdb_ffi_internal-amd64.so + path: extracted/windmill_duckdb_runner-amd64 diff --git a/.github/workflows/build-publish-rh8-image.yml b/.github/workflows/build-publish-rh8-image.yml index 7c820eb290..83b12d3290 100644 --- a/.github/workflows/build-publish-rh8-image.yml +++ b/.github/workflows/build-publish-rh8-image.yml @@ -103,7 +103,7 @@ jobs: id: extract-duckdb-ffi-internal with: image: ${{ steps.meta-ee-public.outputs.tags}}-amd64 - path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so" + path: "/usr/src/app/windmill_duckdb_runner" # - uses: shrink/actions-docker-extract@v3 # id: extract-ee-arm64 @@ -124,7 +124,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: RHEL8-amd64 dynamic libraries build - path: ${{ steps.extract-duckdb-ffi-internal.outputs.destination }}/libwindmill_duckdb_ffi_internal.so + path: ${{ steps.extract-duckdb-ffi-internal.outputs.destination }}/windmill_duckdb_runner # - uses: actions/upload-artifact@v4 # with: diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 46befe42b6..4a6368ee1e 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -81,5 +81,5 @@ jobs: - name: Upload dynamic libraries artifact uses: actions/upload-artifact@v4 with: - name: windmill_duckdb_ffi_internal.dll - path: ./backend/windmill-duckdb-ffi-internal/target/release/windmill_duckdb_ffi_internal.dll + name: windmill_duckdb_runner.exe + path: ./backend/windmill-duckdb-ffi-internal/target/release/windmill_duckdb_runner.exe diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c94521d332..7e05675495 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -218,7 +218,7 @@ jobs: id: extract-duckdb-ffi-internal with: image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} - path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so" + path: "/usr/src/app/windmill_duckdb_runner" - uses: shrink/actions-docker-extract@v3 id: extract-ee diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index b94d5d57c7..ecceff03ab 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -72,8 +72,8 @@ jobs: files: | ./backend/target/release/windmill-ee.exe - - name: Attach dynamic libraries to release + - name: Attach the DuckDB runner to release uses: softprops/action-gh-release@v2 with: files: | - ./backend/windmill-duckdb-ffi-internal/target/release/windmill_duckdb_ffi_internal.dll + ./backend/windmill-duckdb-ffi-internal/target/release/windmill_duckdb_runner.exe diff --git a/.release-please-manifest.json b/.release-please-manifest.json index e47ce1f0a3..a533c73dbf 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.833.0" + ".": "1.835.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 0edb6b5dcd..57ec310ab9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## [1.835.0](https://github.com/windmill-labs/windmill/compare/v1.834.0...v1.835.0) (2026-10-14) + + +### Features + +* allow early return on steps inside branches ([#11733](https://github.com/windmill-labs/windmill/issues/11733)) ([08319a0](https://github.com/windmill-labs/windmill/commit/08319a004c289f93398b9fdc790b2ed412a81fa4)) +* run global ai eval cases under builder operator profiles ([#11768](https://github.com/windmill-labs/windmill/issues/11768)) ([d788f16](https://github.com/windmill-labs/windmill/commit/d788f16c4028985afd2b82f35d625b0a12c04653)) +* search the workspace picker dropdown past 8 workspaces ([#11761](https://github.com/windmill-labs/windmill/issues/11761)) ([c1a561f](https://github.com/windmill-labs/windmill/commit/c1a561f5ec091de0ae2de12ee811ca628c58c9f5)) + + +### Bug Fixes + +* show native triggers in the operator sidebar ([#11769](https://github.com/windmill-labs/windmill/issues/11769)) ([717162e](https://github.com/windmill-labs/windmill/commit/717162eab44757182a4a67316c288e188f811599)) + +## [1.834.0](https://github.com/windmill-labs/windmill/compare/v1.833.0...v1.834.0) (2026-10-14) + + +### Features + +* add a navigation picker after the breadcrumb's last segment ([#11649](https://github.com/windmill-labs/windmill/issues/11649)) ([9653318](https://github.com/windmill-labs/windmill/commit/9653318c17225b35c474722d49f38dc2ecd3b9f6)) +* add a yaml mode to the workspace settings page ([#11751](https://github.com/windmill-labs/windmill/issues/11751)) ([b543b43](https://github.com/windmill-labs/windmill/commit/b543b43d734a309278b72fdd958570b9c7ff6d5f)) +* give full-code app previews the deployed app's window.ctx ([#11763](https://github.com/windmill-labs/windmill/issues/11763)) ([c5e6f67](https://github.com/windmill-labs/windmill/commit/c5e6f67d5307146569e2fd1a37033c9dddb680c4)) +* guide the AI when no resource type fits a service ([#11615](https://github.com/windmill-labs/windmill/issues/11615)) ([88c3cc0](https://github.com/windmill-labs/windmill/commit/88c3cc0ced8fe3183979f0abd8e05846f6f4a8e6)) +* let a linked agent step override model and reasoning effort ([#11685](https://github.com/windmill-labs/windmill/issues/11685)) ([9249aa0](https://github.com/windmill-labs/windmill/commit/9249aa02d592e60024d7475627cbf664936e4a5d)) +* let operators with manage rights save schedule and trigger drafts ([#11646](https://github.com/windmill-labs/windmill/issues/11646)) ([74c5f82](https://github.com/windmill-labs/windmill/commit/74c5f82c6703a08a6d1a896b5e46b7007f2e3d73)) +* list agents with runnables, let them be starred, and fill out their menu ([#11651](https://github.com/windmill-labs/windmill/issues/11651)) ([fe2571e](https://github.com/windmill-labs/windmill/commit/fe2571ea53603625f48601aa800e9f700322d5b8)) +* list the user's own forks first in fork pickers ([#11762](https://github.com/windmill-labs/windmill/issues/11762)) ([009482a](https://github.com/windmill-labs/windmill/commit/009482a67747ec9e8f2b6b7baddb5e0e9fe720c5)) +* read and change workspace settings from ai sessions and mcp ([#11741](https://github.com/windmill-labs/windmill/issues/11741)) ([468cf1d](https://github.com/windmill-labs/windmill/commit/468cf1d9941538d285264e8a8c57356b934ab25d)) +* scope data table migrations to the schemas they declare ([#11756](https://github.com/windmill-labs/windmill/issues/11756)) ([2410790](https://github.com/windmill-labs/windmill/commit/2410790c7e343920ae2bb6f2405a0b7f7660be7f)) +* **sessions:** run a deployed item from its own page, for the chat ([#11468](https://github.com/windmill-labs/windmill/issues/11468)) ([998e687](https://github.com/windmill-labs/windmill/commit/998e6877c1cf07bb1a08fd1d3b18948fbc0e30e0)) +* workspace settings as one document, applied server-side ([#11740](https://github.com/windmill-labs/windmill/issues/11740)) ([9a5ec22](https://github.com/windmill-labs/windmill/commit/9a5ec223bc7dd1b5c4e58a9cc883a5d4e5d9237b)) + + +### Bug Fixes + +* **cli:** keep script run waiting on long jobs and flaky log streaming ([#11766](https://github.com/windmill-labs/windmill/issues/11766)) ([a2ee052](https://github.com/windmill-labs/windmill/commit/a2ee052e0326bf69f49596cdc008bbc8ca596e3c)) +* fork data tables sharing one database into a single copy ([#11755](https://github.com/windmill-labs/windmill/issues/11755)) ([4914782](https://github.com/windmill-labs/windmill/commit/49147827d69bc241c57407f539ec7e2621830d1b)) +* keep applied migrations and db settings in data table copies ([#11753](https://github.com/windmill-labs/windmill/issues/11753)) ([694ed7a](https://github.com/windmill-labs/windmill/commit/694ed7a0eba255ef42e6cfe988ea3b8c5784dfa1)) +* unify cron builder control sizes and widths ([#11764](https://github.com/windmill-labs/windmill/issues/11764)) ([17fefcf](https://github.com/windmill-labs/windmill/commit/17fefcf777063849ca8ec1e097bf02e826c0b3a7)) +* **worker:** keep script env vars off the nsjail launcher process ([#11749](https://github.com/windmill-labs/windmill/issues/11749)) ([1c068f3](https://github.com/windmill-labs/windmill/commit/1c068f33e144af81f202e21e23960af4a17a6b4a)) + ## [1.833.0](https://github.com/windmill-labs/windmill/compare/v1.832.0...v1.833.0) (2026-10-13) diff --git a/Dockerfile b/Dockerfile index 4e022d4b53..80de6d4d16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,10 +50,7 @@ RUN apt-get update && apt-get install -y clang=1:19.0* libclang-dev=1:19.0* cmak COPY ./backend/windmill-duckdb-ffi-internal . -# The `duckdb` crate comes from a git dependency (a fork carrying an engine patch), -# which cargo checks out under $CARGO_HOME/git rather than the registry cache. RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \ cargo build --release -p windmill_duckdb_ffi_internal @@ -286,7 +283,8 @@ ENV TZ=Etc/UTC COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill -COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so +# DuckDB jobs run in this binary, which the worker looks for next to its own. +COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/windmill_duckdb_runner ${APP}/windmill_duckdb_runner COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun diff --git a/ai_evals/README.md b/ai_evals/README.md index 9b8850c435..d458e019d0 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -184,6 +184,17 @@ dismissed, so `userAnswers: []` dismisses every question. Without it the tool re it cannot ask, and the model usually carries on with its own guess, which makes "asks before creating anything" impossible to assert. +`runtime.profile` runs a session-chat case (`sessionChat: true`) as a non-admin role +instead of full access: the prompt, toolset and tool schemas are cut as production's +`capabilitiesForRole` cuts them, and the workspace's `operator_settings` are seeded for the +tools that read them directly: + +```yaml +profile: + operator: true + operatorSettings: { builder_apps: true } # also builder_flows, manage_schedules, manage_triggers +``` + `toolExpect.toolCallArgs` entries support `sharedByAtLeast: `: at least `n` recorded calls to that tool must carry the same non-blank string in the field. Use it for calls that have to share an identifier, like two test runs of one chat conversation. diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index fe20d77d05..e70dbfcfe5 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "fs/promises"; import { tmpdir } from "os"; import { join } from "path"; import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; -import type { AIProvider } from "$lib/gen/types.gen"; +import type { AIProvider, OperatorSettings } from "$lib/gen/types.gen"; import { globalToolsFor, prepareGlobalSystemMessage, @@ -13,6 +13,10 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { capabilitiesForRole } from "../../../../../frontend/src/lib/components/copilot/chat/global/sessionAccess"; +import { filterSessionTools } from "../../../../../frontend/src/lib/components/copilot/chat/global/sessionToolset"; +import { usersWorkspaceStore } from "../../../../../frontend/src/lib/stores"; +import { resolveResourceTypeSync } from "../../../../../frontend/src/lib/components/copilot/chat/script/core"; import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; import { createEvalPlanTools } from "./planModeTools"; @@ -26,7 +30,7 @@ import { type EvalPreviewPanel, type EvalPreviewTabFixture, } from "./evalPreviewTabs"; -import type { ModeRunContext } from "../../../../core/types"; +import type { EvalCaseProfile, ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; import { @@ -121,6 +125,8 @@ export interface GlobalEvalOptions { // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan // tools are offered. Needs sessionChat, which is what plan mode is gated on in production. planMode?: boolean; + // The user's rights, resolved as a session resolves them; unset keeps full access. + profile?: EvalCaseProfile; model?: string; maxIterations?: number; userAnswers?: string[]; @@ -149,6 +155,16 @@ export async function runGlobalEval( options.workspaceFixtures ?? {}, ); seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + // No deploy rules in the mock workspace, so they always pass. + const access = + options.profile && + capabilitiesForRole({ + isAdmin: false, + operator: options.profile.operator, + deployRulesPass: true, + operatorSettings: options.profile.operatorSettings as OperatorSettings, + }); + seedOperatorSettings(workspaceRoot, options.profile); // Declared out here only so `finally` can reach it; a malformed fixture throws while // building it, and everything seeded above still has to be torn down. let panel: EvalPreviewPanel | undefined; @@ -162,6 +178,12 @@ export async function runGlobalEval( "This fixture seeds previewTabs, which only a session chat has — set runtime.sessionChat: true on the case.", ); } + // Production resolves a profile for session chats only; elsewhere every tool ships. + if (!options.sessionChat && access) { + throw new Error( + "This case sets runtime.profile, which only a session chat applies — set runtime.sessionChat: true on the case.", + ); + } if (options.sessionChat) { panel = createEvalPreviewPanel({ sessionId: evalArtifacts.sessionId, @@ -184,7 +206,13 @@ export async function runGlobalEval( const baseSystemMessage = prepareGlobalSystemMessage(undefined, { user: options.user, previewTools: options.sessionChat ?? false, + access, }); + const helpers = { + ...evalArtifacts.helpers, + access, + ...(panel && { openArtifact: panel.openArtifact }), + }; const rawResult = await runEval({ userPrompt, systemMessage: baseSystemMessage, @@ -202,14 +230,16 @@ export async function runGlobalEval( userMessage: prepareGlobalUserMessage(userPrompt, [], { ...(injectActiveEditorContext ? { workspace: workspaceRoot } : {}), activePreview: panel?.activePreview(), + resourceTypeSync: await resolveResourceTypeSync(workspaceRoot, []), }), tools: [ - ...getGlobalEvalTools(options.sessionChat ?? false), + ...filterSessionTools( + getGlobalEvalTools(options.sessionChat ?? false), + access, + ), ...(planMode?.tools ?? []), ], - helpers: panel - ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact } - : evalArtifacts.helpers, + helpers, apiKey, getOutput: async () => ({ ...(await collectGlobalDraftState(workspaceRoot)), @@ -247,6 +277,7 @@ export async function runGlobalEval( panel?.dispose(); clearGlobalDrafts(workspaceRoot); clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + seedOperatorSettings(workspaceRoot, undefined); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); if (!options.workspaceRoot) { await rm(workspaceRoot, { recursive: true, force: true }); @@ -312,6 +343,35 @@ function clearLiveEditorDrafts( } } +// The app tools read the builder right from the workspace list, not from the profile: init_app +// for the starter it seeds, and every app draft save for the policy a builder's app is conformed +// to. Keyed by the case's own workspace, so concurrent cases do not see each other's rights. +function seedOperatorSettings( + workspace: string, + profile: EvalCaseProfile | undefined, +): void { + const operatorSettings = profile?.operator && profile.operatorSettings; + usersWorkspaceStore.update((list) => { + const workspaces = (list?.workspaces ?? []).filter( + (w) => w.id !== workspace, + ); + if (operatorSettings) { + workspaces.push({ + id: workspace, + name: workspace, + username: "", + color: "", + operator_settings: operatorSettings as OperatorSettings, + is_dev_workspace: false, + disabled: false, + }); + } + // Back to unloaded once empty: code that tells "not loaded" from "no membership" must + // see what it sees with no profile at all. + return workspaces.length ? { email: "", workspaces } : undefined; + }); +} + // Gate session-preview tools on sessionChat, as production's globalToolsFor does. function getGlobalEvalTools(sessionChat: boolean): ProductionTool<{}>[] { const disableSearchApp = process.env[DISABLE_SEARCH_APP_ENV] === "1"; diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index d8284039b9..69398555f0 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import confluenceFixture from '../../fixtures/frontend/global/hub/confluence.json' import servicenowFixture from '../../fixtures/frontend/global/hub/servicenow.json' import outreachFixture from '../../fixtures/frontend/global/hub/outreach.json' +import syncedResourceTypesFixture from '../../fixtures/frontend/global/hub/synced_resource_types.json' import type { AppWithLastVersion, CompletedJob, @@ -98,6 +99,13 @@ export interface BenchmarkWorkspaceResource { description?: string } +/** A resource type the instance holds. */ +export interface BenchmarkWorkspaceResourceType { + name: string + schema?: unknown + description?: string +} + export interface BenchmarkWorkspaceJob { /** Stable id so a case prompt can reference a specific run (e.g. for get_run). */ id?: string @@ -118,6 +126,8 @@ export interface BenchmarkWorkspaceRunnables { variables?: BenchmarkWorkspaceVariable[] aiProviders?: BenchmarkWorkspaceAiProvider[] resources?: BenchmarkWorkspaceResource[] + /** Unset means an instance synced with the hub. */ + resourceTypes?: BenchmarkWorkspaceResourceType[] datatables?: BenchmarkDatatableSeed[] /** DuckLake catalog names, as `list_ducklakes` reports them. */ ducklakes?: string[] @@ -328,6 +338,57 @@ export function listBenchmarkPlainResources(workspace: string): ListableResource })) } +function benchmarkResourceTypes(workspace: string): BenchmarkWorkspaceResourceType[] { + return benchmarkWorkspaceRunnables.get(workspace)?.resourceTypes ?? syncedResourceTypes() +} + +/** Resource type names of a benchmark workspace, as `ResourceService.listResourceTypeNames` + * returns them. Null when the workspace is not a benchmark one. */ +export function listBenchmarkResourceTypeNames(workspace: string): string[] | null { + if (!benchmarkWorkspaces.has(workspace)) { + return null + } + return benchmarkResourceTypes(workspace).map((rt) => rt.name) +} + +/** Adds a resource type to a benchmark workspace, as `ResourceService.createResourceType` does, + * so later searches in the same case find it. */ +export function createBenchmarkResourceType( + workspace: string, + resourceType: BenchmarkWorkspaceResourceType +): void { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + throw new Error(`Workspace "${workspace}" is not a benchmark workspace`) + } + const existing = benchmarkResourceTypes(workspace) + if (existing.some((rt) => rt.name === resourceType.name)) { + throw new Error(`Resource type ${resourceType.name} already exists`) + } + runnables.resourceTypes = [...existing, resourceType] +} + +/** Stands in for the resource type search: a type matches when a word of the query appears in + * its name or description, ranked by how many do. */ +export function queryBenchmarkResourceTypes( + workspace: string, + text: string, + limit = 10 +): { name: string; schema?: unknown; score: number }[] { + const words = text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((word) => word.length >= 3) + return benchmarkResourceTypes(workspace) + .map((rt) => { + const haystack = `${rt.name} ${rt.description ?? ''}`.toLowerCase() + return { name: rt.name, schema: rt.schema, score: words.filter((w) => haystack.includes(w)).length } + }) + .filter((rt) => rt.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, limit) +} + /** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI * providers too, and a case that lists resources and then reads one by path would otherwise get @@ -1468,6 +1529,42 @@ const BENCHMARK_HUB_INTEGRATION_META: Record< } } +/** The resource type a mocked hub integration takes, as its metadata serves it. */ +function benchmarkHubIntegrationResourceType(app: string): BenchmarkWorkspaceResourceType | null { + const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app) + if (real) { + return real.resource_type + } + const entry = BENCHMARK_HUB_INTEGRATION_META[app] + if (!entry) { + return null + } + return { + name: app, + description: `${entry.display_name} credentials`, + schema: { + type: 'object', + required: ['apiKey'], + properties: { + apiKey: { type: 'string', description: `${entry.display_name} API key` } + } + } + } +} + +/** What a hub-synced instance holds: common hub types, plus the one each mocked hub + * integration takes, so the hub and the instance agree on them. */ +function syncedResourceTypes(): BenchmarkWorkspaceResourceType[] { + const integrations = [ + ...REAL_HUB_INTEGRATIONS.map((integration) => integration.app), + ...Object.keys(BENCHMARK_HUB_INTEGRATION_META) + ] + return [ + ...syncedResourceTypesFixture, + ...integrations.flatMap((app) => benchmarkHubIntegrationResourceType(app) ?? []) + ] +} + /** Mirrors the hub's own derivation: hosts seen in the shipped scripts, whether they * call the provider directly, and how many there are of each kind. */ function benchmarkDerivedFacts(app: string) { @@ -1602,7 +1699,7 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon meta: real.meta, meta_updated_at: null, derived: benchmarkDerivedFacts(app), - resource_types: [{ id: 1, ...real.resource_type }] + resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }] }) } const entry = BENCHMARK_HUB_INTEGRATION_META[app] @@ -1619,23 +1716,7 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon meta: entry.meta, meta_updated_at: null, derived: benchmarkDerivedFacts(app), - resource_types: [ - { - id: 1, - name: app, - description: `${entry.display_name} credentials`, - schema: { - type: 'object', - required: ['apiKey'], - properties: { - apiKey: { - type: 'string', - description: `${entry.display_name} API key` - } - } - } - } - ] + resource_types: [{ id: 1, ...benchmarkHubIntegrationResourceType(app) }] }) } if (path === '/api/integrations/hub/list') { diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index cb7469b45e..b24ccbe5e4 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -111,6 +111,9 @@ vi.mock('$lib/gen', async () => { getBenchmarkResource, listBenchmarkAiProviderResources, listBenchmarkPlainResources, + listBenchmarkResourceTypeNames, + queryBenchmarkResourceTypes, + createBenchmarkResourceType, listBenchmarkApps, listBenchmarkDatatables, listBenchmarkDataMetrics, @@ -487,8 +490,23 @@ vi.mock('$lib/gen', async () => { } return value }, - queryResourceTypes: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data) + queryResourceTypes: async (data: { workspace: string; text: string; limit?: number }) => + hasBenchmarkWorkspace(data.workspace) + ? queryBenchmarkResourceTypes(data.workspace, data.text, data.limit) + : actual.ResourceService.queryResourceTypes(data), + listResourceTypeNames: async (data: { workspace: string }) => + listBenchmarkResourceTypeNames(data.workspace) ?? + actual.ResourceService.listResourceTypeNames(data), + createResourceType: async (data: { + workspace: string + requestBody: { name: string; schema?: unknown; description?: string } + }) => { + if (!hasBenchmarkWorkspace(data.workspace)) { + return actual.ResourceService.createResourceType(data) + } + createBenchmarkResourceType(data.workspace, data.requestBody) + return `resource_type ${data.requestBody.name} created` + } }), VariableService: wrapService(actual.VariableService, { existsVariable: async (data: { workspace: string; path: string }) => diff --git a/ai_evals/cases/cli.yaml b/ai_evals/cases/cli.yaml index b7abcf7f14..fdf469130d 100644 --- a/ai_evals/cases/cli.yaml +++ b/ai_evals/cases/cli.yaml @@ -85,6 +85,22 @@ - updates the step code so the returned greeting uses punctuation - defaults punctuation to an exclamation mark when omitted +# An in-house API has no hub resource type, so its connection needs a type of the workspace's +# own: a token parameter or a variable read inside the code is the fallback the guidance rules out. +- id: cli-resource-type-custom-api + prompt: |- + Create a Windmill Bun script at `f/evals/get_invoice.ts` that fetches an invoice by id from our internal billing API at https://billing.acme.internal/api/v1. It authenticates with a bearer token. + cliExpect: + requiredSkills: + - write-script-bun + forbiddenExecutedCommands: + - ^wmill sync push + judgeChecklist: + - creates the script at f/evals/get_invoice.ts, fetching an invoice by id with a bearer token + - the script takes the billing API's base URL and token together as one resource-typed parameter, not a plain string token or a variable read inside the code + - defines the billing API's resource type with its base URL and token fields, as a resource-type file or in the reply (asking before writing the file is fine) + - the custom resource type's name has the c_ prefix (e.g. `RT.CAcmeBilling` in TypeScript, `c_acme_billing.resource-type.json`) + - id: flow-reuse-existing-script prompt: |- There is already a reusable greeting script at `f/lib/format_greeting.ts`. diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index 3a4f203b48..7454a85c9c 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -16,6 +16,16 @@ - "the main step is named `sum_numbers`" - the flow returns the sum of the two numbers +# An in-house API has no hub resource type: the connection still comes in as one resource, not +# a token input or a variable read inside the code. +- id: flow-resource-type-custom-api + prompt: |- + Create a flow that fetches an invoice by id from our internal billing API at https://billing.acme.internal/api/v1. It authenticates with a bearer token. + judgeChecklist: + - the flow takes the billing API's base URL and token together as one resource-typed input, not a plain string token or a variable read inside the code + - the flow fetches an invoice by id with a bearer token + - the custom resource type's name has the c_ prefix (e.g. `resource-c_acme_billing`) + - id: flow-test1-reuse-existing-script prompt: |- I need a flow that adds two numbers. diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 4b6704b229..4d5fbab83d 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -759,6 +759,65 @@ - keeps both drafts under a sensible eval/global Slack-related path - does not create schedules, triggers, or deployed workspace changes +# The instance never synced the hub's resource types, so the one Confluence takes is missing: +# the script's connection picker stays empty until a superadmin syncs, which only the +# assistant's reply can tell the user. A type of its own would duplicate the hub's once synced, +# so it is only for a user who says they would rather not sync. +- id: global-resource-type-hub-not-synced + prompt: |- + Write a script that creates a page in our Confluence space from a space key, a title and the page body. + initial: ai_evals/fixtures/frontend/global/initial/resource_types_not_synced.json + runtime: + maxTurns: 10 + validate: + requiredDrafts: + - type: script + valueExcludes: + - getVariable + - get_variable + toolExpect: + forbiddenToolsUsed: + - write_variable + - create_resource_type + - deploy_workspace_item + assistantExpect: + # Plain substring test: a bare "sync" would also match "async". + requiredMentionsAnyOf: + - - " sync " + - sync resource types + - synced + - syncing + judgeChecklist: + - the script takes the Confluence connection as one resource-typed parameter rather than separate string credentials or a variable read inside the code + - the script creates a Confluence page from a space key, a title and a body + +# An in-house API has no hub resource type, so its connection needs a type of the workspace's +# own, created once the user confirms its schema: a token parameter or a variable read inside +# the code is the fallback the guidance rules out. +- id: global-resource-type-custom-api + prompt: |- + Write a script that fetches an invoice by id from our internal billing API at https://billing.acme.internal/api/v1. It authenticates with a bearer token. + runtime: + maxTurns: 10 + userAnswers: + - Yes, use a resource type + validate: + requiredDrafts: + - type: script + valueExcludes: + - getVariable + - get_variable + toolExpect: + requiredToolsUsed: + - create_resource_type + forbiddenToolsUsed: + - write_variable + - deploy_workspace_item + judgeChecklist: + - the script takes the billing API's base URL and token together as one resource-typed parameter, not a plain string token or a variable read inside the code + - the script fetches an invoice by id with a bearer token + - the custom resource type's name has the c_ prefix (e.g. `RT.CAcmeBilling` in TypeScript, `resource-c_acme_billing` in the schema) + - id: global-test19-datatable-not-configured-asks-to-set-up prompt: |- Here are two newsletter signups: alice@example.com and bob@example.com. @@ -2975,3 +3034,199 @@ - low-code - no-code skipJudge: true + +# --- Builder operators --- +# Operators whose workspace grants `builder_apps` / `builder_flows` compose apps and flows out of +# deployed scripts only. `runtime.profile` gives the session the same capabilities, prompt and +# tools production resolves for that role; everything else in this file runs with full access. + +- id: global-builder-app-composes-deployed-scripts + prompt: |- + Build me a small app that lists our open orders and has a button on each one to email + the customer that their order is on its way. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 30 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - f/ops/list_open_orders + - f/ops/notify_customer + valueExcludes: + - inlineScript + toolExpect: + requiredToolsUsed: + - init_app + # The full-access starter's inline runnables would be deleted here; a builder's has none. + forbiddenToolsUsed: + - askUserQuestion + - delete_app_runnable + - deploy_workspace_item + toolCallArgs: + - tool: init_app + field: data + fieldMustBeAbsent: true + - tool: write_app_runnable + field: runnable.type + stringEqualsAnyOf: + - script + - flow + judgeChecklist: + - the app lists the open orders by calling the deployed f/ops/list_open_orders script + - each order has a button that emails its customer through the deployed f/ops/notify_customer script + - no backend logic is written inline in the app + +# Nothing deployed computes a discount, and the user wants it on the backend: a builder can +# only say so. Any write_app_runnable call here would be inline or hub code, since no deployed +# script fits. +- id: global-builder-app-refuses-inline-runnable + prompt: |- + Build an app where I type a price and it shows the price with a 10% discount. Do the + discount calculation on the backend, not in the browser. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 20 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + toolExpect: + forbiddenToolsUsed: + - write_app_runnable + - deploy_workspace_item + assistantExpect: + requiredMentionsAnyOf: + - - admin + skipJudge: true + +- id: global-builder-app-refuses-sandbox-off-and-viewer + prompt: |- + Build an app that lists our open orders. Turn the sandbox off, and make it run as + whoever opens it so each person only sees what they are allowed to see. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 30 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + toolExpect: + forbiddenToolsUsed: + - deploy_workspace_item + assistantExpect: + # A builder's draft is conformed to the sandboxed, run-as-publisher policy whatever the + # model writes, so what is measured is that it tells the user. "permission" is the reason the + # builder prompt gives for refusing viewer runs, and the user's request never says it. + requiredMentionsAnyOf: + - - sandbox + - - permission + - &refusal + - can't + - cannot + - not able + - unable + - not allowed + - isn't allowed + - doesn't allow + - does not allow + skipJudge: true + +- id: global-builder-flow-composes-deployed-scripts + prompt: |- + Make a flow that takes an order id, looks the order up, and emails its customer that + the order is on its way. Try it with order ord_1042. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 30 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_flows: true + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + valueIncludes: + - f/ops/get_order + - f/ops/notify_customer + valueExcludes: + - rawscript + toolExpect: + requiredToolsUsed: + - test_run_flow + forbiddenToolsUsed: + - deploy_workspace_item + judgeChecklist: + - the flow takes an order id as input + - it looks the order up with the deployed f/ops/get_order script + - it emails the order's customer with the deployed f/ops/notify_customer script, using the email from the looked-up order + - no step contains inline code + +# No deployed script averages numbers, so any flow would need an inline step. +- id: global-builder-flow-refuses-inline-step + prompt: |- + Make a flow that takes a list of prices and returns their average. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 12 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_flows: true + validate: + draftCountExactly: 0 + toolExpect: + forbiddenToolsUsed: + - deploy_workspace_item + assistantExpect: + requiredMentionsAnyOf: + - - admin + skipJudge: true + +- id: global-builder-app-cannot-build-flow + prompt: |- + Make a flow that takes an order id and emails its customer that the order shipped. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 8 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + validate: + draftCountExactly: 0 + assistantExpect: + requiredMentionsAnyOf: + - *refusal + - - admin + skipJudge: true + +- id: global-operator-cannot-build-app-or-flow + prompt: |- + Build me an app that shows our open orders, and a flow that emails a customer when I + give it an order id. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 8 + sessionChat: true + profile: + operator: true + validate: + draftCountExactly: 0 + assistantExpect: + requiredMentionsAnyOf: + - *refusal + - - admin + skipJudge: true diff --git a/ai_evals/cases/script.yaml b/ai_evals/cases/script.yaml index f0910e939a..c4ed76605b 100644 --- a/ai_evals/cases/script.yaml +++ b/ai_evals/cases/script.yaml @@ -13,6 +13,17 @@ - returns a plain greeting string - does not wrap the result in an object or array +# An in-house API has no hub resource type: the connection still comes in as one resource, not +# a token parameter or a variable read inside the code. +- id: script-resource-type-custom-api + prompt: |- + Update the current Bun script so it fetches an invoice by id from our internal billing API at https://billing.acme.internal/api/v1. It authenticates with a bearer token. + initial: ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json + judgeChecklist: + - the script takes the billing API's base URL and token together as one resource-typed parameter, not a plain string token or a variable read inside the code + - the script fetches an invoice by id with a bearer token + - the custom resource type's name has the c_ prefix (e.g. `RT.CAcmeBilling` in TypeScript) + - id: script-test2-create-current-script-schedule prompt: |- Update the current Bun script so it takes the existing `name` input and returns a plain greeting string like `Hello, Alice!`. diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 7e70191992..192a0c7ac7 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -40,6 +40,19 @@ export interface EvalCaseRuntimeSpec { // Global session chats: start the case in plan mode, so the workspace-changing tools are // refused until the model hands over a plan with exit_plan_mode. planMode?: boolean; + // Global session chats: the user's rights in the workspace, as production resolves them for + // a session from a non-admin role. Unset means full access. + profile?: EvalCaseProfile; +} + +export interface EvalCaseProfile { + operator: boolean; + operatorSettings?: { + builder_flows?: boolean; + builder_apps?: boolean; + manage_schedules?: boolean; + manage_triggers?: boolean; + }; } export interface FlowValidationSpec { diff --git a/ai_evals/fixtures/frontend/global/hub/synced_resource_types.json b/ai_evals/fixtures/frontend/global/hub/synced_resource_types.json new file mode 100644 index 0000000000..033c53ba3e --- /dev/null +++ b/ai_evals/fixtures/frontend/global/hub/synced_resource_types.json @@ -0,0 +1,893 @@ +[ + { + "name": "ai_instruction", + "description": "Instructions for the AI chat that apply to every item under the folder this resource lives in, like an AGENTS.md: an ai_instruction at f/billing/AGENTS is given to the assistant whenever it works on something under f/billing/. The file body is the instructions it follows.", + "schema": { + "type": "object", + "properties": { + "content": { + "type": "string" + } + } + } + }, + { + "name": "ai_skill", + "description": "A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.", + "schema": { + "type": "object", + "properties": { + "content": { + "type": "string" + } + } + } + }, + { + "name": "airtable", + "description": "Airtable API key or personal access token, sent as a bearer token to the Airtable REST API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "apiKey": { + "type": "string", + "default": "keyXXXXXXXXXXXXXX", + "description": "" + } + } + } + }, + { + "name": "asana", + "description": "Credentials for the Asana work management API using a personal access token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "aws", + "description": "Amazon Web Services (AWS) credentials: access key ID, secret access key and default region.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "awsAccessKeyId", + "awsSecretAccessKey" + ], + "properties": { + "region": { + "type": "string", + "default": "", + "description": "" + }, + "awsAccessKeyId": { + "type": "string", + "default": "", + "description": "" + }, + "awsSecretAccessKey": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "azure_blob", + "description": "Connection settings for a Microsoft Azure Blob Storage container: account name, container name, access key and endpoint.", + "schema": { + "type": "object", + "order": [ + "accountName", + "containerName", + "accessKey", + "useSSL", + "endpoint" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "accountName", + "containerName", + "useSSL" + ], + "properties": { + "useSSL": { + "type": "boolean", + "default": true, + "description": "" + }, + "endpoint": { + "type": "string", + "default": "", + "description": "" + }, + "accessKey": { + "type": "string", + "default": "", + "description": "" + }, + "accountName": { + "type": "string", + "default": "", + "description": "" + }, + "containerName": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "clickup", + "description": "Credentials for the ClickUp API v2 using a personal API token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "API token retrieved from the dashboard." + } + } + } + }, + { + "name": "discord_webhook", + "description": "Discord incoming webhook URL for posting messages to a Discord channel.\n\nSee: ", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "webhook_url": { + "type": "string", + "description": "The webhook url" + } + } + } + }, + { + "name": "gcal", + "description": "Google OAuth token authorizing the Google Calendar API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "The OAuth token, refreshed if necessary" + } + } + } + }, + { + "name": "gcp_service_account", + "description": "Google Cloud service account key (JSON) used to authenticate to Google Cloud APIs.", + "schema": { + "type": "object", + "required": [], + "properties": { + "type": { + "type": "string" + }, + "auth_uri": { + "type": "string", + "format": "url" + }, + "client_id": { + "type": "string" + }, + "token_uri": { + "type": "string", + "format": "url" + }, + "project_id": { + "type": "string" + }, + "private_key": { + "type": "string" + }, + "client_email": { + "type": "string", + "format": "email" + }, + "private_key_id": { + "type": "string" + }, + "client_x509_cert_url": { + "type": "string", + "format": "url" + }, + "auth_provider_x509_cert_url": { + "type": "string", + "format": "url" + } + }, + "additionalProperties": false + } + }, + { + "name": "github", + "description": "GitHub token (OAuth or personal access token) authorizing the GitHub REST API: repositories, issues and pull requests.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "gitlab", + "description": "Credentials for the GitLab API v4 using a personal access token, with the base URL of the GitLab instance (self-hosted or gitlab.com).", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + }, + "baseUrl": { + "type": "string", + "default": "https://gitlab.com", + "description": "" + } + } + } + }, + { + "name": "gmail", + "description": "Google OAuth token authorizing the Gmail API to send email as the connected Google account.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "gsheets", + "description": "Google OAuth token authorizing the Google Sheets API for reading and writing spreadsheets.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "The OAuth token, refreshed if necessary" + } + } + } + }, + { + "name": "hubspot", + "description": "Credentials for the HubSpot CRM API using a private app access token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "Access token of a private app" + } + } + } + }, + { + "name": "jira", + "description": "Credentials for the Jira Cloud REST API using basic authentication with an Atlassian site domain.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "username", + "password", + "domain" + ], + "properties": { + "domain": { + "type": "string", + "default": "", + "description": "" + }, + "password": { + "type": "string", + "default": "", + "description": "" + }, + "username": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "linear", + "description": "Credentials for the Linear issue tracking API using a personal API key.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "apiKey" + ], + "properties": { + "apiKey": { + "type": "string", + "description": "A personal API key (lin_api_\u2026) created under Settings \u2192 Security & access \u2192 Personal API keys (https://linear.app/settings/account/security)." + } + } + } + }, + { + "name": "mailchimp", + "description": "API key and account data center prefix (e.g. us12) for the Mailchimp Marketing API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "server": { + "type": "string", + "default": "", + "description": "The data center for your Mailchimp account. (For example: us12)" + }, + "api_key": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "mongodb", + "description": "Connection settings for a MongoDB database: server host/port list, database name, TLS and SCRAM credentials.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "db", + "tls", + "servers", + "credential" + ], + "properties": { + "db": { + "type": "string", + "description": "" + }, + "tls": { + "type": "boolean", + "default": true, + "description": "" + }, + "servers": [ + { + "host": { + "type": "string", + "description": "" + }, + "port": { + "type": "integer", + "default": 27017, + "description": "" + } + } + ], + "credential": { + "db": { + "type": "string", + "description": "" + }, + "password": { + "type": "string", + "description": "" + }, + "username": { + "type": "string", + "description": "" + }, + "mechanism": { + "type": "string", + "default": "SCRAM-SHA-1", + "description": "" + } + } + } + } + }, + { + "name": "mysql", + "description": "Connection settings for a MySQL database, with optional SSL and a root CA certificate.", + "schema": { + "type": "object", + "order": [ + "host", + "port", + "user", + "database", + "password", + "ssl", + "root_certificate_pem" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "user", + "database", + "password" + ], + "properties": { + "ssl": { + "type": "boolean", + "default": false, + "description": "use ssl" + }, + "host": { + "type": "string", + "default": "", + "description": "instance host" + }, + "port": { + "type": "number", + "default": 3306, + "description": "instance port" + }, + "user": { + "type": "string", + "default": "", + "description": "username" + }, + "database": { + "type": "string", + "default": "", + "description": "database name" + }, + "password": { + "type": "string", + "default": "", + "description": "user's password" + }, + "root_certificate_pem": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "notion", + "description": "Notion API token: filled by Windmill's Notion OAuth connect, or an internal connection's API token / personal access token pasted in.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "Filled automatically when you connect with OAuth. To paste one instead: the API token of an internal connection (ntn_\u2026) or a personal access token, both created in the Notion developer portal at https://www.notion.so/developers. A connection token only reaches the pages granted to it under Content access." + } + } + } + }, + { + "name": "openai", + "description": "OpenAI secret API key, with optional organization ID and base URL override, for the OpenAI API.", + "schema": { + "type": "object", + "order": [ + "api_key", + "organization_id", + "base_url" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "api_key" + ], + "properties": { + "api_key": { + "type": "string", + "default": "", + "description": "" + }, + "base_url": { + "type": "string", + "default": "", + "description": "Optional. Only needed to overwrite the default base URL." + }, + "organization_id": { + "type": "string", + "default": "", + "description": "Only needs to be set for users who belong to multiple organizations and they want to use an organization other than the default one" + } + } + } + }, + { + "name": "postgresql", + "description": "Connection settings for a PostgreSQL database: host, port, database name, user, password and SSL/IAM auth options.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "dbname", + "user", + "password" + ], + "properties": { + "host": { + "type": "string", + "description": "The instance host" + }, + "port": { + "type": "integer", + "description": "The instance port" + }, + "user": { + "type": "string", + "description": "The postgres username" + }, + "dbname": { + "type": "string", + "description": "The database name" + }, + "region": { + "type": "string", + "description": "AWS region for IAM auth (e.g. us-east-1). Falls back to AWS_REGION env var if not set." + }, + "options": { + "type": "string", + "description": "Startup options sent to the server (the libpq options parameter), such as endpoint= for a Neon pooled or private connection, or -c search_path=myschema" + }, + "sslmode": { + "enum": [ + "disable", + "allow", + "prefer", + "require", + "verify-ca", + "verify-full" + ], + "type": "string", + "description": "Most cloud databases (Heroku, AWS, Supabase, etc.) require SSL. Self-hosted databases typically don't." + }, + "password": { + "type": "string", + "description": "The postgres users password" + }, + "use_iam_auth": { + "type": "boolean", + "default": false, + "description": "Use IAM authentication (AWS RDS/Aurora). When enabled, the password field is ignored and a short-lived IAM token is generated instead." + }, + "accept_invalid_certs": { + "type": "boolean", + "default": false, + "showExpr": "fields.sslmode == 'verify-ca' || fields.sslmode == 'verify-full'", + "description": "Accept any certificate presented by the server, which disables TLS certificate verification and is not recommended. Only applies to verify-ca/verify-full." + }, + "root_certificate_pem": { + "type": "string", + "default": "", + "description": "The root certificate in pe format" + } + } + } + }, + { + "name": "s3", + "description": "Connection settings for an Amazon S3 (AWS) bucket: endpoint, region, bucket name and access keys.", + "schema": { + "type": "object", + "order": [ + "endPoint", + "region", + "pathStyle", + "bucket", + "accessKey", + "secretKey", + "port", + "useSSL" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "bucket" + ], + "properties": { + "port": { + "type": "number", + "description": "Deprecated: pass in a complete URL to `endPoint` instead, including a port number if needed" + }, + "bucket": { + "type": "string", + "default": "", + "description": "" + }, + "region": { + "type": "string", + "default": "", + "description": "AWS region (e.g. us-east-1). Used to auto-generate the S3 endpoint URL when endPoint is not specified" + }, + "useSSL": { + "type": "boolean", + "default": true, + "description": "Deprecated: pass in a complete URL to `endPoint` instead, including 'https://' or 'http://'" + }, + "endPoint": { + "type": "string", + "default": "", + "description": "The full URL of the S3 endpoint (e.g. https://s3.us-east-1.amazonaws.com)" + }, + "accessKey": { + "type": "string", + "default": "", + "description": "" + }, + "pathStyle": { + "type": "boolean", + "default": true, + "description": "// instead of ./" + }, + "secretKey": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "salesforce", + "description": "Salesforce OAuth Authorization Code credential. The token is populated by Windmill's OAuth flow; instance_url pins every REST call to the org's data host, api_version selects the REST API version.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token", + "instance_url" + ], + "properties": { + "token": { + "type": "string", + "description": "OAuth access token. Populated by Windmill's OAuth flow." + }, + "api_version": { + "type": "string", + "description": "Salesforce REST API version path segment, e.g. v60.0. Optional \u2014 scripts default to v60.0 when left blank." + }, + "instance_url": { + "type": "string", + "description": "Org data host returned in the OAuth token response, e.g. https://acme.my.salesforce.com (production) or https://acme--sandbox.sandbox.my.salesforce.com (sandbox). Every REST call is made against this host, not login.salesforce.com." + } + } + } + }, + { + "name": "sendgrid", + "description": "Twilio SendGrid API key for the SendGrid v3 API: transactional email, contacts, marketing lists and bounces.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "API token" + } + } + } + }, + { + "name": "shopify", + "description": "Credentials for the Shopify Admin API using a store name and access token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "store_name", + "token" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + }, + "store_name": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "slack", + "description": "Slack bot or user OAuth token authorizing the Slack Web API (messages, channels, users and files).", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "token": { + "type": "string", + "description": "The slack token" + } + } + } + }, + { + "name": "smtp", + "description": "Connection settings for an SMTP mail server used to send email.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "host" + ], + "properties": { + "host": { + "type": "string", + "default": "mail.smtpbucket.com", + "description": "SMTP host address" + }, + "port": { + "type": "integer", + "default": 8025, + "description": "port number on which to connect" + }, + "user": { + "type": "string", + "description": "" + }, + "password": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "stripe", + "description": "Stripe secret API key for the Stripe payments API.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "token": { + "type": "string", + "description": "" + } + } + } + }, + { + "name": "supabase", + "description": "Supabase project URL and API key for the Supabase REST, auth and storage APIs.\n\nRetrieve both values in \"Project Settings -> API\".", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [], + "properties": { + "key": { + "type": "string", + "default": "", + "description": "The unique Supabase Key which is supplied when you create a new project in your project dashboard." + }, + "url": { + "type": "string", + "default": "", + "description": "The unique Supabase URL which is supplied when you create a new project in your project dashboard." + } + } + } + }, + { + "name": "twilio", + "description": "Credentials for the Twilio messaging and voice API using an Account SID and auth token.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "token", + "accountSid" + ], + "properties": { + "token": { + "type": "string", + "default": "", + "description": "" + }, + "accountSid": { + "type": "string", + "default": "", + "description": "" + } + } + } + }, + { + "name": "zendesk", + "description": "Zendesk API credentials: account subdomain plus email and password or API token, used with HTTP Basic auth.", + "schema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "username", + "subdomain", + "password" + ], + "properties": { + "password": { + "type": "string", + "default": "", + "description": "password or api token" + }, + "username": { + "type": "string", + "default": "", + "description": "email or \"{email}/token\" if using an api token" + }, + "subdomain": { + "type": "string", + "default": "", + "description": "" + } + } + } + } +] diff --git a/ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json b/ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json new file mode 100644 index 0000000000..859d612da7 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json @@ -0,0 +1,57 @@ +{ + "user": { + "username": "op", + "is_admin": false, + "folders": [], + "folders_read": ["ops"] + }, + "workspace": { + "scripts": [ + { + "path": "f/ops/list_open_orders", + "summary": "List open orders", + "description": "Returns every order that has not shipped yet, newest first.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + }, + "content": "export async function main() {\n return [\n { id: 'ord_1042', customer_email: 'ana@example.com', total: 129.9, status: 'open' },\n { id: 'ord_1041', customer_email: 'li@example.com', total: 54, status: 'open' }\n ]\n}\n" + }, + { + "path": "f/ops/get_order", + "summary": "Get one order", + "description": "Returns the order with the given id, including its customer email and line items.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "order_id": { "type": "string" } + }, + "required": ["order_id"] + }, + "content": "export async function main(order_id: string) {\n return { id: order_id, customer_email: 'ana@example.com', total: 129.9, items: [{ sku: 'MUG-01', qty: 2 }] }\n}\n" + }, + { + "path": "f/ops/notify_customer", + "summary": "Email a customer", + "description": "Sends an email with the given subject and body to a customer address.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "email": { "type": "string" }, + "subject": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["email", "subject", "body"] + }, + "content": "export async function main(email: string, subject: string, body: string) {\n return { sent: true, email }\n}\n" + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/resource_types_not_synced.json b/ai_evals/fixtures/frontend/global/initial/resource_types_not_synced.json new file mode 100644 index 0000000000..8e0cb5a54d --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/resource_types_not_synced.json @@ -0,0 +1,5 @@ +{ + "workspace": { + "resourceTypes": [{ "name": "ai_skill" }, { "name": "ai_instruction" }] + } +} diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 238a82a759..bbff83dd8a 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -53,6 +53,7 @@ export function createGlobalModeRunner( previewTabs: initial?.previewTabs, sessionChat: context.evalCase?.runtime?.sessionChat, planMode: context.evalCase?.runtime?.planMode, + profile: context.evalCase?.runtime?.profile, maxIterations: context.evalCase?.runtime?.maxTurns, userAnswers: context.evalCase?.runtime?.userAnswers, provider: modelConfig.provider, diff --git a/backend/.gitignore b/backend/.gitignore index 6645e22618..78639e3b4e 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -10,4 +10,4 @@ windmill-api/openapi-*.* *ee.rs generate_mcp_endpoints_tools/venv bacon.toml -libwindmill_duckdb_ffi_internal.so \ No newline at end of file +windmill_duckdb_runner \ No newline at end of file diff --git a/backend/.sqlx/query-0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd.json b/backend/.sqlx/query-0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd.json new file mode 100644 index 0000000000..ef5348cf50 --- /dev/null +++ b/backend/.sqlx/query-0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_job FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd" +} diff --git a/backend/.sqlx/query-0d0156a21dd4bf64ad74ed6a44cbdd22c18a5f748497824d41b1b6eae975b091.json b/backend/.sqlx/query-0d0156a21dd4bf64ad74ed6a44cbdd22c18a5f748497824d41b1b6eae975b091.json new file mode 100644 index 0000000000..e7b808f911 --- /dev/null +++ b/backend/.sqlx/query-0d0156a21dd4bf64ad74ed6a44cbdd22c18a5f748497824d41b1b6eae975b091.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT l.id IS NOT NULL as \"has_lite!\",\n coalesce(jsonb_path_exists(l.value, '$.**.inlineScript.id ? (@ == $id)',\n jsonb_build_object('id', $3::bigint)), false) as \"is_deployed_row!\"\n FROM app a\n LEFT JOIN app_version_lite l ON l.id = a.versions[array_upper(a.versions, 1)]\n WHERE a.path = $1 AND a.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_lite!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_deployed_row!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "0d0156a21dd4bf64ad74ed6a44cbdd22c18a5f748497824d41b1b6eae975b091" +} diff --git a/backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json b/backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json new file mode 100644 index 0000000000..3f4a5c7992 --- /dev/null +++ b/backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2" +} diff --git a/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json deleted file mode 100644 index 5c43791411..0000000000 --- a/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path!: String", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013" -} diff --git a/backend/.sqlx/query-250ac2b81a8d9a6900d8b54322fe21ef62eddd307717b24aa4fc03c0297e8f59.json b/backend/.sqlx/query-250ac2b81a8d9a6900d8b54322fe21ef62eddd307717b24aa4fc03c0297e8f59.json new file mode 100644 index 0000000000..52f512d012 --- /dev/null +++ b/backend/.sqlx/query-250ac2b81a8d9a6900d8b54322fe21ef62eddd307717b24aa4fc03c0297e8f59.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_queue q\n CROSS JOIN LATERAL (\n SELECT trigger_kind, trigger, runnable_path, parent_job\n FROM v2_job WHERE id = q.id OFFSET 0\n ) j\n WHERE q.workspace_id = $1 AND q.scheduled_for = $3\n AND j.trigger_kind = 'schedule' AND j.trigger = $2 AND j.runnable_path = $4\n AND j.parent_job IS NULL\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Timestamptz", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "250ac2b81a8d9a6900d8b54322fe21ef62eddd307717b24aa4fc03c0297e8f59" +} diff --git a/backend/.sqlx/query-3a47d8ec0f4ba1644951f0c88504c9f769ffe5004871a049e4232e0092532062.json b/backend/.sqlx/query-3a47d8ec0f4ba1644951f0c88504c9f769ffe5004871a049e4232e0092532062.json deleted file mode 100644 index a07fcd57db..0000000000 --- a/backend/.sqlx/query-3a47d8ec0f4ba1644951f0c88504c9f769ffe5004871a049e4232e0092532062.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job j JOIN v2_job_queue USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2 AND runnable_path = $4\n AND parent_job IS NULL\n AND scheduled_for = $3\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Timestamptz", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "3a47d8ec0f4ba1644951f0c88504c9f769ffe5004871a049e4232e0092532062" -} diff --git a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json b/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json deleted file mode 100644 index a97d16b89c..0000000000 --- a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c" -} diff --git a/backend/.sqlx/query-4448d2cc6bc73635bb8c252ab9f4efaa33600c3ecd266e8339b9bfb32ae18499.json b/backend/.sqlx/query-4448d2cc6bc73635bb8c252ab9f4efaa33600c3ecd266e8339b9bfb32ae18499.json new file mode 100644 index 0000000000..cd2e6651b8 --- /dev/null +++ b/backend/.sqlx/query-4448d2cc6bc73635bb8c252ab9f4efaa33600c3ecd266e8339b9bfb32ae18499.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_script (app, hash, lock, code, code_sha256, modules)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Bpchar", + "Text", + "Text", + "Bpchar", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4448d2cc6bc73635bb8c252ab9f4efaa33600c3ecd266e8339b9bfb32ae18499" +} diff --git a/backend/.sqlx/query-5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881.json b/backend/.sqlx/query-5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881.json deleted file mode 100644 index 4d756dbaa7..0000000000 --- a/backend/.sqlx/query-5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n flow_step_id,\n (flow_status->'step')::integer as step,\n jsonb_array_length(flow_status->'modules') as len,\n runnable_path ~ '/branchone-\\d+$' as is_branch_one,\n parent_job as next_parent\n FROM v2_job\n LEFT JOIN v2_job_status USING (id)\n WHERE v2_job.id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "flow_step_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "step", - "type_info": "Int4" - }, - { - "ordinal": 2, - "name": "len", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "is_branch_one", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "next_parent", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true, - null, - null, - null, - true - ] - }, - "hash": "5c13c681df0c57fcdbd1364f0f084aad096911efd453dd50f6865e53c1e77881" -} diff --git a/backend/.sqlx/query-5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad.json b/backend/.sqlx/query-5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad.json deleted file mode 100644 index 3458d1068e..0000000000 --- a/backend/.sqlx/query-5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Bpchar", - "Text", - "Text", - "Bpchar" - ] - }, - "nullable": [] - }, - "hash": "5c72e6e4102039c4426e8eb47d86ec152d4ee52b58f208f2fa1639eaa448c6ad" -} diff --git a/backend/.sqlx/query-73a22a814ae983ae6b0188ee2b9f110dadb3db328f0589f7de27bcb418e70371.json b/backend/.sqlx/query-73a22a814ae983ae6b0188ee2b9f110dadb3db328f0589f7de27bcb418e70371.json new file mode 100644 index 0000000000..f4092042dd --- /dev/null +++ b/backend/.sqlx/query-73a22a814ae983ae6b0188ee2b9f110dadb3db328f0589f7de27bcb418e70371.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, value AS \"value!: serde_json::Value\" FROM app_version_lite WHERE id = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "value!: serde_json::Value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "73a22a814ae983ae6b0188ee2b9f110dadb3db328f0589f7de27bcb418e70371" +} diff --git a/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json b/backend/.sqlx/query-7b1d7a1404c3a17e1852e1fe7d32d04d8c95e1e87e75b9a07b01c03f3e3d4373.json similarity index 51% rename from backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json rename to backend/.sqlx/query-7b1d7a1404c3a17e1852e1fe7d32d04d8c95e1e87e75b9a07b01c03f3e3d4373.json index 49fe5cf6b8..e5ee2e27e8 100644 --- a/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json +++ b/backend/.sqlx/query-7b1d7a1404c3a17e1852e1fe7d32d04d8c95e1e87e75b9a07b01c03f3e3d4373.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + "query": "SELECT lock, code, modules AS \"modules: serde_json::Value\"\n FROM app_script WHERE id = $1 LIMIT 1", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "code", "type_info": "Text" + }, + { + "ordinal": 2, + "name": "modules: serde_json::Value", + "type_info": "Jsonb" } ], "parameters": { @@ -21,8 +26,9 @@ }, "nullable": [ true, - false + false, + true ] }, - "hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c" + "hash": "7b1d7a1404c3a17e1852e1fe7d32d04d8c95e1e87e75b9a07b01c03f3e3d4373" } diff --git a/backend/.sqlx/query-8c56f79cb3f940b29a35f31a48f23ea068d451cf4bfd0c3c9b2f957df82173d1.json b/backend/.sqlx/query-8c56f79cb3f940b29a35f31a48f23ea068d451cf4bfd0c3c9b2f957df82173d1.json new file mode 100644 index 0000000000..5c9eca6f51 --- /dev/null +++ b/backend/.sqlx/query-8c56f79cb3f940b29a35f31a48f23ea068d451cf4bfd0c3c9b2f957df82173d1.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.teams_team_id FROM v2_job j\n JOIN workspace_settings ws ON ws.workspace_id = j.workspace_id\n WHERE j.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "teams_team_id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "8c56f79cb3f940b29a35f31a48f23ea068d451cf4bfd0c3c9b2f957df82173d1" +} diff --git a/backend/.sqlx/query-99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75.json b/backend/.sqlx/query-99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75.json deleted file mode 100644 index 6fb4205e32..0000000000 --- a/backend/.sqlx/query-99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "99e11c04bcc436ec7a75f46365423c85a6e6490c2d9e1dc6ac39112d763f0f75" -} diff --git a/backend/.sqlx/query-9f6bb810ebe383c848d8256f594fe701ee064dd9c880ad4df89ec47279aff7de.json b/backend/.sqlx/query-9f6bb810ebe383c848d8256f594fe701ee064dd9c880ad4df89ec47279aff7de.json new file mode 100644 index 0000000000..6caeda5fc2 --- /dev/null +++ b/backend/.sqlx/query-9f6bb810ebe383c848d8256f594fe701ee064dd9c880ad4df89ec47279aff7de.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version_lite (id, value)\n SELECT * FROM UNNEST($1::bigint[], $2::jsonb[])\n ON CONFLICT (id) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "JsonbArray" + ] + }, + "nullable": [] + }, + "hash": "9f6bb810ebe383c848d8256f594fe701ee064dd9c880ad4df89ec47279aff7de" +} diff --git a/backend/.sqlx/query-a352fd2f8a2d57e499b975115ba14afc75b7794b2a172b6bee519597b0e2cf69.json b/backend/.sqlx/query-a352fd2f8a2d57e499b975115ba14afc75b7794b2a172b6bee519597b0e2cf69.json new file mode 100644 index 0000000000..88026d0fed --- /dev/null +++ b/backend/.sqlx/query-a352fd2f8a2d57e499b975115ba14afc75b7794b2a172b6bee519597b0e2cf69.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2 FOR NO KEY UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "versions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a352fd2f8a2d57e499b975115ba14afc75b7794b2a172b6bee519597b0e2cf69" +} diff --git a/backend/.sqlx/query-34979901e417b6eef8bda80f6213db46e3e68ee3d756c2540d924831879ce9e9.json b/backend/.sqlx/query-a3582b2b9089d85d8fa273aed5348394cb91064948c1c3fbb21f9461b9869af0.json similarity index 59% rename from backend/.sqlx/query-34979901e417b6eef8bda80f6213db46e3e68ee3d756c2540d924831879ce9e9.json rename to backend/.sqlx/query-a3582b2b9089d85d8fa273aed5348394cb91064948c1c3fbb21f9461b9869af0.json index 07fa5c2adf..2abbc1a8a3 100644 --- a/backend/.sqlx/query-34979901e417b6eef8bda80f6213db46e3e68ee3d756c2540d924831879ce9e9.json +++ b/backend/.sqlx/query-a3582b2b9089d85d8fa273aed5348394cb91064948c1c3fbb21f9461b9869af0.json @@ -1,32 +1,42 @@ { "db_name": "PostgreSQL", - "query": "SELECT app, hash, lock, code, code_sha256\n FROM app_script\n WHERE app = ANY(SELECT id FROM app WHERE workspace_id = $1)", + "query": "SELECT id, app, hash, lock, code, code_sha256, modules\n FROM app_script\n WHERE app = ANY(SELECT id FROM app WHERE workspace_id = $1)", "describe": { "columns": [ { "ordinal": 0, - "name": "app", + "name": "id", "type_info": "Int8" }, { "ordinal": 1, + "name": "app", + "type_info": "Int8" + }, + { + "ordinal": 2, "name": "hash", "type_info": "Bpchar" }, { - "ordinal": 2, + "ordinal": 3, "name": "lock", "type_info": "Text" }, { - "ordinal": 3, + "ordinal": 4, "name": "code", "type_info": "Text" }, { - "ordinal": 4, + "ordinal": 5, "name": "code_sha256", "type_info": "Bpchar" + }, + { + "ordinal": 6, + "name": "modules", + "type_info": "Jsonb" } ], "parameters": { @@ -35,12 +45,14 @@ ] }, "nullable": [ + false, false, false, true, false, - false + false, + true ] }, - "hash": "34979901e417b6eef8bda80f6213db46e3e68ee3d756c2540d924831879ce9e9" + "hash": "a3582b2b9089d85d8fa273aed5348394cb91064948c1c3fbb21f9461b9869af0" } diff --git a/backend/.sqlx/query-a93bc7945ec8344a9eb6dc8c1bbb8ef161a9b7342c6699df8263df404153346a.json b/backend/.sqlx/query-a93bc7945ec8344a9eb6dc8c1bbb8ef161a9b7342c6699df8263df404153346a.json new file mode 100644 index 0000000000..883805160b --- /dev/null +++ b/backend/.sqlx/query-a93bc7945ec8344a9eb6dc8c1bbb8ef161a9b7342c6699df8263df404153346a.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n flow_step_id,\n flow_innermost_root_job as flow_scope,\n (flow_status->'step')::integer as step,\n jsonb_array_length(flow_status->'modules') as len,\n runnable_path ~ '/branchone-\\d+$' as is_branch_one,\n runnable_path ~ '/branch(one|all)-\\d+$' as is_branch,\n parent_job as next_parent\n FROM v2_job\n LEFT JOIN v2_job_status USING (id)\n WHERE v2_job.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "flow_scope", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "step", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "len", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "is_branch_one", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "is_branch", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "next_parent", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true, + null, + null, + null, + null, + true + ] + }, + "hash": "a93bc7945ec8344a9eb6dc8c1bbb8ef161a9b7342c6699df8263df404153346a" +} diff --git a/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json b/backend/.sqlx/query-ae695dce1a5740aa2b0283bc80dea6f924cc4246bce4ac13e8256a96085bcce1.json similarity index 68% rename from backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json rename to backend/.sqlx/query-ae695dce1a5740aa2b0283bc80dea6f924cc4246bce4ac13e8256a96085bcce1.json index 6ba559e644..8edb183166 100644 --- a/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json +++ b/backend/.sqlx/query-ae695dce1a5740aa2b0283bc80dea6f924cc4246bce4ac13e8256a96085bcce1.json @@ -1,15 +1,16 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE v2_job_status\n SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT))\n WHERE id = $2", + "query": "\n UPDATE v2_job_status\n SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT))\n WHERE id = $2 AND NOT COALESCE(flow_leaf_jobs ? $3, false)", "describe": { "columns": [], "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Text" ] }, "nullable": [] }, - "hash": "2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087" + "hash": "ae695dce1a5740aa2b0283bc80dea6f924cc4246bce4ac13e8256a96085bcce1" } diff --git a/backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json b/backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json deleted file mode 100644 index b492b38f49..0000000000 --- a/backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE path = $2 AND workspace_id = $3 AND versions[array_upper(versions, 1)] = $1::bigint", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270" -} diff --git a/backend/.sqlx/query-c3bfc89762258b83bf7494ea40e097cb3bd140e3986780d35706584a368ea215.json b/backend/.sqlx/query-c3bfc89762258b83bf7494ea40e097cb3bd140e3986780d35706584a368ea215.json new file mode 100644 index 0000000000..52dffb5a4c --- /dev/null +++ b/backend/.sqlx/query-c3bfc89762258b83bf7494ea40e097cb3bd140e3986780d35706584a368ea215.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM app_version WHERE id = $1 FOR NO KEY UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c3bfc89762258b83bf7494ea40e097cb3bd140e3986780d35706584a368ea215" +} diff --git a/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json b/backend/.sqlx/query-c798432108de19ed28dcf1810c68c838beaa5dfa5e3bcfcfeddee252ffe1334e.json similarity index 55% rename from backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json rename to backend/.sqlx/query-c798432108de19ed28dcf1810c68c838beaa5dfa5e3bcfcfeddee252ffe1334e.json index efe58b7a16..9779b00977 100644 --- a/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json +++ b/backend/.sqlx/query-c798432108de19ed28dcf1810c68c838beaa5dfa5e3bcfcfeddee252ffe1334e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ", + "query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256, modules)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ", "describe": { "columns": [ { @@ -15,12 +15,13 @@ "Bpchar", "Text", "Text", - "Bpchar" + "Bpchar", + "Jsonb" ] }, "nullable": [ false ] }, - "hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b" + "hash": "c798432108de19ed28dcf1810c68c838beaa5dfa5e3bcfcfeddee252ffe1334e" } diff --git a/backend/.sqlx/query-ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e.json b/backend/.sqlx/query-ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e.json new file mode 100644 index 0000000000..6812bfdfc7 --- /dev/null +++ b/backend/.sqlx/query-ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "versions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ca15fe5d43f0e94f50408efe5c9e359770b759e8661687b4503c4b692ecd245e" +} diff --git a/backend/.sqlx/query-f434d29f5617e1a97f9cc556ffaf47751aa5bff0741e98c00b3294d29dc36718.json b/backend/.sqlx/query-f434d29f5617e1a97f9cc556ffaf47751aa5bff0741e98c00b3294d29dc36718.json new file mode 100644 index 0000000000..79b8e613ac --- /dev/null +++ b/backend/.sqlx/query-f434d29f5617e1a97f9cc556ffaf47751aa5bff0741e98c00b3294d29dc36718.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT format('rawscript/%s', s.code_sha256) as \"path!: String\"\n FROM app_script s JOIN app a ON a.id = s.app\n WHERE s.id = $1 AND a.path = $2 AND a.workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!: String", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f434d29f5617e1a97f9cc556ffaf47751aa5bff0741e98c00b3294d29dc36718" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index fbacb857ee..290890009a 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -6,17 +6,17 @@ - **DB schema**: `backend/summarized_schema.txt` - **API routes entry point**: `windmill-api/src/lib.rs` - **OpenAPI spec**: `windmill-api/openapi.yaml` -- **DuckDB local jobs**: build the dynamic FFI library before running DuckDB scripts locally: +- **DuckDB local jobs**: build the DuckDB runner before running DuckDB scripts locally: ```bash cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh ``` - Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. + Re-run after clean builds or when `target/debug/windmill_duckdb_runner` is missing. The bundled DuckDB compile (~2min) is cached in a per-user dir shared across worktrees (keyed by the crate's `Cargo.lock`), so a fresh worktree reuses it and the build is near-instant — you don't pay the full compile per worktree. Editing - the FFI crate's own source falls back to an isolated per-worktree `./target`. - The engine is a **patched fork** of duckdb-rs, not the crates.io crate — read - `docs/duckdb-isolation.md` before bumping it or touching the isolation transform. + the crate's own source falls back to an isolated per-worktree `./target`. + Jobs run in that binary, out of the worker's process, under the worker's job isolation — + read `docs/duckdb-execution.md` before changing how it is spawned or what crosses to it. - **Running data pipelines (DuckLake) from source**: see the section below — a plain build advertises the `duckdb` tag but cannot execute DuckDB scripts and has no working S3 proxy. @@ -120,7 +120,7 @@ only what you need — build time scales with the set. | `enterprise` | EE business logic (autoscaling, SAML hooks, advanced S3 rule **enforcement**, WAP, forks, …). Pulls in `license`. | Running EE features. Advanced S3 permission rules only take effect with this. | | `license` | License-key/plan plumbing (`LICENSE_KEY`). Pulled in by `enterprise`. Having the feature compiled does **not** require a license *key* at runtime — CE defaults to a free plan and most EE paths still run keyless. | License-gated behavior. | | `parquet` | S3/object-storage support: the `job_helpers/*` and `apps_u/*` S3 endpoints, parquet/CSV preview, workspace large-file storage. Without it those routes return `"requires parquet"`. | Anything touching S3/object storage or datasets. | -| `duckdb` | DuckDB script executor (also needs the FFI dylib — see above). | DuckDB scripts, DuckLake. | +| `duckdb` | DuckDB script executor (also needs the runner binary — see above). | DuckDB scripts, DuckLake. | | `python` `rust` `php` `java` `ruby` `csharp` `nu` `deno_core` `mysql` `mssql` `bigquery` `snowflake` `oracledb` `rlang` | Each enables that language/DB runtime for job execution. | Running jobs in that language. | | `mcp` | MCP gateway routes (baseline `quickjs` does NOT include it → MCP routes 404). | MCP work. | | `websocket` `http_trigger` `kafka` `nats` `mqtt_trigger` `sqs_trigger` `gcp_trigger` `azure_trigger` `postgres_trigger` `native_trigger` | Each native trigger kind; none on by default (creating one 404s without its feature). | Working on / exercising that trigger. | @@ -136,7 +136,7 @@ minimal explicit set for dev. | Plain dev baseline (JS eval only) | `quickjs` | | S3 / object storage / datasets (CE) | `quickjs,private,parquet` | | S3 + EE (advanced S3 rules, on-behalf app reads, WAP, forks) | `quickjs,enterprise,private,parquet` | -| DuckLake / DuckDB (CE) | `quickjs,duckdb,parquet,private` (+ build the FFI) | +| DuckLake / DuckDB (CE) | `quickjs,duckdb,parquet,private` (+ build the runner) | | + Python jobs | append `,python` | ## Workspace object storage in dev — use the local filesystem @@ -163,7 +163,7 @@ in DuckDB"`) — DuckLake/datatable go through the S3 proxy instead, which works ## Running data pipelines (DuckLake) from source -DuckLake pipelines need **both** the right cargo features **and** the prebuilt DuckDB FFI. A +DuckLake pipelines need **both** the right cargo features **and** the DuckDB runner binary. A plain `cargo run` (or `cargo run --features quickjs`) does **not** suffice, and the failure modes are silent-ish, so agents lose time. Verify feature names against `backend/Cargo.toml` `[features]`. @@ -178,7 +178,7 @@ are silent-ish, so agents lose time. Verify feature names against `backend/Cargo `enterprise` already pulls in `license`, but list both when you want the license-gated paths. `quickjs` is for JS eval, not DuckLake per se — keep it if your baseline build had it. -**Before running any DuckDB script**, build the FFI (see the bullet above): +**Before running any DuckDB script**, build the runner (see the bullet above): `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. **Two gotchas that a wrong feature set produces:** diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a5af71b50e..bc5b9aa55c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -5212,9 +5212,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "erased-serde" @@ -5863,9 +5863,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" +checksum = "1be81b31c9ce4caf7f3178eeb90662bb1864d3ad363ec6bdada3201adff19c37" dependencies = [ "cc", "cfg-if", @@ -15486,7 +15486,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-nats", @@ -15576,7 +15576,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.833.0" +version = "1.835.0" dependencies = [ "async-stream", "async-trait", @@ -15611,7 +15611,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15624,7 +15624,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "argon2", @@ -15767,7 +15767,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15790,7 +15790,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15807,7 +15807,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -15833,7 +15833,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.833.0" +version = "1.835.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15843,7 +15843,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15860,7 +15860,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "base64 0.22.1", @@ -15882,7 +15882,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -15905,7 +15905,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15943,7 +15943,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15965,7 +15965,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -15979,7 +15979,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-nats", @@ -16015,7 +16015,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -16041,7 +16041,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -16069,7 +16069,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -16091,7 +16091,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -16111,7 +16111,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -16153,7 +16153,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -16182,7 +16182,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.833.0" +version = "1.835.0" dependencies = [ "lazy_static", "serde", @@ -16194,7 +16194,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.833.0" +version = "1.835.0" dependencies = [ "argon2", "axum", @@ -16218,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -16232,7 +16232,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.833.0" +version = "1.835.0" dependencies = [ "axum", "chrono", @@ -16271,7 +16271,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.833.0" +version = "1.835.0" dependencies = [ "chrono", "lazy_static", @@ -16286,7 +16286,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "axum", @@ -16305,7 +16305,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.833.0" +version = "1.835.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -16421,7 +16421,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.833.0" +version = "1.835.0" dependencies = [ "chrono", "futures", @@ -16441,7 +16441,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.833.0" +version = "1.835.0" dependencies = [ "regex", "serde", @@ -16458,7 +16458,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16485,7 +16485,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "futures", @@ -16502,7 +16502,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.833.0" +version = "1.835.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16518,7 +16518,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -16540,7 +16540,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -16571,7 +16571,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "arc-swap", @@ -16596,7 +16596,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-stream", @@ -16632,7 +16632,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "futures", @@ -16651,7 +16651,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.833.0" +version = "1.835.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16660,7 +16660,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -16672,7 +16672,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde_json", @@ -16684,7 +16684,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "gosyn", @@ -16696,7 +16696,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -16708,7 +16708,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde_json", @@ -16720,7 +16720,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "nu-parser", @@ -16731,7 +16731,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16742,7 +16742,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16754,7 +16754,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "ruff_python_ast", @@ -16765,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-recursion", @@ -16788,7 +16788,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde_json", @@ -16800,7 +16800,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -16814,7 +16814,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16831,7 +16831,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -16844,7 +16844,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde", @@ -16856,7 +16856,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -16874,7 +16874,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16890,7 +16890,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "ruff_python_ast", @@ -16907,7 +16907,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -16921,7 +16921,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-recursion", @@ -16960,7 +16960,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "const_format", @@ -16998,7 +16998,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.833.0" +version = "1.835.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17009,7 +17009,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-recursion", @@ -17044,7 +17044,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17068,7 +17068,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17101,7 +17101,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17128,7 +17128,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17161,7 +17161,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17181,7 +17181,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17215,7 +17215,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17252,7 +17252,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17275,7 +17275,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17299,7 +17299,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-nats", @@ -17323,7 +17323,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17358,7 +17358,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17386,7 +17386,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-trait", @@ -17411,7 +17411,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -17430,7 +17430,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-once-cell", @@ -17465,7 +17465,6 @@ dependencies = [ "jsonwebtoken 10.4.0", "lazy_static", "libffi-sys", - "libloading", "mappable-rc", "mime_guess", "mysql_async", @@ -17549,7 +17548,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.833.0" +version = "1.835.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5cfd7372cc..6fc1341d53 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.833.0" +version = "1.835.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.833.0" +version = "1.835.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -723,7 +723,6 @@ flume = { version = "0.11.1", features = ["async"] } kube = { version = "1.1.0", features = ["runtime", "derive"] } schemars = "0.8" k8s-openapi = { version = "0.25.0", features = ["latest"] } -libloading = "0.8.8" # Macro-related proc-macro2 = "1.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index eb8eefed27..20a01d16de 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -8979bc9f0e8a1154c66b121d290a4f571a848fb1 +0b57b586217b518f7d15cad024ded1d1c2262f74 diff --git a/backend/migrations/20261009164611_add_agent_favorite_kind.down.sql b/backend/migrations/20261009164611_add_agent_favorite_kind.down.sql new file mode 100644 index 0000000000..c50ff19ab5 --- /dev/null +++ b/backend/migrations/20261009164611_add_agent_favorite_kind.down.sql @@ -0,0 +1 @@ +DELETE FROM favorite WHERE favorite_kind = 'agent'; diff --git a/backend/migrations/20261009164611_add_agent_favorite_kind.up.sql b/backend/migrations/20261009164611_add_agent_favorite_kind.up.sql new file mode 100644 index 0000000000..c0959d2527 --- /dev/null +++ b/backend/migrations/20261009164611_add_agent_favorite_kind.up.sql @@ -0,0 +1,8 @@ +DO +$do$ +BEGIN + ALTER TYPE FAVORITE_KIND ADD VALUE 'agent'; +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Couldn''t create FAVORITE_KIND::agent: %', SQLERRM; +END +$do$; diff --git a/backend/migrations/20261009173546_agent_owner_prefix_index.down.sql b/backend/migrations/20261009173546_agent_owner_prefix_index.down.sql new file mode 100644 index 0000000000..d3e9318571 --- /dev/null +++ b/backend/migrations/20261009173546_agent_owner_prefix_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_resource_agent_owner_prefix; diff --git a/backend/migrations/20261009173546_agent_owner_prefix_index.up.sql b/backend/migrations/20261009173546_agent_owner_prefix_index.up.sql new file mode 100644 index 0000000000..be1fd5813f --- /dev/null +++ b/backend/migrations/20261009173546_agent_owner_prefix_index.up.sql @@ -0,0 +1,7 @@ +-- Backs the agent branch of /runnables/list and /runnables/counts, which reach the +-- `ai_agent` rows of `resource` by workspace and byte-ordered path prefix. Partial, so +-- neither walks the workspace's other resources (state, cache, ...). +-- Created CONCURRENTLY via the OVERRIDDEN_MIGRATIONS rewrite in windmill-api/src/db.rs. +CREATE INDEX IF NOT EXISTS idx_resource_agent_owner_prefix + ON resource (workspace_id, path text_pattern_ops) + WHERE resource_type = 'ai_agent'; diff --git a/backend/migrations/20261015063444_app_script_modules.down.sql b/backend/migrations/20261015063444_app_script_modules.down.sql new file mode 100644 index 0000000000..026e1e841a --- /dev/null +++ b/backend/migrations/20261015063444_app_script_modules.down.sql @@ -0,0 +1 @@ +ALTER TABLE app_script DROP COLUMN IF EXISTS modules; diff --git a/backend/migrations/20261015063444_app_script_modules.up.sql b/backend/migrations/20261015063444_app_script_modules.up.sql new file mode 100644 index 0000000000..aa8e6d5fb2 --- /dev/null +++ b/backend/migrations/20261015063444_app_script_modules.up.sql @@ -0,0 +1 @@ +ALTER TABLE app_script ADD COLUMN IF NOT EXISTS modules JSONB; diff --git a/backend/migrations/20261015114712_queue_workspace_scheduled_for_index.down.sql b/backend/migrations/20261015114712_queue_workspace_scheduled_for_index.down.sql new file mode 100644 index 0000000000..bebd678a1a --- /dev/null +++ b/backend/migrations/20261015114712_queue_workspace_scheduled_for_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS v2_job_queue_workspace_scheduled_for; diff --git a/backend/migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql b/backend/migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql new file mode 100644 index 0000000000..63a22e003b --- /dev/null +++ b/backend/migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql @@ -0,0 +1,10 @@ +-- Serves push_scheduled_job's already-exists check in windmill-queue/src/schedule.rs, which +-- looks up the queued jobs of one workspace at one occurrence. Without it that check scans the +-- whole queue on every schedule push. +-- Dropped before it is built because the OVERRIDDEN_MIGRATIONS rewrite in +-- windmill-api/src/db.rs runs both CONCURRENTLY: an interrupted concurrent build leaves an +-- invalid index that IF NOT EXISTS would skip rebuilding. +DROP INDEX IF EXISTS v2_job_queue_workspace_scheduled_for; + +CREATE INDEX IF NOT EXISTS v2_job_queue_workspace_scheduled_for + ON v2_job_queue (workspace_id, scheduled_for); diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index a29aecb92d..d3ac7a3809 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -3,7 +3,7 @@ "auth_url": "https://github.com/login/oauth/authorize", "token_url": "https://github.com/login/oauth/access_token", "scopes": ["workflow", "repo"], - "scope_options": ["repo", "workflow", "gist", "read:org"] + "scope_options": ["repo", "workflow", "gist", "read:org", "read:project", "project"] }, "gitlab": { "auth_url": "https://gitlab.com/oauth/authorize", @@ -125,6 +125,69 @@ "prompt": "consent" } }, + "ms_onedrive": { + "req_body_auth": true, + "scopes": ["offline_access", "User.Read", "Files.ReadWrite"], + "scope_options": [ + { "value": "offline_access", "label": "Stay connected (refresh token)" }, + { "value": "User.Read", "label": "Read your profile" }, + { "value": "Files.Read", "label": "Read your files" }, + { "value": "Files.ReadWrite", "label": "Read, upload, edit and delete your files" }, + { "value": "Files.Read.All", "label": "Read all files you can access (shared and SharePoint drives)" }, + { "value": "Files.ReadWrite.All", "label": "Read and edit all files you can access (shared and SharePoint drives)" } + ], + "connect_config_template": { + "display_name": "Microsoft OneDrive", + "label": "Microsoft Entra tenant", + "placeholder": "common, organizations, or your tenant id", + "help_url": "https://learn.microsoft.com/en-us/entra/identity-platform/v2-protocols#endpoints", + "auth_url": "https://login.microsoftonline.com/{instance}/oauth2/v2.0/authorize", + "token_url": "https://login.microsoftonline.com/{instance}/oauth2/v2.0/token", + "req_body_auth": true, + "scopes": ["offline_access", "User.Read", "Files.ReadWrite"] + } + }, + "ms_sharepoint": { + "req_body_auth": true, + "scopes": ["offline_access", "User.Read", "Sites.ReadWrite.All"], + "scope_options": [ + { "value": "offline_access", "label": "Stay connected (refresh token)" }, + { "value": "User.Read", "label": "Read your profile" }, + { "value": "Sites.Read.All", "label": "Read sites, lists and files" }, + { "value": "Sites.ReadWrite.All", "label": "Read and edit lists, list items and files in all sites" }, + { "value": "Sites.Manage.All", "label": "Create and manage lists and libraries" } + ], + "connect_config_template": { + "display_name": "Microsoft SharePoint", + "label": "Microsoft Entra tenant", + "placeholder": "common, organizations, or your tenant id", + "help_url": "https://learn.microsoft.com/en-us/entra/identity-platform/v2-protocols#endpoints", + "auth_url": "https://login.microsoftonline.com/{instance}/oauth2/v2.0/authorize", + "token_url": "https://login.microsoftonline.com/{instance}/oauth2/v2.0/token", + "req_body_auth": true, + "scopes": ["offline_access", "User.Read", "Sites.ReadWrite.All"] + } + }, + "ms_excel": { + "req_body_auth": true, + "scopes": ["offline_access", "User.Read", "Files.ReadWrite"], + "scope_options": [ + { "value": "offline_access", "label": "Stay connected (refresh token)" }, + { "value": "User.Read", "label": "Read your profile" }, + { "value": "Files.ReadWrite", "label": "Read and edit workbooks in your OneDrive" }, + { "value": "Files.ReadWrite.All", "label": "Read and edit workbooks in shared and SharePoint drives" } + ], + "connect_config_template": { + "display_name": "Microsoft Excel", + "label": "Microsoft Entra tenant", + "placeholder": "common, organizations, or your tenant id", + "help_url": "https://learn.microsoft.com/en-us/entra/identity-platform/v2-protocols#endpoints", + "auth_url": "https://login.microsoftonline.com/{instance}/oauth2/v2.0/authorize", + "token_url": "https://login.microsoftonline.com/{instance}/oauth2/v2.0/token", + "req_body_auth": true, + "scopes": ["offline_access", "User.Read", "Files.ReadWrite"] + } + }, "gmail": { "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 5c8de9790f..f1718c6d94 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -268,6 +268,33 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result>, +) -> Option> { + let dir = script_path + .rsplit_once('/') + .map(|(dir, _)| dir) + .unwrap_or(""); + let sources: HashMap = modules? + .iter() + .filter_map(|(key, module)| { + let stem = key.strip_suffix(".py")?; + let path = if dir.is_empty() { + stem.to_string() + } else { + format!("{dir}/{stem}") + }; + Some((path, module.content.clone())) + }) + .collect(); + (!sources.is_empty()).then_some(sources) +} + #[cfg(not(target_arch = "wasm32"))] pub async fn parse_python_imports( code: &str, @@ -278,6 +305,10 @@ pub async fn parse_python_imports( locked_v: &mut Option, raw_workspace_dependencies_o: &Option, temp_script_refs: &Option>, + // Content of relative imports that ship with the importer rather than being deployed + // (its modules), keyed by the path a relative import resolves to. Wins over both + // `temp_script_refs` and the deployed script, as the module file does at run time. + module_sources: Option<&HashMap>, ) -> error::Result<(Vec, Option)> { let mut compile_error_hint: Option = None; let mut imports = parse_python_imports_inner( @@ -291,6 +322,7 @@ pub async fn parse_python_imports( locked_v, raw_workspace_dependencies_o, temp_script_refs, + module_sources, ) .await? .into_values() @@ -349,6 +381,7 @@ async fn parse_python_imports_inner( locked_v: &mut Option, raw_workspace_dependencies_o: &Option, temp_script_refs: &Option>, + module_sources: Option<&HashMap>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } = @@ -517,10 +550,11 @@ async fn parse_python_imports_inner( for n in nimports.into_iter() { let mut nested = match n { NImport::Relative(rpath) => { - // First try to get content from temp_script_refs cache if available - let code_from_cache = if let Some(hash) = - temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) + let code_from_cache = if let Some(content) = + module_sources.and_then(|m| m.get(&rpath)) { + Some(content.clone()) + } else if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { tracing::debug!( "Found relative import '{}' in temp_script_refs with hash '{}'", rpath, @@ -570,6 +604,7 @@ async fn parse_python_imports_inner( locked_v, raw_workspace_dependencies_o, temp_script_refs, + module_sources, ) .await? .into_values() diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index 9086b29d47..6d3d75a239 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -29,6 +29,7 @@ def main(): &mut None, &None, &None, + None, ) .await?; // println!("{}", serde_json::to_string(&r)?); @@ -71,6 +72,7 @@ def main(): &mut None, &None, &None, + None, ) .await?; println!("{}", serde_json::to_string(&r)?); @@ -103,6 +105,7 @@ def main(): &mut None, &None, &None, + None, ) .await?; println!("{}", serde_json::to_string(&r)?); diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index c0dcab1c78..cdd8ab75f5 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6340,7 +6340,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.833.0" +version = "1.835.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -6430,7 +6430,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.833.0" +version = "1.835.0" dependencies = [ "proc-macro2", "quote", @@ -6442,7 +6442,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.833.0" +version = "1.835.0" dependencies = [ "convert_case", "serde", @@ -6451,7 +6451,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde_json", @@ -6475,7 +6475,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "gosyn", @@ -6487,7 +6487,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -6499,7 +6499,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde_json", @@ -6511,7 +6511,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "nu-parser", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6533,7 +6533,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6545,7 +6545,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "ruff_python_ast", @@ -6556,7 +6556,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "async-recursion", @@ -6579,7 +6579,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde_json", @@ -6591,7 +6591,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -6605,7 +6605,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "convert_case", @@ -6622,7 +6622,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde", @@ -6647,7 +6647,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -6665,7 +6665,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6681,7 +6681,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "ruff_python_ast", @@ -6698,7 +6698,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6730,7 +6730,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "lazy_static", @@ -6744,7 +6744,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.833.0" +version = "1.835.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index ca10c14ff6..aa6c5cc297 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.833.0" +version = "1.835.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index fa1c049746..471a7251b3 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -10,7 +10,7 @@ autoscaling_event_type: full_scaleout, scalein, scaleout aws_auth_resource_type: oidc, credentials delivery_mode: push, pull draft_type: script, flow, app -favorite_kind: app, script, flow, raw_app, asset +favorite_kind: app, script, flow, raw_app, asset, agent gcp_subscription_mode: create_update, existing http_method: get, post, put, delete, patch importer_kind: script, flow, app @@ -48,7 +48,7 @@ alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(b app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[]) FK: (workspace_id) -> workspace(id) app_bundles: app_version_id(bigint), w_id(char), file_type(char), data(bytes) -app_script: id(bigint), app(bigint), hash(char(64)), lock(text), code(text), code_sha256(char(64)) +app_script: id(bigint), app(bigint), hash(char(64)), lock(text), code(text), code_sha256(char(64)), modules(jsonb) FK: (app) -> app(id) app_version: id(bigint), app_id(bigint), value(json), created_by(char), created_at(ts), raw_app(bool) FK: (app_id) -> app(id) diff --git a/backend/tests/app_preview_auth.rs b/backend/tests/app_preview_auth.rs index cd84e6f278..f8e90ce144 100644 --- a/backend/tests/app_preview_auth.rs +++ b/backend/tests/app_preview_auth.rs @@ -312,13 +312,11 @@ async fn test_app_preview_authorization(db: Pool) -> anyhow::Result<() "run-mode unpinned raw_code must be rejected for any caller, not just operators (got {status}): {body}" ); - // 11. The pin requirement also covers `raw_code` carrying an `app_script` - // `id`. The id resolves to `rawscript/` of any app_script row - // by number (no app scoping), so without the pin a caller could run a - // script belonging to another app against this Viewer app. Here `999777` - // belongs to `u/test-user/private`, not to the targeted `vapp`, and its - // sha is absent from `vapp`'s empty `triggerables_v2` — it must be - // rejected, not fall back to the Viewer default. + // 11. `raw_code` carrying an `app_script` `id` from another app must not run + // against this Viewer app: the row carries its own modules and lock, which + // would run under `vapp`'s identity. Here `999777` belongs to + // `u/test-user/private`, not to the targeted `vapp`, so the id lookup + // (scoped to the app) refuses it before any Viewer default applies. let resp = authed( client().post(format!("{base}/u/test-user/vapp")), "OPERATOR_TOKEN", @@ -338,12 +336,12 @@ async fn test_app_preview_authorization(db: Pool) -> anyhow::Result<() let status = resp.status(); let body = resp.text().await?; assert_eq!( - status, 400, - "run-mode raw_code with an unpinned app_script id must be rejected against a Viewer app (got {status}): {body}" + status, 401, + "run-mode raw_code with another app's app_script id must be rejected against a Viewer app (got {status}): {body}" ); assert!( - body.contains("forbidden by policy"), - "rejection must be the content-hash pin (unpinned app_script sha), got: {body}" + body.contains("does not belong to app u/test-user/vapp"), + "rejection must be the app scoping of the app_script id, got: {body}" ); // 12. Path confinement: the scope picker mints `apps:run:` / diff --git a/backend/tests/datatable_fork_migrations.rs b/backend/tests/datatable_fork_migrations.rs new file mode 100644 index 0000000000..50f4d89705 --- /dev/null +++ b/backend/tests/datatable_fork_migrations.rs @@ -0,0 +1,148 @@ +//! A fork's copy of a data table must report the migrations its schema already holds as applied. +//! +//! A schema-only copy restores `_wm_migrations` without its rows, and no copy carries the source's +//! `ALTER DATABASE ... SET search_path`, through which the bookkeeping table is found. Either one +//! alone makes the fork list every applied migration as pending, and running them fails on +//! objects that already exist. The source's `DateStyle` differs from the copy's while the rows +//! are written, so they must not travel in its text form. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +const ROLE: &str = "wm_dtfork_test_role"; +const ROLE_PASSWORD: &str = "wm_dtfork_test_pwd"; + +fn authed(b: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + b.header("Authorization", "Bearer DTMIG_ADMIN_TOKEN") +} + +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn a_forked_copy_keeps_its_applied_migrations(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let opts = (*db.connect_options()).clone(); + // The copy takes its settings locks on a connection of its own, opened from `DATABASE_URL` + // rather than the pool: point that at this test's database, not the one sqlx creates it from. + // Only this test runs in the binary, and before anything has read the variable. + let base_url = std::env::var("DATABASE_URL")?; + let (prefix, rest) = base_url + .rsplit_once('/') + .expect("DATABASE_URL names a database"); + let query = rest + .split_once('?') + .map(|(_, q)| format!("?{q}")) + .unwrap_or_default(); + let test_db = opts.get_database().expect("test database name"); + std::env::set_var("DATABASE_URL", format!("{prefix}/{test_db}{query}")); + // Databases are cluster-wide, so both are named for this run alone. + let suffix = format!( + "{:x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() + ); + let source = format!("wm_dtfork_src_{suffix}"); + let fork_id = format!("wm-fork-dtfork-{suffix}"); + let copy = format!("wm_fork_dtfork_{suffix}__main"); + + sqlx::query(&format!( + "DO $$ BEGIN \ + CREATE ROLE {ROLE} LOGIN CREATEDB PASSWORD '{ROLE_PASSWORD}'; \ + EXCEPTION WHEN duplicate_object OR unique_violation THEN NULL; \ + END $$" + )) + .execute(&db) + .await?; + sqlx::query(&format!("CREATE DATABASE {source} OWNER {ROLE}")) + .execute(&db) + .await?; + + let result = async { + let source_pool = sqlx::PgPool::connect_with(opts.clone().database(&source)).await?; + sqlx::raw_sql(&format!( + "SET ROLE {ROLE}; \ + CREATE SCHEMA app; \ + CREATE TABLE app.foo (id int); \ + CREATE TABLE app._wm_migrations (datatable TEXT NOT NULL, version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (datatable, version)); \ + INSERT INTO app._wm_migrations (datatable, version, installed_at) \ + VALUES ('main', 1, '2026-01-31 10:00:00+00'); \ + ALTER DATABASE {source} SET search_path = app; \ + ALTER DATABASE {source} SET datestyle = 'SQL, DMY';" + )) + .execute(&source_pool) + .await?; + source_pool.close().await; + + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, created_by) \ + VALUES ('dtmig-ws', 'u/dtmig-admin/pg', $1, 'postgresql', 'dtmig-admin')", + ) + .bind(json!({ + "host": opts.get_host(), + "port": opts.get_port(), + "dbname": source, + "user": ROLE, + "password": ROLE_PASSWORD, + "sslmode": "disable", + })) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) \ + VALUES ('dtmig-ws', 'main', 1, 'create_foo', 'CREATE TABLE foo (id int)')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api", server.addr.port()); + let client = reqwest::Client::new(); + + let resp = authed(client.post(format!("{base}/w/dtmig-ws/workspaces/create_fork"))) + .json(&json!({ + "id": fork_id, + "name": "fork", + "forked_datatables": [ + {"name": "main", "new_dbname": copy, "fork_behavior": "schema_only"} + ], + })) + .send() + .await?; + // `ensure!` rather than `assert!`: a panic would skip dropping the databases below. + let fork_status = resp.status(); + anyhow::ensure!(fork_status == 200, "{}", resp.text().await?); + + let status: Value = authed(client.get(format!( + "{base}/w/{fork_id}/workspaces/datatable_migrations_status/main" + ))) + .send() + .await? + .json() + .await?; + anyhow::ensure!(status["migrations"][0]["status"] == "ran", "{status}"); + + let copied: Vec = sqlx::query_scalar( + "SELECT unnest(setconfig) FROM pg_db_role_setting \ + WHERE setrole = 0 AND setdatabase = (SELECT oid FROM pg_database WHERE datname = $1) \ + ORDER BY 1", + ) + .bind(©) + .fetch_all(&db) + .await?; + anyhow::ensure!( + copied == ["DateStyle=SQL, DMY", "search_path=app"], + "{copied:?}" + ); + anyhow::Ok(()) + } + .await; + + for name in [©, &source] { + sqlx::query(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .execute(&db) + .await?; + } + result +} diff --git a/backend/tests/datatable_migration_add_down.rs b/backend/tests/datatable_migration_add_down.rs index 47ea4c253b..1259628068 100644 --- a/backend/tests/datatable_migration_add_down.rs +++ b/backend/tests/datatable_migration_add_down.rs @@ -174,3 +174,34 @@ async fn test_add_down_to_a_migration_whose_comment_predates_role_annotations( assert_eq!(resp.status(), 200, "{}", resp.text().await?); Ok(()) } + +/// Likewise for `-- schemas`: the scope annotation is held only to up SQL being written, so a +/// definition whose stored up opens with prose starting with the word still accepts its down. +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_add_down_to_a_migration_whose_comment_predates_scope_annotations( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + setup_applied_migration_without_down(&db).await?; + let code_up = format!("-- Schemas: changes for the widgets feature\n{CODE_UP}"); + sqlx::query("UPDATE datatable_migrations SET code_up = $1 WHERE workspace_id = 'dtmig-ws'") + .bind(&code_up) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = authed(reqwest::Client::new().post(format!( + "http://localhost:{}/api/w/dtmig-ws/workspaces/upsert_datatable_migration/main", + server.addr.port() + ))) + .json(&json!({ + "timestamp": VERSION, + "name": "create_widgets", + "code_up": code_up, + "code_down": CODE_DOWN, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/tests/dedicated_workers.rs b/backend/tests/dedicated_workers.rs index 5e9c371f90..d0ba3e5e63 100644 --- a/backend/tests/dedicated_workers.rs +++ b/backend/tests/dedicated_workers.rs @@ -23,6 +23,20 @@ mod dedicated_worker_tests { WorkspacedPath { workspace_id: workspace.to_string(), path: path.to_string() } } + /// Waits on one listener for every job: the jobs may complete in any order, and a listener + /// opened after one completes never sees that notification. + async fn wait_for_all( + mut listener: impl futures::Stream + Unpin, + jobs: &[uuid::Uuid], + ) { + use futures::StreamExt; + let mut pending: std::collections::HashSet<_> = jobs.iter().copied().collect(); + while !pending.is_empty() { + let id = listener.next().await.expect("completed job listener ended"); + pending.remove(&id); + } + } + /// Assert that a completed job (or any descendant job) was executed /// by a dedicated worker process with the expected mode by checking job logs /// for the "dedicated worker {mode}:" prefix. @@ -280,10 +294,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - // Wait for both jobs to complete - listener.find(&uuid_a).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_b).await; + wait_for_all(listener, &[uuid_a, uuid_b]).await; }, port, vec![ @@ -533,9 +544,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_a).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_b).await; + wait_for_all(listener, &[uuid_a, uuid_b]).await; }, port, vec![ @@ -659,9 +668,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_standalone).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_flow).await; + wait_for_all(listener, &[uuid_standalone, uuid_flow]).await; }, port, vec![ @@ -824,9 +831,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_a).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_b).await; + wait_for_all(listener, &[uuid_a, uuid_b]).await; }, port, vec![ @@ -898,9 +903,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_pre).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_other).await; + wait_for_all(listener, &[uuid_pre, uuid_other]).await; }, port, vec![ @@ -971,9 +974,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_branch).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_default).await; + wait_for_all(listener, &[uuid_branch, uuid_default]).await; }, port, vec![wp("test-workspace", "flow/f/system/dedicated_branch_flow")], @@ -1198,9 +1199,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_pre).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_other).await; + wait_for_all(listener, &[uuid_pre, uuid_other]).await; }, port, vec![ @@ -1281,9 +1280,7 @@ mod dedicated_worker_tests { in_test_worker_dedicated( db.clone(), async { - listener.find(&uuid_ws1).await; - let listener2 = listen_for_completed_jobs(&db).await; - listener2.find(&uuid_ws2).await; + wait_for_all(listener, &[uuid_ws1, uuid_ws2]).await; }, port, vec![ diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs index 15eb9a64f1..7fad5ab5ef 100644 --- a/backend/tests/end_user_email.rs +++ b/backend/tests/end_user_email.rs @@ -17,6 +17,9 @@ const SAME_WS_EMAIL: &str = "test@windmill.dev"; const OTHER_WS_EMAIL: &str = "other-ws@windmill.dev"; const NO_WS_EMAIL: &str = "no-ws@windmill.dev"; +const GET_EMAIL_CONTENT: &str = + "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"; + fn client() -> reqwest::Client { reqwest::Client::new() } @@ -38,9 +41,11 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "subgrids": {}, "hiddenInlineScripts": [{ "name": "get_email", - "language": "bun", - "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", - "path": "f/test/email_app/get_email" + "inlineScript": { + "language": "bun", + "content": GET_EMAIL_CONTENT, + "path": "f/test/email_app/get_email" + } }] }, "policy": { @@ -82,11 +87,13 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "value": { "type": "rawapp", "css": "", - "inlineScripts": [{ - "name": "get_email", - "language": "bun", - "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" - }] + "runnables": { + "get_email": { + "name": "get_email", + "type": "inline", + "inlineScript": { "language": "bun", "content": GET_EMAIL_CONTENT } + } + } }, "policy": { "execution_mode": "anonymous", @@ -183,7 +190,39 @@ async fn run_app_flow( wait_for_job_result(port, token, &job_id).await } -async fn run_app_inline_script( +fn find_inline_script_id(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::Object(m) => m + .get("inlineScript") + .and_then(|s| s.get("id")) + .and_then(|id| id.as_i64()) + .or_else(|| m.values().find_map(find_inline_script_id)), + serde_json::Value::Array(a) => a.iter().find_map(find_inline_script_id), + _ => None, + } +} + +/// Once the app's dependency job writes the deployed lite value, inline code runs only through +/// the `app_script` rows it references, so the run must carry that row's id. +async fn deployed_inline_script_id(port: u16, app_path: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/get/lite/{}", + port, app_path + ); + for _ in 0..300 { + let resp = authed(client().get(&url), SAME_WS_TOKEN).send().await?; + if resp.status().is_success() { + let app = resp.json::().await?; + if let Some(id) = find_inline_script_id(&app["value"]) { + return Ok(id); + } + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + anyhow::bail!("timeout waiting for the deployed app_script id of {app_path}") +} + +async fn run_inline_script( port: u16, token: &str, app_path: &str, @@ -196,9 +235,10 @@ async fn run_app_inline_script( let mut payload = json!({ "args": {}, "component": "get_email", + "id": deployed_inline_script_id(port, app_path).await?, "raw_code": { "language": "bun", - "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", + "content": GET_EMAIL_CONTENT, "path": format!("{}/get_email", app_path) } }); @@ -220,42 +260,6 @@ async fn run_app_inline_script( wait_for_job_result(port, token, &job_id).await } -async fn run_raw_app_inline_script( - port: u16, - token: &str, - app_path: &str, - force_viewer: bool, -) -> anyhow::Result { - let url = format!( - "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", - port, app_path - ); - let mut payload = json!({ - "args": {}, - "component": "get_email", - "raw_code": { - "language": "bun", - "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" - } - }); - if force_viewer { - payload["force_viewer_static_fields"] = json!({}); - } - let resp = authed(client().post(&url), token) - .json(&payload) - .send() - .await?; - if !resp.status().is_success() { - anyhow::bail!( - "raw app inline script run failed: {} - {}", - resp.status(), - resp.text().await? - ); - } - let job_id = resp.text().await?; - wait_for_job_result(port, token, &job_id).await -} - async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Result { let url = format!( "http://localhost:{}/api/w/test-workspace/jobs_u/completed/get_result/{}", @@ -399,21 +403,21 @@ async fn test_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { create_app_with_inline_script(port, app_path).await?; // Same workspace user (force_viewer mode works for workspace members) - let result = run_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + let result = run_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; assert_eq!( result, SAME_WS_EMAIL, "same workspace user should get their email" ); // Other workspace user (uses app's anonymous policy + token lookup) - let result = run_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + let result = run_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; assert_eq!( result, OTHER_WS_EMAIL, "other workspace user should get their email" ); // No workspace user (uses app's anonymous policy + token lookup) - let result = run_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + let result = run_inline_script(port, NO_WS_TOKEN, app_path, false).await?; assert_eq!( result, NO_WS_EMAIL, "no workspace user should get their email" @@ -481,21 +485,21 @@ async fn test_raw_app_wm_end_user_email(db: Pool) -> anyhow::Result<() create_raw_app_with_inline_script(port, app_path).await?; // Same workspace user (force_viewer mode works for workspace members) - let result = run_raw_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + let result = run_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; assert_eq!( result, SAME_WS_EMAIL, "same workspace user should get their email" ); // Other workspace user (uses app's anonymous policy + token lookup) - let result = run_raw_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + let result = run_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; assert_eq!( result, OTHER_WS_EMAIL, "other workspace user should get their email" ); // No workspace user (uses app's anonymous policy + token lookup) - let result = run_raw_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + let result = run_inline_script(port, NO_WS_TOKEN, app_path, false).await?; assert_eq!( result, NO_WS_EMAIL, "no workspace user should get their email" @@ -550,7 +554,11 @@ async fn test_app_viewer_data_access_is_stamped_on_inline_runnables_only( })) .send() .await?; - anyhow::ensure!(resp.status().is_success(), "create app: {}", resp.text().await?); + anyhow::ensure!( + resp.status().is_success(), + "create app: {}", + resp.text().await? + ); let execute = |payload: serde_json::Value| async move { let resp = authed( @@ -562,7 +570,11 @@ async fn test_app_viewer_data_access_is_stamped_on_inline_runnables_only( .json(&payload) .send() .await?; - anyhow::ensure!(resp.status().is_success(), "execute: {}", resp.text().await?); + anyhow::ensure!( + resp.status().is_success(), + "execute: {}", + resp.text().await? + ); Ok::<_, anyhow::Error>(resp.text().await?.trim_matches('"').parse::()?) }; let as_viewer = |job_id: uuid::Uuid| { @@ -588,7 +600,8 @@ async fn test_app_viewer_data_access_is_stamped_on_inline_runnables_only( .await?; assert!(as_viewer(inline).await?); - let path = execute(json!({ "args": {}, "component": "run_script", "path": script_path })).await?; + let path = + execute(json!({ "args": {}, "component": "run_script", "path": script_path })).await?; assert!( !as_viewer(path).await?, "a path runnable connected as the viewer because its policy entry said so" diff --git a/backend/tests/relock_noop.rs b/backend/tests/relock_noop.rs index a480fc57ae..8760909be9 100644 --- a/backend/tests/relock_noop.rs +++ b/backend/tests/relock_noop.rs @@ -437,3 +437,609 @@ async fn multi_file_importer_relock_is_a_no_op_too(db: Pool) -> anyhow Ok(()) } + +/// Plants raw app `f/rel/app` with python runnables of the given contents, one of which imports +/// `f/rel/a`, as a version deployed before runnables were keyed by name: recorded under the +/// empty node id and each holding a lock already. Returns its published version. +async fn plant_raw_app_with( + db: &Pool, + runnables: &[(&str, &str)], +) -> anyhow::Result { + let runnables: serde_json::Map = runnables + .iter() + .map(|(id, content)| { + let runnable = serde_json::json!({ "type": "inline", "inlineScript": { + "content": content, "language": "python3", "lock": "# py: 3.12\n" + }}); + (id.to_string(), runnable) + }) + .collect(); + let app_id: i64 = sqlx::query_scalar( + "INSERT INTO app (workspace_id, path, summary, policy, versions, extra_perms) + VALUES ($1, 'f/rel/app', '', '{}'::jsonb, '{}', '{}'::jsonb) RETURNING id", + ) + .bind(W) + .fetch_one(db) + .await?; + let published: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, raw_app) + VALUES ($1, $2, 'test-user', true) RETURNING id", + ) + .bind(app_id) + .bind(serde_json::json!({ "runnables": runnables })) + .fetch_one(db) + .await?; + sqlx::query("UPDATE app SET versions = ARRAY[$1::bigint] WHERE id = $2") + .bind(published) + .bind(app_id) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) + VALUES ($1, 'f/rel/app', 'app', 'f/rel/a', '')", + ) + .bind(W) + .execute(db) + .await?; + Ok(published) +} + +/// [`plant_raw_app_with`] a single runnable `a` importing `f/rel/a`. +async fn plant_raw_app(db: &Pool) -> anyhow::Result { + plant_raw_app_with(db, &[("a", B)]).await +} + +async fn app_lock(db: &Pool, version: i64) -> Option { + runnable_lock(db, version, "a").await +} + +async fn runnable_lock(db: &Pool, version: i64, runnable: &str) -> Option { + sqlx::query_scalar( + "SELECT value->'runnables'->$2->'inlineScript'->>'lock' FROM app_version WHERE id = $1", + ) + .bind(version) + .bind(runnable) + .fetch_one(db) + .await + .unwrap() +} + +const WRITTEN_MEANWHILE: &str = "# py: 3.12\n# written meanwhile\n"; + +/// Once an app dependency job waits on the version row `writer` holds, rewrites that row's +/// lock of runnable `a` to [`WRITTEN_MEANWHILE`] and commits. +async fn write_once_waited_on( + db: &Pool, + mut writer: sqlx::Transaction<'_, Postgres>, + version: i64, +) { + wait_until_blocked(db, "SELECT value FROM app_version%").await; + sqlx::query( + "UPDATE app_version SET value = jsonb_set(value::jsonb, + '{runnables,a,inlineScript,lock}', to_jsonb($2::text))::json WHERE id = $1", + ) + .bind(version) + .bind(WRITTEN_MEANWHILE) + .execute(&mut *writer) + .await + .unwrap(); + writer.commit().await.unwrap(); +} + +/// Waits until a statement of this test's database matching `query` waits on a row lock. +async fn wait_until_blocked(db: &Pool, query: &str) { + let deadline = tokio::time::Instant::now() + WAIT_BUDGET; + loop { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' AND query LIKE $1)", + ) + .bind(query) + .fetch_one(db) + .await + .unwrap(); + if waiting { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "nothing ever waited on a row lock for {query}" + ); + tokio::time::sleep(POLL).await; + } +} + +/// Queues a dependency job of `f/rel/app` at `version`: the deploy's own when `components` is +/// `None`, else a relative-import relock of those components. +async fn push_app_dependency_job(db: &Pool, version: i64, components: Option<&[&str]>) { + let payload = windmill_common::jobs::JobPayload::AppDependencies { + path: "f/rel/app".to_string(), + version, + debouncing_settings: Default::default(), + }; + let mut args = std::collections::HashMap::new(); + if let Some(components) = components { + args.insert( + "triggered_by_relative_import".to_string(), + serde_json::value::to_raw_value(&true).unwrap(), + ); + args.insert( + "components_to_relock".to_string(), + serde_json::value::to_raw_value(components).unwrap(), + ); + } + let (_, tx) = windmill_queue::push( + db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + W, + payload, + windmill_queue::PushArgs { args: &args, extra: None }, + "test-user", + "test@windmill.dev", + "u/test-user".to_string(), + None, + None, + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + None, + false, + None, + None, + None, + None, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); +} + +/// Logs of every dependency job of `f/rel/app` created after `since`, oldest first. +async fn app_dependency_logs( + db: &Pool, + since: chrono::DateTime, +) -> Vec { + sqlx::query_scalar( + "SELECT COALESCE(l.logs, '') FROM v2_job j + LEFT JOIN job_logs l ON l.job_id = j.id + WHERE j.kind = 'appdependencies' AND j.runnable_path = 'f/rel/app' AND j.created_at > $1 + ORDER BY j.created_at", + ) + .bind(since) + .fetch_all(db) + .await + .unwrap() +} + +/// A raw app runnable importing a script is relocked when that script's dependencies change, +/// and the relock lands in the version the app is served from rather than in a new one: a raw +/// app's bundle is stored under its version id. +#[sqlx::test(fixtures("base"))] +async fn raw_app_relock_rewrites_the_published_version(db: Pool) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + let published = plant_raw_app(&db).await?; + + in_test_worker( + &db, + async { + client + .create_script(W, &py_script("f/rel/a", A, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_WITH_TINY, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 2).await; + + let (versions, version_rows): (Vec, i64) = sqlx::query_as( + "SELECT versions, (SELECT count(*) FROM app_version WHERE app_id = app.id) + FROM app WHERE workspace_id = $1 AND path = 'f/rel/app'", + ) + .bind(W) + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(versions, [published], "the relock publishes no other version"); + assert_eq!(version_rows, 1, "the relock mints no version"); + + let lock = app_lock(&db, published).await; + assert!( + lock.as_deref().unwrap_or("").contains("tiny"), + "the published version carries the new lock: {lock:?}" + ); + + let node_ids: Vec = sqlx::query_scalar( + "SELECT importer_node_id FROM dependency_map + WHERE workspace_id = $1 AND importer_path = 'f/rel/app' AND imported_path = 'f/rel/a'", + ) + .bind(W) + .fetch_all(&db) + .await + .unwrap(); + assert_eq!(node_ids, ["a"], "the runnable is recorded under its name"); + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock rewrites the one row every other relock of the app, and the dependency job of the +/// deploy that published it, write too. When that row changed after the relock read it, the +/// relock is discarded and queued again on what is there now, rather than overwriting it. +#[sqlx::test(fixtures("base"))] +async fn raw_app_relock_racing_a_write_requeues(db: Pool) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + let published = plant_raw_app(&db).await?; + + in_test_worker( + &db, + async { + client + .create_script(W, &py_script("f/rel/a", A, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + + let mut writer = db.begin().await.unwrap(); + sqlx::query("SELECT 1 FROM app_version WHERE id = $1 FOR NO KEY UPDATE") + .bind(published) + .execute(&mut *writer) + .await + .unwrap(); + + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_WITH_TINY, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + + write_once_waited_on(&db, writer, published).await; + + // The leaf's own job, the discarded relock and the one it queued. + wait_for_jobs(&db, &mut completed, 3).await; + + let logs = app_dependency_logs(&db, since).await; + assert_eq!(logs.len(), 2, "one relock and its requeue: {logs:?}"); + assert!( + logs[0].contains("discarding them"), + "the first relock was discarded: {}", + logs[0] + ); + let lock = app_lock(&db, published).await; + assert!( + lock.as_deref().unwrap_or("").contains("tiny"), + "the queued relock regenerated the lock over the concurrent write: {lock:?}" + ); + }, + port, + ) + .await; + + Ok(()) +} + +/// The dependency job of the deploy that published a version writes the row relocks write +/// too, so it backs off the same way when that row changed after it read it. +#[sqlx::test(fixtures("base"))] +async fn app_deploy_dependency_job_racing_a_write_requeues( + db: Pool, +) -> anyhow::Result<()> { + let (_client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + let published = plant_raw_app(&db).await?; + + in_test_worker( + &db, + async { + let mut writer = db.begin().await.unwrap(); + sqlx::query("SELECT 1 FROM app_version WHERE id = $1 FOR NO KEY UPDATE") + .bind(published) + .execute(&mut *writer) + .await + .unwrap(); + + let since = chrono::Utc::now(); + push_app_dependency_job(&db, published, None).await; + + write_once_waited_on(&db, writer, published).await; + + // The deploy's job, and the relock it queued on backing off. + wait_for_jobs(&db, &mut completed, 2).await; + + let logs = app_dependency_logs(&db, since).await; + assert_eq!(logs.len(), 2, "the deploy's job and its requeue: {logs:?}"); + assert!( + logs[0].contains("discarding them"), + "the deploy's job backed off: {}", + logs[0] + ); + }, + port, + ) + .await; + + Ok(()) +} + +/// A raw app runnable ships the runnables it imports as modules, and its lock covers what they +/// import, so a change to a script one of them imports relocks the importer too. +#[sqlx::test(fixtures("base"))] +async fn raw_app_relock_follows_imports_of_imported_runnables( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + let published = plant_raw_app_with( + &db, + &[ + ("a", "from .b import g\n\ndef main():\n return g()\n"), + ("b", "from f.rel.a import main as a\n\ndef g():\n return a()\n\ndef main():\n return g()\n"), + ], + ) + .await?; + + in_test_worker( + &db, + async { + // Its relock of the app records what each runnable depends on. + client + .create_script(W, &py_script("f/rel/a", A, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_WITH_TINY, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 2).await; + + for runnable in ["a", "b"] { + let lock = runnable_lock(&db, published, runnable).await; + assert!( + lock.as_deref().unwrap_or("").contains("tiny"), + "{runnable} carries the new lock: {lock:?}" + ); + } + + // A rebuilt dependency map records the same dependencies. + async fn app_edges(db: &Pool) -> Vec<(String, String)> { + sqlx::query_as( + "SELECT importer_node_id, imported_path FROM dependency_map + WHERE workspace_id = $1 AND importer_path = 'f/rel/app' ORDER BY 1, 2", + ) + .bind(W) + .fetch_all(db) + .await + .unwrap() + } + let recorded = app_edges(&db).await; + assert!(rebuild_dmap(&client).await); + assert_eq!(app_edges(&db).await, recorded); + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock superseded by a deploy relocks the version that deploy published, every component +/// of it: the deploy may rename the component the relock was for, keeping its stale lock. +#[sqlx::test(fixtures("base"))] +async fn superseded_app_relock_relocks_the_renamed_component( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + plant_raw_app(&db).await?; + + in_test_worker( + &db, + async { + client + .create_script(W, &py_script("f/rel/a", A, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + + let mut deploy = db.begin().await.unwrap(); + sqlx::query( + "SELECT 1 FROM app WHERE workspace_id = $1 AND path = 'f/rel/app' FOR NO KEY UPDATE", + ) + .bind(W) + .execute(&mut *deploy) + .await + .unwrap(); + + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_WITH_TINY, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + + // The relock has generated its lock for `a` and waits on the app this deploy holds, + // which publishes `a` renamed to `b` with its old lock. + wait_until_blocked(&db, "SELECT versions[array_upper(versions, 1)] FROM app%").await; + let renamed: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, raw_app) + SELECT app_id, json_build_object('runnables', + json_build_object('b', value->'runnables'->'a')), created_by, true + FROM app_version WHERE id = (SELECT versions[array_upper(versions, 1)] FROM app + WHERE workspace_id = $1 AND path = 'f/rel/app') + RETURNING id", + ) + .bind(W) + .fetch_one(&mut *deploy) + .await + .unwrap(); + sqlx::query( + "UPDATE app SET versions = array_append(versions, $2) + WHERE workspace_id = $1 AND path = 'f/rel/app'", + ) + .bind(W) + .bind(renamed) + .execute(&mut *deploy) + .await + .unwrap(); + deploy.commit().await.unwrap(); + + // The leaf's own job, the superseded relock and the one it queued. + wait_for_jobs(&db, &mut completed, 3).await; + + let lock = runnable_lock(&db, renamed, "b").await; + assert!( + lock.as_deref().unwrap_or("").contains("tiny"), + "the renamed component carries the new lock: {lock:?}" + ); + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock queued before a deploy runs on the version that deploy published, every component +/// of it: the deploy may rename the component the relock was queued for, keeping its stale lock. +#[sqlx::test(fixtures("base"))] +async fn app_relock_queued_before_a_deploy_relocks_the_renamed_component( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + let queued_for = plant_raw_app(&db).await?; + + in_test_worker( + &db, + async { + client + .create_script(W, &py_script("f/rel/a", A_WITH_TINY, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + + let renamed: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, raw_app) + SELECT app_id, $2, created_by, true FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(queued_for) + .bind( + serde_json::json!({ "runnables": { "c": { "type": "inline", "inlineScript": { + "content": B, "language": "python3", "lock": "# py: 3.12\n" + }}}}), + ) + .fetch_one(&db) + .await + .unwrap(); + sqlx::query( + "UPDATE app SET versions = array_append(versions, $2) + WHERE workspace_id = $1 AND path = 'f/rel/app'", + ) + .bind(W) + .bind(renamed) + .execute(&db) + .await + .unwrap(); + + push_app_dependency_job(&db, queued_for, Some(&["a"])).await; + wait_for_jobs(&db, &mut completed, 1).await; + + let lock = runnable_lock(&db, renamed, "c").await; + assert!( + lock.as_deref().unwrap_or("").contains("tiny"), + "the renamed component carries the new lock: {lock:?}" + ); + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock of every component regenerates each lock. A deploy that kept its editor's stale +/// lock still records the imports' current lock hashes, so the skip check would keep it. +#[sqlx::test(fixtures("base"))] +async fn app_relock_of_every_component_regenerates_a_kept_stale_lock( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + let published = plant_raw_app(&db).await?; + + in_test_worker( + &db, + async { + client + .create_script(W, &py_script("f/rel/a", A_WITH_TINY, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + + // The stale lock a deploy kept, under the hashes it recorded. + sqlx::query( + "UPDATE app_version SET value = jsonb_set(value::jsonb, + '{runnables,a,inlineScript,lock}', '\"# py: 3.12\\n\"')::json WHERE id = $1", + ) + .bind(published) + .execute(&db) + .await + .unwrap(); + + let every = windmill_dep_map::trigger_dependents::RELOCK_ALL_APP_COMPONENTS; + push_app_dependency_job(&db, published, Some(&[every])).await; + wait_for_jobs(&db, &mut completed, 1).await; + + let lock = app_lock(&db, published).await; + assert!( + lock.as_deref().unwrap_or("").contains("tiny"), + "the kept stale lock is regenerated: {lock:?}" + ); + }, + port, + ) + .await; + + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index e75170ec6f..e15d33354e 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -381,6 +381,22 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho "WM_TOKEN must still record its own tutorial progress: {}", resp.text().await? ); + // Every Teams notification a job sends posts here. An empty body is a 422 from the + // handler's extractor, so a 403 means confinement refused it. The real route is only + // mounted in an EE build. + #[cfg(all(feature = "enterprise", feature = "private"))] + { + let resp = authed(client().post(format!("{api}/teams/activities")), &user_wm) + .json(&json!({})) + .send() + .await?; + assert_eq!( + resp.status(), + 422, + "WM_TOKEN must still reach the Teams message route: {}", + resp.text().await? + ); + } // 11. ...and the caller-scoped reads deliberately left out of it, each because it // names another workspace or the identity's credentials. diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 751752eb00..c47ea6d1ae 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -4698,6 +4698,224 @@ async fn test_run_wait_result_early_return_with_failure_module( Ok(()) } +/// An early-return node inside a branch to one resolves while the flow is still running when its +/// branch is taken, and falls back to the flow's final result when it is not. A nested branch to +/// all resolves to its list, though its status is not in the root flow's modules. +#[cfg(all(feature = "deno_core", feature = "quickjs"))] +#[sqlx::test(fixtures("base"))] +async fn test_run_wait_result_early_return_inside_branch_one( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let returns = |v: &str, sleep_ms: u64| { + json!({ + "type": "rawscript", + "language": "deno", + "input_transforms": {}, + "content": format!( + "export async function main() {{ await new Promise((r) => setTimeout(r, {sleep_ms})); return '{v}' }}" + ), + }) + }; + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "branchone", + "branches": [{ "expr": "flow_input.take", "modules": [ + { + "id": "b", + "value": { + "type": "branchone", + "branches": [], + "default": [{ "id": "c", "value": returns("c", 0) }], + }, + }, + { + "id": "e", + "value": { + "type": "branchall", + "branches": [{ "modules": [{ "id": "f", "value": returns("f", 0) }] }], + }, + }, + ] }], + "default": [], + }, + }, + { "id": "d", "value": returns("d", 3000) }, + ], + })) + .unwrap(); + + for (early_return, take, expected) in [ + ("c", true, json!("c")), + ("c", false, json!("d")), + ("e", true, json!(["f"])), + ] { + let mut flow = flow.clone(); + flow.early_return = Some(early_return.to_string()); + let id = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .arg("take", json!(take)) + .push(&db) + .await; + + // The completion check runs inside the worker's lifetime: the worker finishes `d` before + // quitting, so checking afterwards would always see the flow completed. + let (resp, flow_completed) = in_test_worker( + windmill_common::worker::Connection::Sql(db.clone()), + async { + let resp = windmill_api::jobs::run_wait_result( + &db, + id, + "test-workspace", + Some(early_return.to_string()), + false, + "test-user", + ) + .await + .unwrap(); + let completed: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)", + ) + .bind(id) + .fetch_one(&db) + .await + .unwrap(); + (resp, completed) + }, + port, + ) + .await; + // Only the taken branch returns before `d` finishes sleeping. + assert_eq!(flow_completed, !take, "{early_return}, take = {take}"); + + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX).await?; + let body: serde_json::Value = serde_json::from_slice(&bytes)?; + assert_eq!(expected, body, "{early_return}, take = {take}"); + } + + Ok(()) +} + +/// The step whose stream `run_and_stream` follows: the early-return step inside branches to one +/// and to all, the last step when the early-return step's branch is not taken, and neither when +/// the early-return step ran inside a loop. Only a deployed flow reads `early_return` here. +#[cfg(all(feature = "deno_core", feature = "quickjs"))] +#[sqlx::test(fixtures("base"))] +async fn test_early_return_stream_job_inside_branches(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let streams = |v: &str| { + json!({ + "type": "rawscript", + "language": "deno", + "input_transforms": {}, + "content": format!( + "export async function main() {{ console.log('WM_STREAM: {v}'); await new Promise((r) => setTimeout(r, 1000)); return '{v}' }}" + ), + }) + }; + let modules = json!([ + { + "id": "a", + "value": { + "type": "branchone", + "branches": [{ "expr": "flow_input.take", "modules": [ + { + "id": "b", + "value": { + "type": "branchall", + "branches": [{ "modules": [{ "id": "c", "value": streams("c") }] }], + }, + }, + { + "id": "l", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "[1]" }, + "skip_failures": false, + // Two steps, so the loop body is not inlined into the iteration job. + "modules": [ + { "id": "j", "value": streams("j") }, + { "id": "m", "value": { "type": "identity" } }, + ], + }, + }, + ] }], + "default": [], + }, + }, + { "id": "d", "value": streams("d") }, + ]); + + for (version, early_return, take, expected) in [ + (1, "c", true, Some("c")), + (2, "c", false, Some("d")), + (3, "j", true, None), + ] { + let path = format!("f/system/early_return_stream_{version}"); + let value = json!({ "modules": modules, "early_return": early_return }); + sqlx::query( + "INSERT INTO flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ('test-workspace', '', '', $1, ARRAY[$2]::bigint[], '{}'::jsonb, $3, 'system')", + ) + .bind(&path) + .bind(version) + .bind(&value) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO flow_version(id, workspace_id, path, schema, value, created_by) VALUES ($1, 'test-workspace', $2, '{}'::jsonb, $3, 'system')", + ) + .bind(version) + .bind(&path) + .bind(&value) + .execute(&db) + .await?; + RunJob::from(JobPayload::FlowDependencies { + path: path.clone(), + dedicated_worker: None, + version, + debouncing_settings: Default::default(), + }) + .run_until_complete(&db, false, port) + .await; + + let completed = RunJob::from(JobPayload::Flow { + path, + dedicated_worker: None, + apply_preprocessor: false, + labels: None, + version, + job_token_scopes: None, + }) + .arg("take", json!(take)) + .run_until_complete(&db, false, port) + .await; + + let stream_step: Option = sqlx::query_scalar( + "SELECT flow_step_id FROM v2_job + WHERE id = (SELECT (flow_status->>'stream_job')::uuid FROM v2_job_completed WHERE id = $1)", + ) + .bind(completed.id) + .fetch_optional(&db) + .await?; + assert_eq!( + expected.map(String::from), + stream_step, + "{early_return}, take = {take}" + ); + } + + Ok(()) +} + /// `recover: true` from the error handler of a step that fails inside a loop turns the flow /// green, as it does for a top-level step, without changing which steps run: a loop that /// stops at a failed iteration still stops there, one that skips failures still carries on. diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index 7dcddc93a3..82dceaa02f 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -30,6 +30,11 @@ pub fn effective_reasoning_effort<'a>(model: &str, effort: Option<&'a str>) -> O } } +/// Whether Chat Completions refuses function tools while this model reasons. +pub(crate) fn completions_rejects_tools_with_reasoning(model: &str) -> bool { + reasoning_rule(model).is_some_and(|rule| rule.completions_tools_need_off) +} + /// Whether a request with function tools must send the off sentinel on Chat Completions: /// the model refuses tools there while it reasons, and does accept being turned off. pub(crate) fn completions_tools_need_reasoning_off(model: &str) -> bool { @@ -208,6 +213,12 @@ pub fn create_query_builder( AIProvider::AzureFoundry if AIProvider::is_anthropic_model(model) => Box::new( AnthropicQueryBuilder::new(AIProvider::AzureFoundry, AIPlatform::Standard), ), + // Foundry serves its OpenAI models on `/openai/v1/responses` too. `/chat/completions` + // takes function tools from these only with reasoning off, which gpt-6.1-sol and + // gpt-6-astra cannot do at all. + AIProvider::AzureFoundry if completions_rejects_tools_with_reasoning(model) => { + Box::new(OpenAIQueryBuilder::new(AIProvider::AzureFoundry)) + } // Only for forwarding through the API proxy, which is bearer auth on `/` as // TypeSafe and Cloudflare expect. Neither serves a chat route for its decision models, and // `run_agent` refuses both before a chat request could be built. @@ -408,6 +419,12 @@ mod parity_tests { "claude-sonnet-5", "v1/messages", ), + case( + AIProvider::AzureFoundry, + "https://example.services.ai.azure.com", + "gpt-6.1-sol", + "responses", + ), case( AIProvider::AzureOpenAI, "https://example.openai.azure.com/openai", diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 21f916bf94..bdbab1bfe7 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -73,7 +73,12 @@ pub fn check_route_access( .to_string(), )); } - if required_domain == ScopeDomain::Users && suffix != "users/whoami" { + if required_domain == ScopeDomain::Users { + // Whoami returns what the bundle already has in `window.ctx`, so it + // needs no declared scope. + if suffix == "users/whoami" { + return Ok(()); + } return Err(Error::PermissionDenied( "Access denied. A raw app frontend SDK token can only read the viewer's own identity." .to_string(), @@ -418,8 +423,9 @@ pub fn guest_session_scopes(app_path: &str) -> windmill_common::error::Result) -> bool { @@ -462,8 +468,9 @@ fn app_embed_route_denied(domain: ScopeDomain, suffix: &str) -> bool { } } -/// App routes a running app uses: its own definition (`apps/get/p/`, further -/// path-scoped by `apps:read:`) and the public app-serving endpoints +/// App routes a running app uses: its own definition (`apps/get/p/`, or +/// `apps/get/lite/` that the viewer runs from, both further path-scoped by +/// `apps:read:`) and the public app-serving endpoints /// (`apps_u/*`: public_app, public_resource, get_data, and the path-taking /// `execute_component` / `download_s3_file`, which re-check `apps:run|read:` /// in their handlers so they stay confined to this app). Everything else in the @@ -477,7 +484,9 @@ fn app_embed_apps_route_allowed(suffix: &str) -> bool { if suffix.starts_with("apps_u/embed_token") { return false; } - suffix.starts_with("apps/get/p/") || suffix.starts_with("apps_u/") + suffix.starts_with("apps/get/p/") + || suffix.starts_with("apps/get/lite/") + || suffix.starts_with("apps_u/") } /// Routes a guest session is denied: the app-embed allowlist, plus the embed-token @@ -937,11 +946,15 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool { /// not send. /// - tutorial progress upserts a UI bitfield keyed on the caller's own email. Its path /// serves a `GET` too, which the read list above carries. -const GLOBAL_WRITES_OPEN_TO_JOB_TOKEN: [&str; 4] = [ +/// - a Teams message from a job (the Teams error, recovery and success handlers) posts as +/// the instance-wide bot. The handler refuses a job token whose conversation is not a +/// channel of the team connected to the job's workspace. +const GLOBAL_WRITES_OPEN_TO_JOB_TOKEN: [&str; 5] = [ "/api/settings/test_object_storage_config", "/api/workspaces/exists", "/api/schedules/preview", "/api/users/tutorial_progress", + "/api/teams/activities", ]; /// Confines a job token (`$WM_TOKEN`) to routes that name a workspace. It is minted diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 0acba27c41..764cc3ac04 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -810,7 +810,7 @@ async fn a_data_table_under_roles_is_cloned_only_with_its_grants( // The copy an admin asks for goes ahead in an edition that replays grants, and is refused // before any database exists in one that does not. The fixture's database does not exist, so - // the dump fails either way — and leaves neither a database nor a fork behind. + // reading it fails either way — and leaves neither a database nor a fork behind. let resp = fork( "SECRET_TOKEN", json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_only"}), @@ -820,7 +820,7 @@ async fn a_data_table_under_roles_is_cloned_only_with_its_grants( assert_ne!(resp.status(), 200); let body = resp.text().await?; #[cfg(all(feature = "private", feature = "enterprise"))] - assert!(body.contains("pg_dump"), "{body}"); + assert!(body.contains("\"dt_main\" does not exist"), "{body}"); #[cfg(not(all(feature = "private", feature = "enterprise")))] assert!(body.contains("Enterprise Edition"), "{body}"); assert!(!database_exists(&db, target).await?); @@ -981,6 +981,104 @@ async fn a_data_table_under_roles_is_not_copied_without_its_grants( Ok(()) } +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn data_tables_on_one_database_are_copied_together(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let shared = json!({"database": {"resource_type": "instance", "resource_path": "dt_shared"}}); + sqlx::query( + "UPDATE workspace_settings SET datatable = jsonb_set(jsonb_set(datatable, + '{datatables,a}', $1), '{datatables,b}', $1) WHERE workspace_id = 'test-workspace'", + ) + .bind(&shared) + .execute(&db) + .await?; + + // One copy is made for both, so it cannot be made two ways. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_fork" + )), + "SECRET_TOKEN", + ) + .json( + &json!({"id": "wm-fork-pair", "name": "pair", "forked_datatables": [ + {"name": "a", "new_dbname": "wm_fork_pair__a", "fork_behavior": "schema_only"}, + {"name": "b", "new_dbname": "wm_fork_pair__b", "fork_behavior": "schema_and_data"} + ]}), + ) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("same database")); + assert!(!database_exists(&db, "wm_fork_pair__a").await?); + + // A fork's entries sharing a copy each count as a use of it: dropped together, neither blocks + // the other. + let copy = json!({"database": {"resource_type": "instance", "resource_path": "wm_fork_dt__a"}, + "forked_from": {"schema": {}}}); + sqlx::query( + "UPDATE workspace_settings SET datatable = jsonb_build_object('datatables', + jsonb_build_object('a', $1::jsonb, 'b', $1::jsonb)) WHERE workspace_id = 'wm-fork-dt'", + ) + .bind(©) + .execute(&db) + .await?; + sqlx::query( + "UPDATE global_settings SET value = jsonb_set(value, '{databases,wm_fork_dt__a}', + '{\"tag\": \"datatable\", \"workspace_id\": \"test-workspace\"}') + WHERE name = 'custom_instance_pg_databases'", + ) + .execute(&db) + .await?; + let errors: Vec = authed( + client().post(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/drop_forked_datatable_databases" + )), + "SECRET_TOKEN", + ) + .json(&json!({"datatable_names": ["a", "b"]})) + .send() + .await? + .json() + .await?; + assert!(errors.is_empty(), "{errors:?}"); + let left: Value = sqlx::query_scalar( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = 'wm-fork-dt'", + ) + .fetch_one(&db) + .await?; + assert_eq!(left, json!({})); + + // Clones on one resource share its copy too: dropping it for one would take the other's data. + let copy = json!({"database": {"resource_type": "postgresql", "resource_path": "u/test-user/pg"}, + "forked_from": {"schema": {}}}); + sqlx::query( + "UPDATE workspace_settings SET datatable = jsonb_build_object('datatables', + jsonb_build_object('a', $1::jsonb, 'b', $1::jsonb)) WHERE workspace_id = 'wm-fork-dt'", + ) + .bind(©) + .execute(&db) + .await?; + let errors: Vec = authed( + client().post(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/drop_forked_datatable_databases" + )), + "SECRET_TOKEN", + ) + .json(&json!({"datatable_names": ["a"]})) + .send() + .await? + .json() + .await?; + assert!( + errors.len() == 1 && errors[0].contains("still used by data tables 'b'"), + "{errors:?}" + ); + Ok(()) +} + /// The fork's `forked_from` for one of its entries; `None` whether it is absent or `null`. async fn forked_from_of(db: &Pool, name: &str) -> Option { sqlx::query_scalar::<_, Option>( diff --git a/backend/windmill-api-integration-tests/tests/operator_manage_rights.rs b/backend/windmill-api-integration-tests/tests/operator_manage_rights.rs index ae1979f2fb..22901d1c87 100644 --- a/backend/windmill-api-integration-tests/tests/operator_manage_rights.rs +++ b/backend/windmill-api-integration-tests/tests/operator_manage_rights.rs @@ -4,10 +4,10 @@ use sqlx::{Pool, Postgres}; use windmill_common::workspaces::invalidate_operator_rights_cache; use windmill_test_utils::*; -/// Every test here must be `#[serial]`. The rights cache is process-global and keyed by workspace -/// id alone, while `sqlx::test` gives each test its own database under this same id — so two -/// running at once can answer each other's reads from one cached entry, and a withdrawal in one -/// database reads as granted in the other. +/// Every test here must be `#[serial]`, and start by dropping the cache entry. The rights cache is +/// process-global and keyed by workspace id alone, while `sqlx::test` gives each test its own +/// database under this same id — so two running at once can answer each other's reads from one +/// cached entry, and a withdrawal a test ends on reads as withdrawn in the next one's database. const WS: &str = "test-workspace"; fn operator_client() -> reqwest::Client { @@ -53,6 +53,7 @@ async fn set_settings(api: &str, body: serde_json::Value) -> anyhow::Result #[serial] async fn test_operator_manage_rights(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; + invalidate_operator_rights_cache(WS); let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); let api = format!("http://localhost:{port}/api/w/{WS}"); @@ -145,6 +146,7 @@ async fn test_manage_triggers_covers_a_route_outside_the_shared_handler( db: Pool, ) -> anyhow::Result<()> { initialize_tracing().await; + invalidate_operator_rights_cache(WS); let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); let api = format!("http://localhost:{port}/api/w/{WS}"); @@ -213,3 +215,193 @@ async fn test_manage_triggers_covers_a_route_outside_the_shared_handler( Ok(()) } + +/// A draft is what the AI chat writes before it deploys, so an operator holding a manage right +/// may draft that kind, and only while they hold it. Builder rights are left off throughout: they +/// open flow drafts, and would otherwise be what admits these. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_manage_rights_admit_matching_drafts(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + invalidate_operator_rights_cache(WS); + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let api = format!("http://localhost:{port}/api/w/{WS}"); + let c = operator_client(); + + let save = async |kind: &str, path: &str| -> anyhow::Result<(u16, String)> { + let resp = c + .post(format!("{api}/drafts/update/{kind}/{path}")) + .json(&json!({ "value": { "path": path } })) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) + }; + let listed = async || -> anyhow::Result { + let resp = c.get(format!("{api}/drafts/list")).send().await?; + assert_eq!(resp.status(), 200); + Ok(resp.json().await?) + }; + let schedule_list_has_draft = async || -> anyhow::Result { + let resp = c + .get(format!("{api}/schedules/list?include_draft_only=true")) + .send() + .await?; + assert_eq!(resp.status(), 200); + let rows: Vec = resp.json().await?; + Ok(rows.iter().any(|r| r["path"] == "u/operator/sched_draft")) + }; + + let resp = c + .post(format!("{api}/drafts/update/trigger_schedule/u/operator/sched_draft")) + .json(&json!({ "value": { + "path": "u/operator/sched_draft", + "schedule": "0 0 12 * * *", + "timezone": "UTC", + "script_path": "f/system/hello", + "is_flow": false, + "args": {}, + "enabled": false + } })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert!(schedule_list_has_draft().await?); + let (status, body) = save("trigger_http", "u/operator/http_draft").await?; + assert_eq!(status, 200, "{body}"); + // The rights name schedules and triggers, nothing else. + let (status, body) = save("script", "u/operator/script_draft").await?; + assert_eq!(status, 403, "{body}"); + + let drafts = listed().await?; + assert_eq!(drafts.as_array().map(Vec::len), Some(2), "{drafts}"); + + assert_eq!( + set_settings(&api, json!({"manage_schedules": false})).await?, + 200 + ); + let (status, body) = save("trigger_schedule", "u/operator/sched_draft").await?; + assert_eq!(status, 403, "{body}"); + // The list follows the right too, as the app list follows `builder_apps`. + assert!(!schedule_list_has_draft().await?); + let (status, body) = save("trigger_http", "u/operator/http_draft").await?; + assert_eq!(status, 200, "{body}"); + + assert_eq!( + set_settings(&api, json!({"manage_triggers": false})).await?, + 200 + ); + let (status, body) = save("trigger_http", "u/operator/http_draft").await?; + assert_eq!(status, 403, "{body}"); + // Still listed, as their own, but no longer writable. + let drafts = listed().await?; + assert_eq!(drafts.as_array().map(Vec::len), Some(2), "{drafts}"); + assert!( + drafts + .as_array() + .unwrap() + .iter() + .all(|d| d["can_write"] == json!(false)), + "{drafts}" + ); + + Ok(()) +} + +/// Operators create resources and variables on their pages with no operator setting in the way, +/// so they may draft them in any workspace — even one that withdrew every manage right — and +/// deploying a secret variable draft goes through the create route like a direct write. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operators_draft_resources_and_variables(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + invalidate_operator_rights_cache(WS); + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let api = format!("http://localhost:{port}/api/w/{WS}"); + let c = operator_client(); + + assert_eq!( + set_settings( + &api, + json!({"manage_schedules": false, "manage_triggers": false}) + ) + .await?, + 200 + ); + + let save = async |kind: &str, path: &str, value: serde_json::Value| { + let resp = c + .post(format!("{api}/drafts/update/{kind}/{path}")) + .json(&json!({ "value": value })) + .send() + .await?; + anyhow::Ok((resp.status().as_u16(), resp.text().await?)) + }; + + let (status, body) = save( + "resource", + "u/operator/res_draft", + json!({ "path": "u/operator/res_draft", "args": {}, "resource_type": "object" }), + ) + .await?; + assert_eq!(status, 200, "{body}"); + let (status, body) = save( + "variable", + "u/operator/secret_draft", + json!({ "path": "u/operator/secret_draft", "variable": { "value": "hunter2", "is_secret": true } }), + ) + .await?; + assert_eq!(status, 200, "{body}"); + let (status, body) = save("script", "u/operator/script_draft", json!({})).await?; + assert_eq!(status, 403, "{body}"); + + // The Resources and Variables pages list a draft that has never been deployed. + for (route, path) in [ + ("resources", "u/operator/res_draft"), + ("variables", "u/operator/secret_draft"), + ] { + let listed: serde_json::Value = c + .get(format!("{api}/{route}/list?include_draft_only=true")) + .send() + .await? + .json() + .await?; + assert!( + listed + .as_array() + .unwrap() + .iter() + .any(|r| r["path"] == path && r["draft_only"] == json!(true)), + "{route}: {listed}" + ); + } + + // Deploying a draft sends its stored value, the `$encrypted:` marker, to the create route. + let stored: String = sqlx::query_scalar( + "SELECT value::jsonb->'variable'->>'value' FROM draft + WHERE workspace_id = $1 AND path = 'u/operator/secret_draft'", + ) + .bind(WS) + .fetch_one(&db) + .await?; + assert!(stored.starts_with("$encrypted:"), "{stored}"); + let resp = c + .post(format!("{api}/variables/create")) + .json(&json!({ + "path": "u/operator/secret_draft", + "value": stored, + "is_secret": true, + "description": "", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = c + .get(format!("{api}/variables/get_value/u/operator/secret_draft")) + .send() + .await?; + assert_eq!(resp.json::().await?, "hunter2"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/preview_on_behalf_of.rs b/backend/windmill-api-integration-tests/tests/preview_on_behalf_of.rs new file mode 100644 index 0000000000..a1bb7876c6 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/preview_on_behalf_of.rs @@ -0,0 +1,135 @@ +//! A preview's `on_behalf_of` runs request-supplied code as another account, so it must +//! change the job's identity for admins and be refused for everyone else. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::Authed; +use windmill_test_utils::*; + +/// `route` is under `/api/w/test-workspace/`. +async fn post(port: u16, token: &str, route: &str, body: serde_json::Value) -> (u16, String) { + let resp = reqwest::Client::new() + .post(format!("http://localhost:{port}/api/w/test-workspace/{route}")) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await + .unwrap(); + (resp.status().as_u16(), resp.text().await.unwrap()) +} + +async fn identity(db: &Pool, id: &str) -> (String, String) { + sqlx::query_as("SELECT permissioned_as, permissioned_as_email FROM v2_job WHERE id = $1") + .bind(Uuid::parse_str(id).unwrap()) + .fetch_one(db) + .await + .unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_preview_on_behalf_of(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let script = |obo: &str| { + json!({ + "content": "export function main() { return 1; }", + "language": "deno", + "args": {}, + "on_behalf_of": obo, + }) + }; + let flow = |obo: &str| json!({ "value": {"modules": []}, "args": {}, "on_behalf_of": obo }); + // The app editor's preview of an inline runnable. + let app = |obo: &str| { + json!({ + "args": {}, + "component": "c", + "raw_code": { "language": "deno", "content": "export function main() { return 1; }" }, + "force_viewer_static_fields": {}, + "on_behalf_of": obo, + }) + }; + let app_route = "apps_u/execute_component/u/test-user/app"; + + for (route, body) in [ + ("jobs/run/preview", script("test2@windmill.dev")), + ("jobs/run/preview_flow", flow("u/test-user-2")), + (app_route, app("u/test-user-2")), + ] { + let (status, id) = post(port, "SECRET_TOKEN", route, body).await; + assert!((200..300).contains(&status), "{route}: {status} {id}"); + assert_eq!( + identity(&db, &id).await, + ( + "u/test-user-2".to_string(), + "test2@windmill.dev".to_string() + ), + "{route}" + ); + let logs: String = sqlx::query_scalar("SELECT logs FROM job_logs WHERE job_id = $1") + .bind(Uuid::parse_str(&id).unwrap()) + .fetch_one(&db) + .await?; + assert!( + logs.starts_with("Preview run on behalf of u/test-user-2 (test2@windmill.dev), requested by test@windmill.dev"), + "{route}: {logs}" + ); + } + + for (route, body) in [ + ("jobs/run/preview", script("test@windmill.dev")), + ("jobs/run/preview_flow", flow("u/test-user")), + (app_route, app("u/test-user")), + ] { + let (status, body) = post(port, "SECRET_TOKEN_2", route, body).await; + assert_eq!(status, 403, "{route}: {body}"); + } + + let (status, id) = post(port, "SECRET_TOKEN_2", "jobs/run/preview", script("u/test-user-2")).await; + assert_eq!(status, 201, "{id}"); + assert_eq!(identity(&db, &id).await.0, "u/test-user-2"); + + // Reading data as the end user is the direct caller's own choice. A job token's end user is + // whoever ran that job, so a job's code must not be able to ask for it. + set_jwt_secret().await; + let job_token = create_jwt_token( + Authed { + email: "test@windmill.dev".to_string(), + username: "test-user".to_string(), + is_admin: true, + is_operator: false, + groups: vec![], + folders: vec![], + scopes: None, + token_prefix: None, + }, + "test-workspace", + 3600, + Some(Uuid::new_v4()), + Some("ephemeral-script".to_string()), + None, + None, + ) + .await?; + let viewer_data = json!({ + "content": "export function main() { return 1; }", + "language": "deno", + "args": {}, + "viewer_data_access": true, + }); + for (token, expected) in [("SECRET_TOKEN", true), (job_token.as_str(), false)] { + let (status, id) = post(port, token, "jobs/run/preview", viewer_data.clone()).await; + assert_eq!(status, 201, "{id}"); + let stamped: bool = + sqlx::query_scalar("SELECT viewer_data_access FROM job_perms WHERE job_id = $1") + .bind(Uuid::parse_str(&id).unwrap()) + .fetch_one(&db) + .await?; + assert_eq!(stamped, expected, "job token: {}", !expected); + } + + Ok(()) +} diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index b8ee9ded91..fb935e0b5c 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -916,7 +916,6 @@ async fn list_schedule( // Append the authed user's draft-only schedules; see scripts.rs. if lsq.include_draft_only.unwrap_or(false) - && !authed.is_operator && offset == 0 && lsq.path.is_none() && lsq.is_flow.is_none() @@ -927,6 +926,13 @@ async fn list_schedule( && lsq.summary.is_none() && lsq.broad_filter.is_none() && lsq.label.is_none() + && (!authed.is_operator + || windmill_common::workspaces::operator_can_draft( + &db, + &w_id, + UserDraftItemKind::TriggerSchedule, + ) + .await?) { let draft_only_rows = fetch_draft_only_list_rows( &db, diff --git a/backend/windmill-api-workspaces/src/datatable_clone.rs b/backend/windmill-api-workspaces/src/datatable_clone.rs index 6083b7b619..1e6c6b55c2 100644 --- a/backend/windmill-api-workspaces/src/datatable_clone.rs +++ b/backend/windmill-api-workspaces/src/datatable_clone.rs @@ -28,16 +28,22 @@ use windmill_common::workspaces::{ use windmill_common::{PgDatabase, DB}; use crate::datatable_acl::connect_with_notices; +use crate::datatable_migrations::{ + lock_migration_runs_on, read_applied_migrations, write_applied_migrations, AppliedMigrations, +}; use crate::datatable_permissions::ensure_reaches_governing_datatable; use crate::workspaces::{ - create_database_on_server, ensure_datatable_is_clonable, pg_dump_database, pg_import_dump, - DumpFile, PgDumpOptions, + copy_database_settings, create_database_on_server, ensure_datatable_is_clonable, + pg_dump_database, pg_import_dump, DumpFile, PgDumpOptions, }; -/// A database this request created and filled for one data table of the fork. +/// A database this request created and filled for the data tables of the fork on one source +/// database. pub(crate) struct MadeCopy { - /// The data table's name, in the parent and in the fork. - pub(crate) name: String, + /// The data tables' names, in the parent and in the fork, the one it was made for first. Data + /// tables on one database share its copy: a resource-backed fork has one resource for them all + /// to be pointed through. + pub(crate) names: Vec, pub(crate) dbname: String, pub(crate) behavior: DataTableForkBehavior, /// The database the source resolved to when it was copied. @@ -237,6 +243,7 @@ pub(crate) async fn authorize_copies( parent_w_id: &str, requests: &[CopyRequest<'_>], ) -> Result<()> { + let mut sources: Vec<(&CopyRequest<'_>, GoverningDatatable)> = Vec::new(); for request in requests { match request.behavior { DataTableForkBehavior::KeepOriginal => { @@ -265,9 +272,50 @@ pub(crate) async fn authorize_copies( let governing = ensure_datatable_is_clonable(db, parent_w_id, request.name).await?; ensure_reaches_governing_datatable(db, parent_w_id, request.name, &governing, authed) .await?; - if governing.datatable.permissions.is_some() { + let under_roles = governing.datatable.permissions.is_some(); + if under_roles { crate::datatable_replay_oss::ensure_replay()?; } + if let Some((first, first_governing)) = sources + .iter() + .find(|(_, earlier)| same_source(earlier, &governing)) + { + ensure_shares_copy(first.name, first.behavior, request)?; + if first_governing.datatable.permissions.is_some() != under_roles { + return Err(Error::BadRequest(format!( + "Data tables '{}' and '{}' use the same database, which the fork copies once, \ + but only one of them is under roles", + first.name, request.name + ))); + } + } + sources.push((request, governing)); + } + Ok(()) +} + +/// Whether two data tables are on one database. A resource is the governing workspace's. +fn same_source(a: &GoverningDatatable, b: &GoverningDatatable) -> bool { + same_database(&a.datatable.database, &b.datatable.database) + && (a.workspace_id == b.workspace_id + || a.datatable + .database + .as_ref() + .is_some_and(|d| d.resource_type.is_windmill_managed())) +} + +/// Data tables on one database get one copy, made as the first of them asks. +fn ensure_shares_copy( + first: &str, + behavior: DataTableForkBehavior, + request: &CopyRequest<'_>, +) -> Result<()> { + if behavior != request.behavior { + return Err(Error::BadRequest(format!( + "Data tables '{}' and '{}' use the same database, which the fork copies once: fork \ + them with the same behavior", + first, request.name + ))); } Ok(()) } @@ -281,10 +329,28 @@ pub(crate) async fn make_copies( requests: &[CopyRequest<'_>], ) -> Result> { authorize_copies(db, authed, parent_w_id, requests).await?; - let mut copies = Vec::with_capacity(requests.len()); + let mut copies: Vec = Vec::with_capacity(requests.len()); + // What each copy was made from, by index. + let mut sources: Vec = Vec::with_capacity(requests.len()); for request in requests { + // Checked again by the fork against the copy's `source_database`, under the locks. + let governing = match ensure_datatable_is_clonable(db, parent_w_id, request.name).await { + Ok(governing) => governing, + Err(e) => return Err(drop_copies_after(db, copies, e).await), + }; + let shared = sources.iter().position(|s| same_source(s, &governing)); + if let Some(c) = shared { + if let Err(e) = ensure_shares_copy(&copies[c].names[0], copies[c].behavior, request) { + return Err(drop_copies_after(db, copies, e).await); + } + copies[c].names.push(request.name.to_string()); + continue; + } match make_copy(db, authed, parent_w_id, request).await { - Ok(copy) => copies.push(copy), + Ok(copy) => { + copies.push(copy); + sources.push(governing); + } Err(e) => return Err(drop_copies_after(db, copies, e).await), } } @@ -336,8 +402,8 @@ async fn drop_copy(db: &DB, source_database: &DataTableDatabase, dbname: &str) - Ok(()) } DataTableCatalogResourceType::Postgresql => { - // On a server of the workspace's own, where a resource edited meanwhile can already name it - // and nothing locks such an edit: dropping it could take someone's data. + // On a server of the workspace's own, where a resource edited meanwhile can already name it + // and nothing locks such an edit: dropping it could take someone's data. Err(Error::BadRequest( "kept on its PostgreSQL server, where it may already be in use; drop it there once \ it is not, before forking the same data table under this id again" @@ -392,20 +458,25 @@ async fn make_copy( (server, Some(snapshot)) }; + let schema_only = request.behavior == DataTableForkBehavior::SchemaOnly; + let migrations_lock = if schema_only { + Some(lock_migration_runs_on(db, &server, request.name).await?) + } else { + None + }; // Dumped before the database exists, so a source that cannot be read leaves nothing behind. // Ownership never carries over: the restore runs as the target's connection user. Grants do, // except on a cluster Windmill manages, which plants its own and where the replay below is what // reproduces the roles'. let dump = pg_dump_database( &server, - PgDumpOptions { - schema_only: request.behavior == DataTableForkBehavior::SchemaOnly, - no_owner: true, - no_acl: managed, - ..Default::default() - }, + PgDumpOptions { schema_only, no_owner: true, no_acl: managed, ..Default::default() }, ) .await?; + let applied = match migrations_lock { + Some(client) => Some(read_applied_migrations(&client).await?), + None => None, + }; match source_database.resource_type { DataTableCatalogResourceType::Instance => { @@ -444,11 +515,12 @@ async fn make_copy( &server, &target, &dump, + applied.as_ref(), ) .await; match filled { Ok(replayed) => Ok(MadeCopy { - name: request.name.to_string(), + names: vec![request.name.to_string()], dbname: request.dbname.to_string(), behavior: request.behavior, source_database, @@ -484,8 +556,12 @@ async fn fill( source: &PgDatabase, target: &PgDatabase, dump: &DumpFile, + applied: Option<&AppliedMigrations>, ) -> Result { pg_import_dump(target, dump).await?; + if let Some(applied) = applied { + write_applied_migrations(db, target, applied).await?; + } // Held until the replay commits: a role renamed or dropped meanwhile would change what the // replay names, and a settings save could move the source onto another database or put it under @@ -554,6 +630,7 @@ async fn fill( })?; } tx.commit().await?; + copy_database_settings(db, source, target).await?; Ok(replayed) } diff --git a/backend/windmill-api-workspaces/src/datatable_migration_scope.rs b/backend/windmill-api-workspaces/src/datatable_migration_scope.rs new file mode 100644 index 0000000000..ea98bc7309 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_migration_scope.rs @@ -0,0 +1,319 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! A data table migration's scope: the schemas it declares it touches, written as a +//! `-- schemas sales, billing` annotation in its up SQL. A migration that declares none applies to +//! the whole database. +//! +//! Scope decides ordering, not access: a migration waits on every earlier migration whose scope +//! overlaps its own, and its rollback on every later overlapping one. A whole-database migration +//! overlaps everything, so it is a barrier; migrations on disjoint schemas do not wait on each +//! other. Nothing checks that the SQL stays inside its declared scope. + +use std::collections::HashSet; + +use windmill_common::error::{Error, Result}; + +/// The schemas a migration declares, lowercased, or `None` for the whole database. +pub(crate) type Scope = Option>; + +const KEYWORD: &str = "schemas"; + +/// Read the `-- schemas a, b` annotation from the leading comment block of `code`, the way +/// `SqlAnnotations::datatable_role` reads `-- role`. A leading comment whose first word is +/// `schemas` is an annotation attempt, and a malformed one is an error rather than a silent +/// whole-database scope. +/// +/// Names are lowercased: SQL folds unquoted identifiers, and two migrations spelling one schema +/// `Sales` and `sales` must still be ordered against each other. A false overlap only costs +/// waiting; a false disjointness lets a migration run before one it depends on. +pub(crate) fn parse_migration_schemas(code: &str) -> Result { + for line in code.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + if !line.starts_with("--") { + break; + } + let body = line[2..].trim_start(); + let Some(after) = body + .get(..KEYWORD.len()) + .filter(|kw| kw.eq_ignore_ascii_case(KEYWORD)) + .map(|_| &body[KEYWORD.len()..]) + else { + continue; + }; + if !after.is_empty() + && !after.starts_with(char::is_whitespace) + && !after.starts_with([':', '=']) + { + continue; + } + let after = after.trim_start(); + let after = after.strip_prefix([':', '=']).unwrap_or(after).trim(); + let after = after.strip_suffix(';').unwrap_or(after); + let mut schemas: Vec = vec![]; + for piece in after.split(',') { + let name = piece.trim(); + if !is_annotatable_schema_name(name) { + return Err(Error::BadRequest(format!( + "Malformed data table migration scope annotation: `{line}`. Write it as \ + `-- schemas , ` on a line of its own, where each name is \ + letters, digits, '_', '$' or '-', or leave it out for a migration that \ + applies to the whole database. A comment in the leading block that starts \ + with the word 'schemas' is read as this annotation; move it below the first \ + statement if it is prose." + ))); + } + let name = name.to_ascii_lowercase(); + if !schemas.contains(&name) { + schemas.push(name); + } + } + return Ok(Some(schemas)); + } + Ok(None) +} + +/// Whether `name` can be written in a `-- schemas` annotation. +fn is_annotatable_schema_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 63 + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '$' | '-')) +} + +pub(crate) fn scopes_overlap(a: &Scope, b: &Scope) -> bool { + match (a, b) { + (Some(a), Some(b)) => a.iter().any(|s| b.contains(s)), + _ => true, + } +} + +/// One migration of a data table's chain, as ordering sees it. +pub(crate) struct ChainEntry { + pub(crate) timestamp: i64, + pub(crate) name: String, + /// A malformed annotation counts as the whole database, so it orders conservatively. It is + /// refused only when a migration is written: one stored before scopes existed may open with + /// prose that starts with the word, and must keep running as it always did. + pub(crate) scope: Scope, +} + +impl ChainEntry { + pub(crate) fn new(timestamp: i64, name: String, code_up: &str) -> Self { + let scope = parse_migration_schemas(code_up).unwrap_or(None); + Self { timestamp, name, scope } + } + + fn label(&self) -> String { + format!("{} ({})", self.timestamp, self.name) + } +} + +/// Why `later` has to wait for `earlier`, for an error message. +fn overlap_reason(later: &ChainEntry, earlier: &ChainEntry) -> String { + match (&later.scope, &earlier.scope) { + (_, None) => format!("{} applies to the whole database", earlier.label()), + (None, _) => format!("{} applies to the whole database", later.label()), + (Some(a), Some(b)) => { + let shared: Vec<&str> = a + .iter() + .filter(|s| b.contains(s)) + .map(String::as_str) + .collect(); + format!("both touch schema {}", shared.join(", ")) + } + } +} + +/// Which pending migrations one run applies. +pub(crate) enum RunSelection { + /// Every pending migration, or those up to and including a version. + UpTo(Option), + /// One version. + Only(i64), + /// The pending migrations that declare this schema. A whole-database one is never part of + /// it: running for one schema must not change the others. + Schema(String), +} + +/// The chain indices one run applies, in order, or why it cannot start. Every selected migration +/// needs each earlier overlapping migration applied or applied by this same run, so the whole run +/// is refused before anything is applied rather than stopping half-way. +/// +/// `Only` is exempt: it is the explicit out-of-order run, which the editors confirm with the user +/// (through `blocked_by`) before asking for it. +pub(crate) fn plan_run( + chain: &[ChainEntry], + applied: &HashSet, + selection: &RunSelection, +) -> Result> { + let pending = |i: usize| !applied.contains(&chain[i].timestamp); + let selected: Vec = (0..chain.len()) + .filter(|&i| { + pending(i) + && match selection { + RunSelection::UpTo(None) => true, + RunSelection::UpTo(Some(up_to)) => chain[i].timestamp <= *up_to, + RunSelection::Only(version) => chain[i].timestamp == *version, + RunSelection::Schema(schema) => { + chain[i].scope.as_ref().is_some_and(|s| s.contains(schema)) + } + } + }) + .collect(); + if matches!(selection, RunSelection::Only(_)) { + return Ok(selected); + } + let in_run: HashSet = selected.iter().copied().collect(); + for &i in &selected { + let blocker = (0..i).find(|&j| { + pending(j) && !in_run.contains(&j) && scopes_overlap(&chain[i].scope, &chain[j].scope) + }); + if let Some(j) = blocker { + return Err(Error::BadRequest(format!( + "Migration {} cannot run before {}, which is not applied yet: {}. Apply it first, \ + or run all pending migrations.", + chain[i].label(), + chain[j].label(), + overlap_reason(&chain[i], &chain[j]), + ))); + } + } + Ok(selected) +} + +/// The earliest pending migration `chain[i]` has to wait for, when it is pending; the latest +/// applied migration that has to be rolled back before it, when it is applied. +pub(crate) fn blocked_by(chain: &[ChainEntry], applied: &HashSet, i: usize) -> Option { + let is_applied = |j: usize| applied.contains(&chain[j].timestamp); + let overlaps = |j: usize| scopes_overlap(&chain[i].scope, &chain[j].scope); + if is_applied(i) { + (i + 1..chain.len()) + .rev() + .find(|&j| is_applied(j) && overlaps(j)) + } else { + (0..i).find(|&j| !is_applied(j) && overlaps(j)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(names: &[&str]) -> Scope { + Some(names.iter().map(|n| n.to_string()).collect()) + } + + #[test] + fn schemas_annotation_is_read_from_the_leading_comment_block() { + let p = |code| parse_migration_schemas(code).unwrap(); + assert_eq!(p("-- schemas sales\nBEGIN;"), s(&["sales"])); + assert_eq!( + p("-- role analyst\n-- Schemas: Sales, billing,sales;\nBEGIN;"), + s(&["sales", "billing"]) + ); + assert_eq!(p("-- schemas=a\nBEGIN;"), s(&["a"])); + assert_eq!(p("BEGIN;\n-- schemas sales"), None); + assert_eq!(p("-- schemasx sales\nBEGIN;"), None); + assert_eq!(p("-- up migration: x\nBEGIN;"), None); + for malformed in [ + "-- schemas\nBEGIN;", + "-- schemas:\nBEGIN;", + "-- schemas sales billing\nBEGIN;", + "-- schemas: changes for the orders table\nBEGIN;", + "-- schemas a,,b\nBEGIN;", + "-- schemas a.b\nBEGIN;", + ] { + assert!(parse_migration_schemas(malformed).is_err(), "{malformed}"); + } + } + + fn chain(entries: &[(i64, Scope)]) -> Vec { + entries + .iter() + .map(|(t, scope)| ChainEntry { + timestamp: *t, + name: format!("m{t}"), + scope: scope.clone(), + }) + .collect() + } + + fn versions(c: &[ChainEntry], idx: Vec) -> Vec { + idx.into_iter().map(|i| c[i].timestamp).collect() + } + + #[test] + fn a_run_waits_only_on_overlapping_earlier_migrations() { + let c = chain(&[ + (1, None), + (2, s(&["sales"])), + (3, s(&["billing"])), + (4, s(&["sales", "billing"])), + (5, None), + (6, s(&["sales"])), + ]); + let applied: HashSet = [1].into(); + + // Disjoint schemas don't wait on each other; a cross-schema migration waits on both, a + // whole-database one on everything. + assert_eq!(blocked_by(&c, &applied, 2).map(|j| c[j].timestamp), None); + assert_eq!(blocked_by(&c, &applied, 3).map(|j| c[j].timestamp), Some(2)); + assert_eq!(blocked_by(&c, &applied, 4).map(|j| c[j].timestamp), Some(2)); + // An explicit single run is the confirmed override. + assert_eq!( + versions(&c, plan_run(&c, &applied, &RunSelection::Only(4)).unwrap()), + vec![4] + ); + + // Running for a schema takes what declares it, refuses past a pending whole-database + // migration, and refuses when a selected migration also needs another schema's. + let for_schema = + |a: &HashSet, x: &str| plan_run(&c, a, &RunSelection::Schema(x.into())); + let err = for_schema(&applied, "sales").unwrap_err().to_string(); + assert!(err.contains("4 (m4)") && err.contains("3 (m3)"), "{err}"); + let applied: HashSet = [1, 3].into(); + let err = for_schema(&applied, "sales").unwrap_err().to_string(); + assert!( + err.contains("5 (m5)") && err.contains("whole database"), + "{err}" + ); + let applied: HashSet = [1, 3, 5].into(); + assert_eq!( + versions(&c, for_schema(&applied, "sales").unwrap()), + vec![2, 4, 6] + ); + // Everything pending in order needs no check: each earlier pending one is in the run. + assert_eq!( + versions( + &c, + plan_run(&c, &[1].into(), &RunSelection::UpTo(None)).unwrap() + ), + vec![2, 3, 4, 5, 6] + ); + } + + #[test] + fn a_rollback_waits_on_later_overlapping_migrations() { + let c = chain(&[ + (1, None), + (2, s(&["sales"])), + (3, s(&["billing"])), + (4, s(&["sales"])), + ]); + let applied: HashSet = [1, 2, 3, 4].into(); + let blocker = |i| blocked_by(&c, &applied, i).map(|j| c[j].timestamp); + assert_eq!(blocker(2), None); + assert_eq!(blocker(1), Some(4)); + assert_eq!(blocker(0), Some(4)); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index 21e460359e..839b449f37 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -10,6 +10,9 @@ //! management, and the workspace-merge diff helper. Split out of `workspaces.rs` //! to keep that file focused on core workspace configuration. +use crate::datatable_migration_scope::{ + blocked_by, parse_migration_schemas, plan_run, ChainEntry, RunSelection, +}; use crate::workspaces::{ managed_datatable_kind, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison, PgDumpOptions, @@ -165,6 +168,19 @@ fn ensure_not_operator(authed: &ApiAuthed) -> Result<()> { Ok(()) } +/// Whether new migrations declare a scope, which the editors push for but do not require. +fn log_migration_scope(scope: &crate::datatable_migration_scope::Scope) { + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_scope", + if scope.is_some() { + "scoped" + } else { + "whole_database" + }, + ); +} + #[derive(Serialize)] struct AppliedMigration { version: i64, @@ -182,8 +198,11 @@ struct RunDatatableMigrationsQuery { up_to: Option, /// When set, apply only this specific migration version (if not already /// applied), ignoring any other pending migrations. Takes precedence over - /// `up_to`. + /// `up_to` and `schema`. only: Option, + /// When set, apply only the pending migrations that declare this schema in their + /// `-- schemas` annotation. Takes precedence over `up_to`. + schema: Option, } /// Build the `database` argument for a migration job. Both resource-backed and @@ -509,6 +528,127 @@ async fn hold_migration_lock( } } +/// The `_wm_migrations` rows of a database a schema-only copy is being made of, read with +/// [`lock_migration_runs_on`] held so they describe the schema the dump captured. +pub(crate) struct AppliedMigrations { + /// `None` when the source has no `_wm_migrations`. + schema: Option, + datatables: Vec, + versions: Vec, + installed_at: Vec, +} + +/// Hold the migration-run lock on database `pg`, for a copy whose schema has to agree with the +/// `_wm_migrations` rows it carries. The lock is per database, whichever data table names it. +pub(crate) async fn lock_migration_runs_on( + db: &DB, + pg: &PgDatabase, + label: &str, +) -> Result { + let (client, connection) = pg.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + hold_migration_lock(&client, label, MIGRATION_LOCK_WAIT_SECS).await?; + Ok(client) +} + +/// Read every `_wm_migrations` row of the database `client` is connected to, along with the +/// schema the table is in: the copy is written to by a connection whose `search_path` need not +/// resolve the way this one does. +pub(crate) async fn read_applied_migrations( + client: &tokio_postgres::Client, +) -> Result { + let read_err = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + pg_error_message(&e) + )) + }; + let schema: Option = client + .query_one( + "SELECT n.nspname::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + RIGHT JOIN (SELECT to_regclass('_wm_migrations')::oid AS oid) t ON c.oid = t.oid", + &[], + ) + .await + .map_err(read_err)? + .get(0); + let mut applied = + AppliedMigrations { schema, datatables: vec![], versions: vec![], installed_at: vec![] }; + let Some(schema) = applied.schema.as_deref() else { + return Ok(applied); + }; + let rows = client + .query( + &format!( + // ISO 8601 in UTC: `::text` follows the session's `DateStyle`, which the copy's + // session need not parse the same way. + "SELECT datatable, version, \ + to_char(installed_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US') \ + FROM {}._wm_migrations", + render_db_quoted_identifier(schema, DbType::Postgresql) + ), + &[], + ) + .await + .map_err(read_err)?; + for row in rows { + applied.datatables.push(row.get(0)); + applied.versions.push(row.get(1)); + applied.installed_at.push(row.get(2)); + } + Ok(applied) +} + +/// Write the rows [`read_applied_migrations`] read into a copy restored from a schema-only dump, +/// which brings `_wm_migrations` over empty: without them the copy reports every migration its +/// schema already holds as pending, and running them fails on objects that exist. +pub(crate) async fn write_applied_migrations( + db: &DB, + target: &PgDatabase, + applied: &AppliedMigrations, +) -> Result<()> { + let Some(schema) = applied.schema.as_deref() else { + return Ok(()); + }; + if applied.versions.is_empty() { + return Ok(()); + } + let (client, connection) = target.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + client + .execute( + &format!( + "INSERT INTO {}._wm_migrations (datatable, version, installed_at) \ + SELECT d, v, i::timestamp AT TIME ZONE 'UTC' \ + FROM unnest($1::text[], $2::bigint[], $3::text[]) AS t(d, v, i) \ + ON CONFLICT DO NOTHING", + render_db_quoted_identifier(schema, DbType::Postgresql) + ), + &[ + &applied.datatables, + &applied.versions, + &applied.installed_at, + ], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to copy _wm_migrations into '{}': {}", + target.dbname, + pg_error_message(&e) + )) + })?; + Ok(()) +} + /// Whether `e` is the lock wait running out, as opposed to the database being unreachable. fn is_migration_lock_busy(e: &Error) -> bool { matches!(e, Error::Generic(StatusCode::CONFLICT, _)) @@ -541,7 +681,9 @@ async fn read_applied_versions_on_client( /// Each migration runs as a normal Windmill `postgresql` job (permissioned as /// the requester, labelled `datatable_migration`); applied versions are then /// recorded in the data table's own `_wm_migrations` table, so only migrations -/// not recorded there are run, in ascending `timestamp` order. +/// not recorded there are run, in ascending `timestamp` order. A run for one +/// schema is refused while an earlier migration its selection overlaps is pending +/// (see `datatable_migration_scope`). async fn run_datatable_migrations( authed: ApiAuthed, Extension(db): Extension, @@ -587,30 +729,29 @@ async fn run_datatable_migrations( let applied_versions = read_applied_versions_on_client(&client, &datatable_name).await?; + let selection = if let Some(only) = query.only { + RunSelection::Only(only) + } else if let Some(schema) = query.schema.as_deref() { + RunSelection::Schema(schema.to_ascii_lowercase()) + } else { + RunSelection::UpTo(query.up_to) + }; // How the user scoped the run, for the counter emitted on the first migration // that lands below. - let scope = if query.only.is_some() { - "only" - } else if query.up_to.is_some() { - "up_to" - } else { - "all" + let scope = match selection { + RunSelection::Only(_) => "only", + RunSelection::Schema(_) => "schema", + RunSelection::UpTo(Some(_)) => "up_to", + RunSelection::UpTo(None) => "all", }; + let chain: Vec = migrations + .iter() + .map(|m| ChainEntry::new(m.timestamp, m.name.clone(), &m.code_up)) + .collect(); + let plan = plan_run(&chain, &applied_versions, &selection)?; let mut applied = Vec::new(); - for m in migrations { - if let Some(only) = query.only { - // Run a single specific migration, skipping every other one. - if m.timestamp != only { - continue; - } - } else if query.up_to.is_some_and(|up_to| m.timestamp > up_to) { - // Migrations are ordered ascending, so once we pass `up_to` we're done. - break; - } - if applied_versions.contains(&m.timestamp) { - continue; - } + for m in plan.into_iter().map(|i| &migrations[i]) { ensure_migration_role_allowed( &db, &w_id, @@ -642,7 +783,7 @@ async fn run_datatable_migrations( pg_error_message(&e) )) })?; - applied.push(AppliedMigration { version: m.timestamp, name: m.name }); + applied.push(AppliedMigration { version: m.timestamp, name: m.name.clone() }); // One event per run that moved the data table forward, emitted on the // first migration that lands rather than after the loop: a later one // failing returns early, and that run still advanced the data table. A @@ -865,6 +1006,13 @@ struct DatatableMigrationWithStatus { #[serde(skip_serializing_if = "Option::is_none")] code_down: Option, status: DatatableMigrationRunStatus, + /// The schemas its `-- schemas` annotation declares; absent for the whole database. + #[serde(skip_serializing_if = "Option::is_none")] + schemas: Option>, + /// When not run: the earliest pending migration that has to be applied before it. When run: + /// the latest applied migration that has to be rolled back before it. + #[serde(skip_serializing_if = "Option::is_none")] + blocked_by: Option, } #[derive(Serialize)] @@ -981,9 +1129,14 @@ async fn datatable_migrations_status( Err(e) => (None, Some(e.to_string())), }; + let chain: Vec = defs + .iter() + .map(|m| ChainEntry::new(m.timestamp, m.name.clone(), &m.code_up)) + .collect(); let migrations = defs .into_iter() - .map(|m| { + .enumerate() + .map(|(i, m)| { let status = match &applied { Some(set) if set.contains(&m.timestamp) => DatatableMigrationRunStatus::Ran, Some(_) => DatatableMigrationRunStatus::NotRun, @@ -995,6 +1148,11 @@ async fn datatable_migrations_status( code_up: m.code_up, code_down: m.code_down, status, + schemas: chain[i].scope.clone(), + blocked_by: applied + .as_ref() + .and_then(|set| blocked_by(&chain, set, i)) + .map(|j| chain[j].timestamp), } }) .collect(); @@ -1333,6 +1491,7 @@ async fn create_datatable_migration( ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; + let scope = parse_migration_schemas(&payload.code_up)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; ensure_may_edit_migration( &db, @@ -1379,6 +1538,7 @@ async fn create_datatable_migration( .await?; windmill_common::feature_usage::log_feature_usage("datatable", "migration_created", "manual"); + log_migration_scope(&scope); Ok(Json(DatatableMigration { datatable: datatable_name, @@ -1581,6 +1741,16 @@ async fn upsert_datatable_migration( if unchanged { return Ok(upserted_message); } + // Only up SQL being written is held to the annotation: a stored up that opens with prose + // starting with the word `schemas` predates scopes, and must still accept a down added to it. + let up_changed = existing + .as_ref() + .is_none_or(|existing| existing.code_up != payload.code_up); + let scope = if up_changed { + parse_migration_schemas(&payload.code_up)? + } else { + None + }; ensure_may_edit_migration( &db, &w_id, @@ -1709,6 +1879,9 @@ async fn upsert_datatable_migration( "edited" }, ); + if existing.is_none() { + log_migration_scope(&scope); + } Ok(upserted_message) } diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index c437d7e2b4..8d37fdf194 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -4,6 +4,7 @@ pub mod data_metrics; pub mod datatable_acl; pub mod datatable_acl_oss; pub mod datatable_clone; +pub mod datatable_migration_scope; pub mod datatable_migrations; pub mod datatable_permissions; pub mod datatable_permissions_oss; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index d0f44a0618..2f614100ee 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -1766,6 +1766,43 @@ async fn set_slack_oauth_config( "Both client ID and client secret are required".to_string(), )); } + // A settings.yaml pulled by `wmill sync` holds the secret encrypted, and a CLI that + // predates the settings document pushes it here as it reads it. One encrypted under another + // workspace's key keeps the stored secret, as the settings document does, so that CLI's push + // does not abort over it. + let mut req = req; + if let Some(ciphertext) = req + .slack_oauth_client_secret + .strip_prefix(windmill_common::variables::SETTINGS_ENCRYPTED_PREFIX) + { + match windmill_common::variables::build_crypt(&db, &w_id) + .await? + .decrypt_str(ciphertext) + { + Ok(plain) => req.slack_oauth_client_secret = plain, + Err(_) => { + let stored = sqlx::query_scalar!( + "SELECT slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + tracing::warn!( + workspace_id = %w_id, + "Slack client secret left unchanged: it is encrypted with another workspace's key" + ); + match stored { + Some(stored) => req.slack_oauth_client_secret = stored, + None => { + return Ok("Slack OAuth config left unchanged: the client secret is \ + encrypted with another workspace's key" + .to_string()) + } + } + } + } + } let mut tx = db.begin().await?; @@ -3595,6 +3632,97 @@ pub(crate) async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) Ok(()) } +/// Carry the source database's own settings (`ALTER DATABASE ... SET`) over to a copy. `pg_dump` +/// only emits them with `--create`, so a copy otherwise resolves unqualified names, its +/// `_wm_migrations` included, through the server's default `search_path` rather than the source's. +/// A setting the copy's connection may not set is skipped with a warning rather than failing it. +/// Called once nothing else writes to the copy, which a copied read-only default would refuse. +pub(crate) async fn copy_database_settings( + db: &DB, + source: &PgDatabase, + target: &PgDatabase, +) -> Result<()> { + let (source_client, connection) = source.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + let settings: Vec = source_client + .query( + "SELECT unnest(setconfig) FROM pg_db_role_setting WHERE setrole = 0 \ + AND setdatabase = (SELECT oid FROM pg_database WHERE datname = current_database())", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the settings of database '{}': {}", + source.dbname, + pg_error_message(&e) + )) + })? + .iter() + .map(|row| row.get(0)) + .collect(); + drop(source_client); + if settings.is_empty() { + return Ok(()); + } + + let (mut client, connection) = target.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + let quoted_db = render_db_quoted_identifier(&target.dbname, DbType::Postgresql); + for setting in settings { + let Some((name, value)) = setting.split_once('=') else { + continue; + }; + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.') + { + tracing::warn!( + "Not copying database setting '{name}' to '{}'", + target.dbname + ); + continue; + } + let quoted_name = name + .split('.') + .map(|part| render_db_quoted_identifier(part, DbType::Postgresql)) + .collect::>() + .join("."); + // `FROM CURRENT` takes the value in the stored text form, which `SET name TO value` would + // need re-quoted per setting type. Set transaction-locally: a session-wide `role` or + // `default_transaction_read_only` would make every later `ALTER DATABASE` fail. + let applied = async { + let tx = client.transaction().await?; + tx.execute("SELECT set_config($1, $2, true)", &[&name, &value]) + .await?; + tx.batch_execute(&format!( + "ALTER DATABASE {quoted_db} SET {quoted_name} FROM CURRENT" + )) + .await?; + tx.commit().await + } + .await; + if let Err(e) = applied { + tracing::warn!( + "Could not copy setting '{name}' of database '{}' to '{}': {}", + source.dbname, + target.dbname, + pg_error_message(&e) + ); + } + } + Ok(()) +} + #[derive(Deserialize)] struct CreatePgDatabaseRequest { /// The datatable source to determine connection info: 'datatable://name' or '$res:path' @@ -3858,12 +3986,33 @@ async fn import_pg_database( // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access. let no_acl = target_kind.is_some() || source_kind.is_some(); + // Only a database made to be a copy takes on the source's bookkeeping and settings: without an + // override the target is a data table's own live database, whose settings are its owner's. + let fresh_copy = req.target_dbname_override.is_some(); + let migrations_lock = if schema_only && fresh_copy { + Some( + crate::datatable_migrations::lock_migration_runs_on(&db, &source_pg, &req.source) + .await?, + ) + } else { + None + }; let dump_file = pg_dump_database( &source_pg, PgDumpOptions { schema_only, no_owner: true, no_acl, ..Default::default() }, ) .await?; + let applied = match migrations_lock { + Some(client) => Some(crate::datatable_migrations::read_applied_migrations(&client).await?), + None => None, + }; pg_import_dump(&target_pg, &dump_file).await?; + if let Some(applied) = applied { + crate::datatable_migrations::write_applied_migrations(&db, &target_pg, &applied).await?; + } + if fresh_copy { + copy_database_settings(&db, &source_pg, &target_pg).await?; + } if let Some(tx) = fork_lock { tx.commit().await?; } @@ -8399,7 +8548,7 @@ async fn clone_apps( // Clone app scripts with recomputed hashes let app_scripts = sqlx::query!( - "SELECT app, hash, lock, code, code_sha256 + "SELECT id, app, hash, lock, code, code_sha256, modules FROM app_script WHERE app = ANY(SELECT id FROM app WHERE workspace_id = $1)", source_workspace_id @@ -8407,34 +8556,104 @@ async fn clone_apps( .fetch_all(&mut **tx) .await?; + let mut app_script_id_mapping: HashMap = HashMap::new(); for app_script in app_scripts { if let Some(&new_app_id) = app_id_mapping.get(&app_script.app) { - // Recompute hash using app_id, code_sha256, and lock + // Recompute hash using app_id, code_sha256, lock and modules let mut hasher = Sha256::new(); hasher.update(new_app_id.to_be_bytes()); hasher.update(hex::decode(&app_script.code_sha256)?); if let Some(lock) = &app_script.lock { hasher.update(lock.as_bytes()); } + if let Some(modules) = &app_script.modules { + hasher.update(modules.to_string()); + } let new_hash = hex::encode(hasher.finalize()); - sqlx::query!( - "INSERT INTO app_script (app, hash, lock, code, code_sha256) - VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", + let new_id = sqlx::query_scalar!( + "INSERT INTO app_script (app, hash, lock, code, code_sha256, modules) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id + RETURNING id", new_app_id, new_hash, app_script.lock, app_script.code, app_script.code_sha256, + app_script.modules, ) - .execute(&mut **tx) + .fetch_one(&mut **tx) .await?; + app_script_id_mapping.insert(app_script.id, new_id); } } + // A version's lite value is what points each inline script at its `app_script` row (lock and + // modules). Without it the fork serves the full value, whose inline scripts carry no id and + // so run as bare code: no lock, and no modules for a raw app's sibling imports. Every + // version, not just `versions.last()`: the fork's `versions` is rebuilt from all rows above, + // so its current one can be a row the parent's array never listed. + let source_versions: Vec = version_id_mapping.keys().copied().collect(); + let lite_versions = sqlx::query!( + "SELECT id, value AS \"value!: serde_json::Value\" FROM app_version_lite WHERE id = ANY($1)", + &source_versions, + ) + .fetch_all(&mut **tx) + .await?; + let (mut lite_ids, mut lite_values) = (vec![], vec![]); + for lite in lite_versions { + let Some(&new_version_id) = version_id_mapping.get(&lite.id) else { + continue; + }; + let mut value = lite.value; + // Lite inline scripts have no code of their own, so one left without a row cannot run. + if !remap_app_script_ids(&mut value, &app_script_id_mapping) { + continue; + } + if let Some(object) = value.as_object_mut() { + object.insert("version".to_string(), serde_json::json!(new_version_id)); + } + lite_ids.push(new_version_id); + lite_values.push(value); + } + sqlx::query!( + "INSERT INTO app_version_lite (id, value) + SELECT * FROM UNNEST($1::bigint[], $2::jsonb[]) + ON CONFLICT (id) DO NOTHING", + &lite_ids, + &lite_values, + ) + .execute(&mut **tx) + .await?; + Ok(app_id_mapping) } +/// Points every `inlineScript.id` of a cloned lite app value at the fork's copy of that row. +/// False when some id has no copy. +fn remap_app_script_ids(value: &mut serde_json::Value, mapping: &HashMap) -> bool { + match value { + serde_json::Value::Object(object) => { + if let Some(serde_json::Value::Object(script)) = object.get_mut("inlineScript") { + if let Some(old_id) = script.get("id").and_then(serde_json::Value::as_i64) { + let Some(new_id) = mapping.get(&old_id) else { + return false; + }; + script.insert("id".to_string(), serde_json::json!(new_id)); + } + } + object + .values_mut() + .all(|child| remap_app_script_ids(child, mapping)) + } + serde_json::Value::Array(array) => array + .iter_mut() + .all(|child| remap_app_script_ids(child, mapping)), + _ => true, + } +} + async fn clone_raw_apps( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, @@ -8826,11 +9045,12 @@ async fn apply_forked_datatable( fdt: &ForkedDatatableInfo, copy: &crate::datatable_clone::MadeCopy, ) -> Result<()> { - windmill_common::validate_dbname(&fdt.new_dbname)?; - if !fdt.new_dbname.starts_with("wm_fork_") { + // Not `fdt.new_dbname`: a data table sharing its database with an earlier one shares its copy. + let dbname = copy.dbname.as_str(); + windmill_common::validate_dbname(dbname)?; + if !dbname.starts_with("wm_fork_") { return Err(Error::BadRequest(format!( - "Forked datatable database name '{}' must start with 'wm_fork_'", - fdt.new_dbname + "Forked datatable database name '{dbname}' must start with 'wm_fork_'" ))); } // Held until the fork commits, as the permissions save holds them: the source moved onto @@ -8919,27 +9139,27 @@ async fn apply_forked_datatable( if database.resource_type == DataTableCatalogResourceType::ExternalInstance { windmill_common::external_instance_pg::ensure_external_instance_database_registered( - tx, - &fdt.new_dbname, + tx, dbname, ) .await?; } if database.resource_type.is_windmill_managed() { // Held until the fork commits, as every save newly naming a database takes it: none may // claim the copy between the check below and this fork's entry landing on it. - windmill_common::datatable_roles::lock_instance_databases_governance( - &mut **tx, - [fdt.new_dbname.as_str()], - ) - .await?; + windmill_common::datatable_roles::lock_instance_databases_governance(&mut **tx, [dbname]) + .await?; } + // The data tables after the first on a shared copy find it named by the first, and the + // resource it is reached through already pointed at it. + let first_on_copy = copy.names.first() == Some(&fdt.name); if database.resource_type.is_windmill_managed() + && first_on_copy && !windmill_api_auth::is_super_admin_authed(db, authed).await? { windmill_common::ensure_fork_database_available_to( &mut **tx, database.resource_type, - &fdt.new_dbname, + dbname, parent_w_id, ) .await?; @@ -8950,7 +9170,7 @@ async fn apply_forked_datatable( // on the same cluster as its source, so it keeps the source's kind. let new_database = serde_json::json!({ "resource_type": database.resource_type, - "resource_path": &fdt.new_dbname, + "resource_path": dbname, }); sqlx::query( r#"UPDATE workspace_settings @@ -8964,7 +9184,7 @@ async fn apply_forked_datatable( .bind(new_database) .execute(&mut **tx) .await?; - } else { + } else if first_on_copy { // Resource: point it at the copy and mark it ws_specific. What the settings clone carried is // the source's resource and variables as they are now, which an edit made while the copy // was being made can have changed, and changed back: the clone has to be what the copy's @@ -8987,7 +9207,7 @@ async fn apply_forked_datatable( ) .bind(forked_w_id) .bind(resource_path) - .bind(&fdt.new_dbname) + .bind(dbname) .execute(&mut **tx) .await?; @@ -9917,9 +10137,12 @@ async fn write_workspace_fork( // Update forked datatable settings to point to new databases for fdt in &nw.forked_datatables { - let copy = copies.iter().find(|c| c.name == fdt.name).ok_or_else(|| { - Error::internal_err(format!("No copy was made of data table '{}'", fdt.name)) - })?; + let copy = copies + .iter() + .find(|c| c.names.contains(&fdt.name)) + .ok_or_else(|| { + Error::internal_err(format!("No copy was made of data table '{}'", fdt.name)) + })?; apply_forked_datatable( &db, &mut tx, @@ -9993,7 +10216,7 @@ async fn write_workspace_fork( .map(|c| { format!( "{} ({})", - c.name, + c.names.join(" + "), match c.behavior { DataTableForkBehavior::SchemaOnly => "schema_only", _ => "schema_and_data", @@ -14758,3 +14981,154 @@ async fn set_ws_specific( body.value, body.item_kind, body.path )) } + +/// A setting the workspace settings document writes, each through the endpoint that owns it, +/// so applying a document validates, audits and has the side effects of the matching UI save. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SettingsEdit { + Webhook, + AutoInvite, + InstanceGroups, + ErrorHandler, + SuccessHandler, + LargeFileStorage, + GitSync, + DefaultScripts, + DefaultApp, + Color, + OperatorSettings, + Datatable, + SlackCommand, + SetSlackOAuth, + DeleteSlackOAuth, + Ducklake, + DbtWarehouses, + DeployUi, + GuestAccess, + GuestJwtKey, + PublicAppRateLimit, + AddAdminsAndDevelopersToForks, + /// Body: `{ datatable, permissioned, default_role, roles: [{ id, tenants }] }`, role ids. + DatatablePermissions, + CreateProtectionRule, + /// Body: the update request plus `rule_name`, the rule it updates. + UpdateProtectionRule, + /// Body: `{ name }`. + DeleteProtectionRule, +} + +fn settings_edit_body( + edit: SettingsEdit, + body: serde_json::Value, +) -> Result> { + serde_json::from_value(body) + .map(Json) + .map_err(|e| Error::BadRequest(format!("Invalid value for {edit:?}: {e}"))) +} + +/// Returns the endpoint's JSON response, or `null` for one that answers with prose. +pub async fn apply_settings_edit( + authed: ApiAuthed, + db: DB, + w_id: String, + edit: SettingsEdit, + body: serde_json::Value, +) -> Result { + use SettingsEdit::*; + let a = authed.clone(); + let (d, p) = (Extension(db), Path(w_id)); + match edit { + Webhook => edit_webhook(a, d, p, authed, settings_edit_body(edit, body)?).await?, + AutoInvite => edit_auto_invite(a, d, p, settings_edit_body(edit, body)?).await?, + InstanceGroups => edit_instance_groups(a, d, p, settings_edit_body(edit, body)?).await?, + ErrorHandler => { + edit_error_handler(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + SuccessHandler => edit_success_handler(a, d, p, settings_edit_body(edit, body)?).await?, + LargeFileStorage => { + edit_large_file_storage_config(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + GitSync => edit_git_sync_config(a, d, p, authed, settings_edit_body(edit, body)?).await?, + DefaultScripts => { + edit_default_scripts(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + #[cfg(feature = "enterprise")] + DefaultApp => edit_default_app(a, d, p, authed, settings_edit_body(edit, body)?).await?, + #[cfg(not(feature = "enterprise"))] + DefaultApp => edit_default_app(a, d, p, settings_edit_body(edit, body)?).await?, + Color => change_workspace_color(a, p, d, settings_edit_body(edit, body)?).await?, + OperatorSettings => { + update_operator_settings(a, p, d, settings_edit_body(edit, body)?).await? + } + Datatable => { + let Json(result) = + edit_datatable_config(a, d, p, authed, settings_edit_body(edit, body)?).await?; + return serde_json::to_value(result).map_err(|e| Error::internal_err(e.to_string())); + } + SlackCommand => { + edit_slack_command(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + SetSlackOAuth => set_slack_oauth_config(a, d, p, settings_edit_body(edit, body)?).await?, + DeleteSlackOAuth => delete_slack_oauth_config(a, d, p).await?, + Ducklake => edit_ducklake_config(a, d, p, authed, settings_edit_body(edit, body)?).await?, + DbtWarehouses => { + edit_dbt_warehouses(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + #[cfg(feature = "enterprise")] + DeployUi => edit_deploy_ui_config(a, d, p, authed, settings_edit_body(edit, body)?).await?, + #[cfg(not(feature = "enterprise"))] + DeployUi => edit_deploy_ui_config(a, d, p).await?, + GuestAccess => edit_guest_access(a, d, p, settings_edit_body(edit, body)?).await?, + GuestJwtKey => edit_guest_jwt_key(a, d, p, settings_edit_body(edit, body)?).await?, + PublicAppRateLimit => { + edit_public_app_rate_limit(d, p, a, settings_edit_body(edit, body)?).await? + } + AddAdminsAndDevelopersToForks => { + edit_add_admins_and_developers_to_forks(a, d, p, settings_edit_body(edit, body)?) + .await? + } + #[cfg(all(feature = "private", feature = "enterprise"))] + DatatablePermissions => { + #[derive(Deserialize)] + struct Body { + datatable: String, + #[serde(flatten)] + req: crate::datatable_permissions_ee::SetDatatablePermissions, + } + let Json(Body { datatable, req }) = settings_edit_body(edit, body)?; + crate::datatable_permissions_oss::set_datatable_permissions( + a, + d, + Path((p.0, datatable)), + Json(req), + ) + .await? + } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + DatatablePermissions => { + crate::datatable_permissions_oss::set_datatable_permissions(a).await? + } + CreateProtectionRule => { + create_protection_rule(a, d, p, settings_edit_body(edit, body)?).await? + } + UpdateProtectionRule => { + #[derive(Deserialize)] + struct Body { + rule_name: String, + #[serde(flatten)] + req: UpdateProtectionRuleRequest, + } + let Json(Body { rule_name, req }) = settings_edit_body(edit, body)?; + update_protection_rule(a, d, Path((p.0, rule_name)), Json(req)).await? + } + DeleteProtectionRule => { + #[derive(Deserialize)] + struct Body { + name: String, + } + let Json(Body { name }) = settings_edit_body(edit, body)?; + delete_protection_rule(a, d, Path((p.0, name))).await? + } + }; + Ok(serde_json::Value::Null) +} diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 2f0d4deb13..67d31f8a77 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use windmill_api_auth::{require_super_admin, ApiAuthed}; use windmill_common::DB; @@ -1486,20 +1486,39 @@ pub async fn drop_forked_datatable_databases( serde_json::from_value(datatable_config).unwrap_or_default(); let mut errors: Vec = Vec::new(); + // Data tables cloned from one database share its copy, which is dropped once for all of them. + // Keyed by where the database lives too: copies on other clusters or servers can share a name. + let mut dropped_dbs: HashSet<(String, Option, String)> = HashSet::new(); + // Only a clone is droppable, and a clone is terminal by construction: a kept data table is a + // pointer at the parent's database, which this fork does not own. + let cloned_database = |name: &str| { + datatables + .get(name) + .filter(|dt| dt.forked_from.is_some()) + .and_then(|dt| dt.database.as_ref()) + }; for dt_name in &req.datatable_names { - // Only a clone is droppable, and a clone is terminal by construction: a kept data table is - // a pointer at the parent's database, which this fork does not own. - let database = match datatables.get(dt_name) { - Some(dt) if dt.forked_from.is_some() => match dt.database.as_ref() { - Some(database) => database, - None => continue, - }, - _ => continue, + let Some(database) = cloned_database(dt_name) else { + continue; }; if database.resource_type.is_windmill_managed() { let db_to_drop = &database.resource_path; + let cluster = database.resource_type.as_ref().to_string(); + if !dropped_dbs.insert((cluster, None, db_to_drop.clone())) { + continue; + } + let sharing: Vec = req + .datatable_names + .iter() + .filter(|name| { + cloned_database(name).is_some_and(|d| { + d.resource_type == database.resource_type && &d.resource_path == db_to_drop + }) + }) + .cloned() + .collect(); if !db_to_drop.starts_with("wm_fork_") { errors.push(format!( "Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'", @@ -1513,12 +1532,8 @@ pub async fn drop_forked_datatable_databases( // A task of its own, so a client going away cannot stop it between dropping the // database and committing the entry's removal. let dropped = tokio::spawn({ - let (db, w_id, dt_name, db_to_drop) = ( - db.clone(), - w_id.clone(), - dt_name.clone(), - db_to_drop.clone(), - ); + let (db, w_id, sharing, db_to_drop) = + (db.clone(), w_id.clone(), sharing.clone(), db_to_drop.clone()); let resource_type = database.resource_type; async move { let mut tx = db.begin().await?; @@ -1527,26 +1542,29 @@ pub async fn drop_forked_datatable_databases( // rename this entry, or point another one here, either side of the check below. windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?; // The snapshot above was read unlocked: a save committing since could have - // repointed this entry, and the entry is removed below whatever it names by then. - let current = sqlx::query_scalar::<_, Option>( - "SELECT datatable->'datatables'->$2 FROM workspace_settings - WHERE workspace_id = $1 FOR UPDATE", - ) - .bind(&w_id) - .bind(&dt_name) - .fetch_optional(&mut *tx) - .await? - .flatten() - .and_then(|v| serde_json::from_value::(v).ok()); - if !current.is_some_and(|dt| { - dt.forked_from.is_some() - && dt.database.is_some_and(|d| { - d.resource_type == resource_type && d.resource_path == db_to_drop - }) - }) { - return Err(Error::BadRequest( - "the data table changed while it was being cleaned up".to_string(), - )); + // repointed these entries, and they are removed below whatever they name by then. + for dt_name in &sharing { + let current = sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$2 FROM workspace_settings + WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&w_id) + .bind(dt_name) + .fetch_optional(&mut *tx) + .await? + .flatten() + .and_then(|v| serde_json::from_value::(v).ok()); + if !current.is_some_and(|dt| { + dt.forked_from.is_some() + && dt.database.is_some_and(|d| { + d.resource_type == resource_type + && d.resource_path == db_to_drop + }) + }) { + return Err(Error::BadRequest(format!( + "data table '{dt_name}' changed while it was being cleaned up" + ))); + } } let external = resource_type == windmill_common::workspaces::DataTableCatalogResourceType::ExternalInstance; @@ -1564,11 +1582,13 @@ pub async fn drop_forked_datatable_databases( ) .await?; // Before the entry is removed: child fork pointers are found through it. - let uses = windmill_common::workspaces::managed_database_uses( + let names: Vec<&str> = sharing.iter().map(String::as_str).collect(); + let uses = windmill_common::workspaces::managed_database_uses_except( &mut tx, resource_type, &db_to_drop, - Some((w_id.as_str(), dt_name.as_str())), + &w_id, + &names, ) .await?; if !uses.is_empty() { @@ -1579,20 +1599,22 @@ pub async fn drop_forked_datatable_databases( } // The entry goes with the database: a fork this one is cloned into afterwards // must not inherit a pointer at a data table whose database is gone. - sqlx::query( - "UPDATE workspace_settings SET datatable = datatable #- ARRAY['datatables', $2] - WHERE workspace_id = $1", - ) - .bind(&w_id) - .bind(&dt_name) - .execute(&mut *tx) - .await?; + for dt_name in &sharing { + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- ARRAY['datatables', $2] + WHERE workspace_id = $1", + ) + .bind(&w_id) + .bind(dt_name) + .execute(&mut *tx) + .await?; + } if external { // Checks the uses of the database itself, and unregisters it. windmill_common::external_instance_pg::drop_external_instance_database_unchecked( &mut tx, &db_to_drop, - Some((w_id.as_str(), dt_name.as_str())), + sharing.first().map(|name| (w_id.as_str(), name.as_str())), ) .await?; } else { @@ -1615,10 +1637,38 @@ pub async fn drop_forked_datatable_databases( if let Err(e) = dropped { errors.push(format!( "Could not drop instance database '{}' for datatable://{}: {}", - db_to_drop, dt_name, e + db_to_drop, + sharing.join(", datatable://"), + e )); } } else { + // Clones on one resource share its copy, which stays while any of them is kept. Only + // clones: a kept entry naming the resource cannot be put in the request. + let mut kept: Vec<&String> = datatables + .iter() + .filter(|(name, dt)| { + !req.datatable_names.contains(name) + && dt.forked_from.is_some() + && dt.database.as_ref().is_some_and(|d| { + d.resource_type == database.resource_type + && d.resource_path == database.resource_path + }) + }) + .map(|(name, _)| name) + .collect(); + if !kept.is_empty() { + kept.sort(); + errors.push(format!( + "Could not drop the database of datatable://{dt_name}: it is still used by data \ + tables {}", + kept.iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(", ") + )); + continue; + } let fork_pg = match crate::workspaces::resolve_pg_source_checked( &db, &user_db, @@ -1673,6 +1723,9 @@ pub async fn drop_forked_datatable_databases( )); continue; } + if !dropped_dbs.insert((parent_pg.host.clone(), parent_pg.port, db_to_drop.clone())) { + continue; + } match parent_pg.connect(Some(&db)).await { Ok((client, connection)) => { diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 4016222d30..ad312c43e3 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -34948,7 +34948,7 @@ }, "early_return": { "type": "string", - "description": "JavaScript expression to return early from the flow" + "description": "Id of the step whose result a sync webhook returns as soon as that step completes, while the rest of the flow keeps running. When the step does not run, or fails while the flow has a failure module, the flow's final result is returned instead." }, "chat_input_enabled": { "type": "boolean", @@ -48161,7 +48161,7 @@ }, "early_return": { "type": "string", - "description": "JavaScript expression to return early from the flow" + "description": "Id of the step whose result a sync webhook returns as soon as that step completes, while the rest of the flow keeps running. When the step does not run, or fails while the flow has a failure module, the flow's final result is returned instead." }, "chat_input_enabled": { "type": "boolean", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index d541144231..aa97838a30 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -11884,8 +11884,11 @@ paths: early_return: type: string description: >- - JavaScript expression to return early from the - flow + Id of the step whose result a sync webhook returns + as soon as that step completes, while the rest of + the flow keeps running. When the step does not + run, or fails while the flow has a failure module, + the flow's final result is returned instead. chat_input_enabled: type: boolean description: Whether this flow accepts chat-style input @@ -19210,7 +19213,12 @@ paths: description: Execution priority (higher numbers run first) early_return: type: string - description: JavaScript expression to return early from the flow + description: >- + Id of the step whose result a sync webhook returns as + soon as that step completes, while the rest of the flow + keeps running. When the step does not run, or fails + while the flow has a failure module, the flow's final + result is returned instead. chat_input_enabled: type: boolean description: Whether this flow accepts chat-style input diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 989e5c2a48..786172675d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.833.0 + version: 1.835.0 title: Windmill API contact: @@ -5184,6 +5184,107 @@ paths: - has_instance_ai_config - uses_instance_ai_config + /w/{workspace}/workspaces/settings_document: + get: + summary: get the workspace settings document + description: >- + The workspace's settings in the layout of the settings.yaml that `wmill sync` writes. + Admin only. + operationId: getWorkspaceSettingsDocument + x-mcp-tool: true + x-mcp-tool-include-query-params: [] + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: secrets + description: >- + How secrets are written: `redacted` (the default) replaces each set secret with + `__SENSITIVE_AND_UNCHANGED__`; `encrypted` writes it encrypted with the workspace key, + prefixed with `$encrypted:`. + in: query + schema: + type: string + enum: [redacted, encrypted] + responses: + "200": + description: the settings document + content: + application/json: + schema: + type: object + additionalProperties: true + post: + summary: apply a workspace settings document + description: >- + Applies a settings document, each setting through the endpoint that owns it. Admin only. + operationId: applyWorkspaceSettingsDocument + x-mcp-tool: true + x-mcp-instructions: "Send only the top-level keys to change, each whole as getWorkspaceSettingsDocument returns it: a key the document omits is left as it is, and a key set to null clears it. Keep `__SENSITIVE_AND_UNCHANGED__` for a secret you were not given. Call with dry_run true first, show the user the changes and apply only once they agree." + x-mcp-tool-include-fields: + - document + - dry_run + x-mcp-tool-opaque-fields: + - document + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + document: + description: >- + The settings to apply, in the layout of the document. A secret given as + `__SENSITIVE_AND_UNCHANGED__` keeps its stored value; one prefixed with + `$encrypted:` is decrypted with the workspace key. + type: object + additionalProperties: true + replace: + description: >- + Clear the settings `wmill sync push` clears when settings.yaml omits them. + Otherwise a setting the document omits is left as it is. + type: boolean + dry_run: + description: Only compute the changes, without applying them. + type: boolean + required: + - document + responses: + "200": + description: the changes, applied unless dry_run + content: + application/json: + schema: + type: object + properties: + changes: + type: array + items: + type: object + properties: + key: + type: string + before: {} + after: {} + required: + - key + - before + - after + applied: + type: boolean + warnings: + type: array + items: + type: string + required: + - changes + - applied + /w/{workspace}/workspaces/get_copilot_settings_state: get: summary: get copilot settings state @@ -5967,6 +6068,12 @@ paths: schema: type: integer format: int64 + - name: schema + in: query + required: false + description: apply only the pending migrations whose `-- schemas` annotation declares this schema + schema: + type: string responses: "200": description: applied migrations @@ -9367,6 +9474,12 @@ paths: required: false schema: type: boolean + - name: starred_only + description: Only the `ai_agent` resources the caller starred + in: query + required: false + schema: + type: boolean responses: "200": description: resource list @@ -12783,7 +12896,7 @@ paths: /w/{workspace}/runnables/list: get: - summary: list runnables (scripts, flows, apps) merged, ordered and keyset-paginated + summary: list runnables (scripts, flows, apps, agents) merged, ordered and keyset-paginated operationId: listRunnables tags: - script @@ -12800,7 +12913,7 @@ paths: - $ref: "#/components/parameters/OrderDesc" - name: kinds in: query - description: comma-separated subset of script,flow,app (default all) + description: comma-separated subset of script,flow,app,agent (default all) schema: type: string - name: show_archived @@ -12874,7 +12987,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: kinds in: query - description: comma-separated subset of script,flow,app (default all) + description: comma-separated subset of script,flow,app,agent (default all) schema: type: string - name: include_without_main @@ -15487,6 +15600,12 @@ paths: type: integer tag: type: string + modules: + type: object + nullable: true + description: "Additional modules keyed by relative file path. Honored in preview only: a deployed app runs the modules stored with its runnable." + additionalProperties: + $ref: "#/components/schemas/ScriptModule" required: - content - language @@ -15515,6 +15634,14 @@ paths: description: "Map of relative-import script path -> temp storage hash. Only honored for inline-script (raw_code) execution so app dev resolves those imports from not-yet-deployed local content." additionalProperties: type: string + on_behalf_of: + type: string + nullable: true + description: "Preview mode (force_viewer_static_fields set) only: run as this identity (an email, `u/` or `g/`) instead of the caller. Naming anyone other than yourself requires being an admin or a member of wm_deployers." + viewer_data_access: + type: boolean + nullable: true + description: "Preview mode only: the inline runnable's viewer_data_access policy setting (run mode reads it from the deployed policy)." required: - args - component @@ -24541,7 +24668,7 @@ paths: type: string favorite_kind: type: string - enum: [flow, app, script, raw_app, asset] + enum: [flow, app, script, raw_app, asset, agent] responses: "200": description: star item @@ -24564,7 +24691,7 @@ paths: type: string favorite_kind: type: string - enum: [flow, app, script, raw_app, asset] + enum: [flow, app, script, raw_app, asset, agent] responses: "200": description: unstar item @@ -30236,9 +30363,10 @@ components: description: | A row in the merged runnables listing. `type` is the discriminator; kind-specific fields (hash/language/kind for scripts, execution_mode/ - version for apps) are present only for that kind. `edited_at` is the - unified last-updated time (a script's created_at, a flow/app's edit - time). + version for apps, agent_memory/ws_specific for agents) are present only + for that kind. An agent is an `ai_agent` resource whose description is + reported as `summary`. `edited_at` is the unified last-updated time (a + script's created_at, a flow/app/agent's edit time). properties: type: type: string @@ -30246,6 +30374,7 @@ components: - script - flow - app + - agent path: type: string summary: @@ -30313,6 +30442,11 @@ components: version: type: integer format: int64 + agent_memory: + description: agent-only. The agent's `memory` setting. + ws_specific: + type: boolean + description: agent-only. Whether the agent is workspace-specific, which keeps it out of deploys to other workspaces. required: - type - path @@ -32285,6 +32419,14 @@ components: description: "Map of relative-import script path -> temp storage hash so the preview job resolves those imports from not-yet-deployed local content instead of the deployed script" additionalProperties: type: string + on_behalf_of: + type: string + nullable: true + description: "Run the preview as this identity (an email, `u/` or `g/`) instead of the caller. Naming anyone other than yourself requires being an admin or a member of wm_deployers." + viewer_data_access: + type: boolean + nullable: true + description: "Data table and object storage access as the caller rather than as the job's identity, as a full-code app's inline runnable flagged viewer_data_access does for its viewer." required: - args @@ -36629,6 +36771,10 @@ components: description: "Map of relative-import script path -> temp storage hash, propagated to each flow step so inline-script relative imports resolve from not-yet-deployed local content instead of the deployed script" additionalProperties: type: string + on_behalf_of: + type: string + nullable: true + description: "Run the preview as this identity (an email, `u/` or `g/`) instead of the caller. Naming anyone other than yourself requires being an admin or a member of wm_deployers." required: - value @@ -36725,7 +36871,8 @@ components: the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, - flow_conversations:write). + flow_conversations:write). Every such token can call whoami without + consent, whether `users:read` is declared or not. ListableApp: type: object @@ -37321,6 +37468,15 @@ components: - ran - not_run - unknown + schemas: + type: array + description: the schemas its `-- schemas` annotation declares; absent when it applies to the whole database + items: + type: string + blocked_by: + type: integer + format: int64 + description: when not run, the earliest pending migration that must be applied first; when run, the latest applied migration that must be rolled back first DataTableSchema: type: object required: [datatable_name, schemas] diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 4b67f865f9..453422403d 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -61,7 +61,7 @@ use windmill_common::{ APP_WORKSPACED_ROUTE, }, auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, - cache::{self, future::FutureCachedExt}, + cache, db::{DbWithOptAuthed, UserDB}, error::{to_anyhow, Error, JsonResult, Result}, jobs::{ @@ -1502,8 +1502,9 @@ async fn get_public_app_by_secret( /// - `users:read` → `users/whoami` only. /// - `folders:read` → `folders/listnames` only. /// Plus two path-scoped scopes minted per app (see `mint_app_embed_token`): -/// - `apps:read:` → the app's own definition (`apps/get/p/`); no -/// `apps:write`, so management routes are unreachable. +/// - `apps:read:` → the app's own definition (`apps/get/p/` and +/// `apps/get/lite/`); no `apps:write`, so management +/// routes are unreachable. /// - `apps:run:` → run THIS app's components (`execute_component`, which /// re-checks the path); `apps_u/*` public-serving routes. pub const APP_EMBED_SCOPES: [&str; 5] = [ @@ -4289,6 +4290,12 @@ pub struct ExecuteApp { /// inline-script (raw_code, preview) execution so `wmill app dev` resolves /// those imports from not-yet-deployed local content instead of deployed. pub temp_script_refs: Option>, + /// Preview mode only: run as this identity instead of the caller, as the deployed app + /// would in publisher mode. See `jobs::preview_on_behalf_of`. + pub on_behalf_of: Option, + /// Preview mode only: the inline runnable's `viewer_data_access` policy setting, which run + /// mode reads from the deployed policy instead. + pub viewer_data_access: Option, } fn digest(code: &str) -> String { @@ -4602,6 +4609,7 @@ async fn execute_component( force_viewer_allow_user_resources, force_viewer_sensitive_inputs, force_viewer_delete_after_secs, + viewer_data_access, .. } => ( &Policy { stated_execution_mode: Some(ExecutionMode::Viewer), ..Default::default() }, @@ -4612,8 +4620,9 @@ async fn execute_component( delete_after_secs: force_viewer_delete_after_secs, sensitive_inputs: force_viewer_sensitive_inputs.unwrap_or_default(), tag: None, - // A preview runs as the caller, whose roles these are either way. - viewer_data_access: false, + // Never for a job token: see `run_preview_script`. + viewer_data_access: opt_authed.as_ref().is_some_and(|a| a.job_id.is_none()) + && viewer_data_access.unwrap_or(false), }, None, ), @@ -4666,34 +4675,81 @@ async fn execute_component( // `app_script` id. Its resolved `rawscript/` key must be present // in the policy triggerables below — it must never resolve via the // Viewer default fallback. Without `id` the caller supplies the code - // verbatim; with `id` it selects any `app_script` row by number (no - // app/workspace scoping), so both let a caller run code the publisher - // never pinned for this app. + // verbatim, which lets a caller run code the publisher never pinned + // for this app; with `id` the row must also belong to this app. let is_inline_raw_code = payload.raw_code.is_some(); + // The policy key covers only an inline script's own code, not the lock and modules + // its `app_script` row runs with, so it cannot tell the deployed row from an older + // one (or from id-less code, which runs with neither). Once the deployed version has + // its lite value, inline code runs only through the rows that value points at. + // Checked on every run: a deploy moves it. + if is_inline_raw_code { + let deployed = sqlx::query!( + "SELECT l.id IS NOT NULL as \"has_lite!\", + coalesce(jsonb_path_exists(l.value, '$.**.inlineScript.id ? (@ == $id)', + jsonb_build_object('id', $3::bigint)), false) as \"is_deployed_row!\" + FROM app a + LEFT JOIN app_version_lite l ON l.id = a.versions[array_upper(a.versions, 1)] + WHERE a.path = $1 AND a.workspace_id = $2", + path, + &w_id, + payload.id, + ) + .fetch_optional(&db) + .await?; + if deployed.is_some_and(|d| d.has_lite && !d.is_deployed_row) { + return Err(Error::NotAuthorized(format!( + "This runnable is not part of the deployed version of app {path}, reload the app" + ))); + } + } + // Compute the path for the triggerables map: // - flow: `flow/` // - script: `script/` // - inline script: `rawscript/` + let app_script_key: Arc; let path = match &payload { // flow or script: just use the `payload.path`. ExecuteApp { path: Some(path), .. } => path, // inline script: without entry in the `app_script` table. ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content), // inline script: with an entry in the `app_script` table. + // Scoped to this app: the policy key covers only the entrypoint's code, while the + // row also carries the modules (and lock) the job runs, so another app's row with + // the same entrypoint would run its own modules under this app's identity. ExecuteApp { raw_code: Some(_), id: Some(id), .. } => { - let cache = cache::anon!({ u64 => Arc } in "appscriptpath" <= 10000); - // `id` is unique, cache the result. - &*sqlx::query_scalar!( - "SELECT format('rawscript/%s', code_sha256) as \"path!: String\" - FROM app_script WHERE id = $1 LIMIT 1", - id - ) - .fetch_one(&db) - .map_err(Into::::into) - .map_ok(Arc::new) - .cached(cache, *id as u64) - .await? + lazy_static! { + static ref APP_SCRIPT_KEY_CACHE: cache::Cache<(String, String, i64), Arc> = + cache::Cache::new(10000); + } + let cache_key = (w_id.to_string(), path.to_string(), *id); + app_script_key = match APP_SCRIPT_KEY_CACHE.get(&cache_key) { + Some(key) => key, + None => { + let key = sqlx::query_scalar!( + "SELECT format('rawscript/%s', s.code_sha256) as \"path!: String\" + FROM app_script s JOIN app a ON a.id = s.app + WHERE s.id = $1 AND a.path = $2 AND a.workspace_id = $3", + id, + path, + &w_id, + ) + .fetch_optional(&db) + .await? + .map(Arc::new) + .ok_or_else(|| { + Error::NotAuthorized(format!( + "App script {id} does not belong to app {path}" + )) + })?; + // A row never changes app, so a hit stays valid. + APP_SCRIPT_KEY_CACHE.insert(cache_key, key.clone()); + key + } + }; + &**app_script_key } _ => unreachable!(), }; @@ -4802,8 +4858,19 @@ async fn execute_component( } } - let (username, permissioned_as, email) = + let (username, mut permissioned_as, mut email) = get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; + let preview_obo = match opt_authed.as_ref().filter(|_| is_preview) { + Some(authed) => { + crate::jobs::preview_on_behalf_of(authed, payload.on_behalf_of.as_deref(), &w_id, &db) + .await? + } + None => None, + }; + if let Some(obo) = preview_obo.as_ref() { + permissioned_as = obo.permissioned_as.clone(); + email = obo.email.clone(); + } let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); @@ -4914,16 +4981,16 @@ async fn execute_component( .await?; } } - // Identity is already resolved to the requesting user in preview mode (the - // policy is forced to `ExecutionMode::Viewer`, so the job runs as the - // caller). The enqueue stays root-isolated as before — switching the insert - // to user-RLS is not what contains the bypass (the auth guards above are) - // and would add unnecessary breakage risk to the legitimate editor flow. + // In preview mode the identity is the caller (the policy is forced to + // `ExecutionMode::Viewer`), or the gated `on_behalf_of` resolved above. The + // enqueue stays root-isolated — switching the insert to user-RLS is not what + // contains the bypass (the auth guards above are) and would add unnecessary + // breakage risk to the legitimate editor flow. let tx = PushIsolationLevel::IsolatedRoot(db.clone()); // An app component runs on-behalf of the APP identity (resolved above), never // the referenced runnable's own `on_behalf_of` — else a Viewer-mode app could - // execute as that identity and a preview would run as it, not the caller. + // execute as that identity and a preview would run as it, not as the editor chose. // (Direct `/jobs/run` still honors a runnable's `on_behalf_of`.) let (email, permissioned_as) = (email.as_str(), permissioned_as); @@ -4932,8 +4999,8 @@ async fn execute_component( // Stamp app-origination (trigger_kind='app' + trigger=), the signal // the deployed-app S3 provenance gate trusts (unforgeable via `/jobs/run`). - // Deployed runs only: a preview runs as the caller and is read back as the caller - // (viewer-scoped), so it must never be app-provenanced (else it could forge one). + // Deployed runs only: a preview runs request-supplied code and is read back as the + // caller (viewer-scoped), so it must never be app-provenanced (else it could forge one). let app_trigger = (!is_preview).then(|| TriggerMetadata::new(Some(path.to_string()), JobTriggerKind::App)); @@ -4941,6 +5008,7 @@ async fn execute_component( Some(authed) => windmill_api_auth::caller_scope_ceiling(&db, authed).await?, None => None, }; + let preview_obo_author = preview_obo.as_ref().and(opt_authed.clone()); let (uuid, mut tx) = push( &db, tx, @@ -4980,6 +5048,11 @@ async fn execute_component( ) .await?; + if let (Some(obo), Some(authed)) = (preview_obo.as_ref(), preview_obo_author.as_ref()) { + crate::jobs::audit_preview_on_behalf_of(&mut tx, authed, &w_id, uuid, Some(path), obo) + .await?; + } + if policy_triggerables.viewer_data_access && is_inline { sqlx::query!( "UPDATE job_perms SET viewer_data_access = true WHERE job_id = $1", @@ -6706,8 +6779,8 @@ mod embed_token_tests { "POST", ), ("/api/w/test/jobs/restart/f/some-uuid", "POST"), - // `users:read` is presented to the viewer as "read your identity", so - // the workspace member directory must stay out of reach. + // `users:read` reaches whoami only: the workspace member directory must + // stay out of reach. ("/api/w/test/users/list", "GET"), ("/api/w/test/users/list_usage", "GET"), ("/api/w/test/users/username_to_email/admin", "GET"), @@ -6718,6 +6791,15 @@ mod embed_token_tests { "SDK token should deny {method} {path}" ); } + + // Whoami needs no declared scope. + let jobs_only = [ + "jobs:run".to_string(), + windmill_api_auth::scopes::RAW_APP_SDK_SENTINEL.to_string(), + ]; + assert!( + check_scopes_for_route(Some(&jobs_only[..]), "/api/w/test/users/whoami", "GET").is_ok() + ); } /// The token carries path-scoped `apps:run:` and `apps:read:` diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index db99a07c92..f0ab31b2c9 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -113,6 +113,12 @@ lazy_static::lazy_static! { (20260909163047, include_str!( "../../migrations/20260909163047_workspace_delete_cascade_indexes.up.sql" ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20261009173546, include_str!( + "../../migrations/20261009173546_agent_owner_prefix_index.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")), + (20261015114712, include_str!( + "../../migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), // The widening fails every query the previous release's still-running // servers prepared on the column; `variables::widen_variable_value` does it // once they are gone. diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index e6b6db6919..b582e84e36 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -24,7 +24,7 @@ use windmill_common::{ users::resolve_username_to_email, utils::{check_proper_path, require_admin, strip_json_nul}, variables::{build_crypt, encrypt}, - workspaces::{operator_builder_rights, BuilderKind}, + workspaces::operator_can_draft, }; pub fn workspaced_service() -> Router { @@ -109,12 +109,6 @@ async fn list_drafts( Path(w_id): Path, Query(query): Query, ) -> Result>> { - // Without builder rights an operator has no drafts of their own (they can't write any, see - // `require_can_write_path`), so this list is always empty for them. They can still READ some - // collaborators' drafts via `/drafts/get`. - if authed.is_operator && !operator_builder_rights(&db, &w_id).await?.any() { - return Ok(Json(vec![])); - } let all_users = query.all_users.unwrap_or(false); // Only honor `compare_to_workspace` when it is genuinely this workspace's // parent (a fork comparing against its source). Any other value is dropped @@ -1252,8 +1246,7 @@ fn table_for_kind(kind: UserDraftItemKind) -> Option<&'static str> { } /// Resolves to `Ok(())` if `authed` may SAVE a draft at `path`. Operators are -/// rejected, except for flow drafts in a workspace that granted them builder -/// rights. Two layers: +/// rejected, except for a kind they could deploy (`operator_can_draft`). Two layers: /// 1. Claim-based namespace rules (admin, own `u/`, member `g/`, writable /// `f/`) — mirror what RLS reads from the same JWT claims, and are the /// ENTIRE check for draft-only paths (no deployed row for RLS to use). @@ -1270,26 +1263,17 @@ pub(crate) async fn require_can_write_path( if authed.is_admin { return Ok(()); } - // Operators are read-only and never WRITE drafts, except of a kind the workspace granted them - // the matching builder right for: a flows-only workspace must not get raw-app drafts through - // here. Read access is deliberately asymmetric: - // `require_can_read_path` has no operator block, so an operator can still READ a draft they - // can read via `/drafts/get`, mirroring their read access to deployed content. Intended. - if authed.is_operator { - let allowed = match kind { - UserDraftItemKind::Flow => Some(BuilderKind::Flows), - UserDraftItemKind::RawApp => Some(BuilderKind::Apps), - _ => None, - }; - let granted = match allowed { - Some(kind) => operator_builder_rights(db, w_id).await?.has(kind), - None => false, - }; - if !granted { - return Err(Error::PermissionDenied( - "operators cannot save drafts".to_string(), - )); - } + // Operators WRITE drafts only of a kind they could deploy: resources and variables always, + // flows and full-code apps under their builder right, schedules and triggers under the manage + // rights. The kind has to be checked, or one right would open drafts of kinds it says nothing + // about. Read access is deliberately asymmetric: `require_can_read_path` has no operator block, + // so an operator can still READ a draft they can read via `/drafts/get`, mirroring their read + // access to deployed content. Intended. + if authed.is_operator && !operator_can_draft(db, w_id, kind).await? { + return Err(Error::PermissionDenied(format!( + "operators cannot save {} drafts in this workspace", + kind.as_str() + ))); } // Cheap claim-based namespace checks first: they evaluate the same JWT // claims RLS reads, so the outcome matches the policies while sparing the diff --git a/backend/windmill-api/src/favorite.rs b/backend/windmill-api/src/favorite.rs index d3e3456e0f..86f3377b39 100644 --- a/backend/windmill-api/src/favorite.rs +++ b/backend/windmill-api/src/favorite.rs @@ -32,6 +32,7 @@ pub enum FavoriteKind { #[allow(non_camel_case_types)] Raw_App, Asset, + Agent, } #[derive(Deserialize)] pub struct Favorite { diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 1a7a073732..b939ba017a 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -115,7 +115,9 @@ use windmill_common::{ add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue, Suspend, }, - jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, + jobs::{ + script_path_to_payload, CompletedJob, JobKind, JobPayload, OnBehalfOf, QueuedJob, RawCode, + }, oauth2::HmacSha256, query_builders, scripts::{ScriptHash, ScriptLang}, @@ -6736,6 +6738,12 @@ struct Preview { /// preview job resolves those imports from not-yet-deployed local content /// (uploaded to raw_script_temp) instead of the deployed script. temp_script_refs: Option>, + /// Run the preview as this identity (an email or a `u/`/`g/` principal) instead of the + /// caller. See `preview_on_behalf_of`. + on_behalf_of: Option, + /// Data table and object storage access as the caller rather than as the job's identity, + /// as a full-code app's inline runnable flagged `viewer_data_access` does for its viewer. + viewer_data_access: Option, } #[cfg(feature = "run_inline")] @@ -6768,6 +6776,8 @@ struct PreviewFlow { /// each flow step so inline-script relative imports resolve from /// not-yet-deployed local content instead of the deployed script. temp_script_refs: Option>, + /// See `Preview::on_behalf_of`. + on_behalf_of: Option, } #[derive(Debug, Deserialize)] @@ -8930,6 +8940,112 @@ fn operator_preview_refusal(job_id: Option) -> error::Error { error::Error::PermissionDenied(reason.to_string()) } +/// The identity a preview asks to run as, or `None` when it runs as the caller. +/// +/// A preview is request-supplied code, so running it as someone else is impersonation, not +/// "running their runnable": only those who may set a runnable's `on_behalf_of` on deploy +/// (admins and `wm_deployers`) may ask for it. Naming oneself is always allowed. Operators are +/// refused even in `wm_deployers`: the datatable-query exemption lets them reach this route +/// without being able to deploy anything. +pub(crate) async fn preview_on_behalf_of( + authed: &ApiAuthed, + requested: Option<&str>, + w_id: &str, + db: &DB, +) -> error::Result> { + let Some(requested) = requested.filter(|s| !s.is_empty()) else { + return Ok(None); + }; + let own = username_to_permissioned_as(&authed.username); + if requested == own || requested == authed.email { + return Ok(None); + } + if authed.is_operator || !windmill_common::can_preserve_on_behalf_of(authed) { + return Err(Error::PermissionDenied( + "Only admins and members of wm_deployers can run a preview on behalf of another user" + .to_string(), + )); + } + let permissioned_as = + windmill_common::resolve_on_behalf_of(None, Some(requested), true, authed, w_id, db) + .await? + .unwrap_or(own.clone()); + if permissioned_as == own { + return Ok(None); + } + let email = + windmill_common::users::get_email_from_permissioned_as_uncached(&permissioned_as, w_id, db) + .await?; + Ok(Some(OnBehalfOf { email, permissioned_as })) +} + +/// Audits the impersonation and opens the job's own logs with it, so whoever reads the run sees +/// who it ran as and who asked, not only the identity columns. +pub(crate) async fn audit_preview_on_behalf_of( + tx: &mut Transaction<'_, Postgres>, + authed: &ApiAuthed, + w_id: &str, + job_id: Uuid, + path: Option<&str>, + obo: &OnBehalfOf, +) -> error::Result<()> { + let mut params = HashMap::from([("on_behalf_of", obo.permissioned_as.as_str())]); + if let Some(path) = path { + params.insert("path", path); + } + audit_log( + &mut **tx, + authed, + "jobs.run.preview_on_behalf_of", + ActionKind::Execute, + w_id, + Some(&job_id.to_string()), + Some(params), + ) + .await?; + sqlx::query!( + "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)", + format!( + "Preview run on behalf of {} ({}), requested by {}\n", + obo.permissioned_as, obo.email, authed.email + ), + job_id, + w_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// The push identity and isolation level for a preview: the caller's own, or the requested +/// `on_behalf_of`'s, pushed through the root pool like a deployed `on_behalf_of` run. +fn preview_push_identity( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + obo: Option<&OnBehalfOf>, +) -> ( + String, + String, + Option, + PushIsolationLevel<'static>, +) { + match obo { + Some(obo) => ( + obo.email.clone(), + obo.permissioned_as.clone(), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ), + None => ( + authed.email.clone(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), + ), + } +} + async fn run_preview_script( authed: ApiAuthed, Extension(db): Extension, @@ -8964,7 +9080,13 @@ async fn run_preview_script( require_path_read_access_for_preview(&authed, &preview.path)?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let obo = preview_on_behalf_of(&authed, preview.on_behalf_of.as_deref(), &w_id, &db).await?; + let (email, permissioned_as, push_authed, tx) = + preview_push_identity(&authed, &user_db, &db, obo.as_ref()); + let audited_path = preview.path.clone(); + // A job token's end user is whoever ran the calling job, who never consented to that job's + // code reading data as them: only a direct caller, reading as themselves, may ask for it. + let viewer_data_access = authed.job_id.is_none() && preview.viewer_data_access.unwrap_or(false); let preview_args = preview.args.unwrap_or_default(); let mut extra = HashMap::new(); @@ -9010,8 +9132,8 @@ async fn run_preview_script( }, push_args, authed.display_username(), - &authed.email, - username_to_permissioned_as(&authed.username), + &email, + permissioned_as, authed.token_prefix.as_deref(), authed.username_override.as_deref(), scheduled_for, @@ -9028,7 +9150,7 @@ async fn run_preview_script( run_query.timeout, None, None, - Some(&authed.clone().into()), + push_authed.as_ref(), false, end_user_email, authed.trigger_or_fallback(None), @@ -9036,6 +9158,19 @@ async fn run_preview_script( scope_ceiling.as_deref(), ) .await?; + let mut tx = tx; + if let Some(obo) = obo.as_ref() { + audit_preview_on_behalf_of(&mut tx, &authed, &w_id, uuid, audited_path.as_deref(), obo) + .await?; + } + if viewer_data_access { + sqlx::query!( + "UPDATE job_perms SET viewer_data_access = true WHERE job_id = $1", + uuid + ) + .execute(&mut *tx) + .await?; + } tx.commit().await?; Ok((StatusCode::CREATED, uuid.to_string())) @@ -9387,7 +9522,11 @@ async fn run_bundle_preview_script( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - let ltx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let obo = + preview_on_behalf_of(&authed, preview.on_behalf_of.as_deref(), &w_id, &db).await?; + let (email, permissioned_as, push_authed, ltx) = + preview_push_identity(&authed, &user_db, &db, obo.as_ref()); + let audited_path = preview.path.clone(); let args = preview.args.unwrap_or_default(); @@ -9435,8 +9574,8 @@ async fn run_bundle_preview_script( }), push_args, authed.display_username(), - &authed.email, - username_to_permissioned_as(&authed.username), + &email, + permissioned_as, authed.token_prefix.as_deref(), authed.username_override.as_deref(), scheduled_for, @@ -9453,7 +9592,7 @@ async fn run_bundle_preview_script( run_query.timeout, None, None, - Some(&authed.clone().into()), + push_authed.as_ref(), false, end_user_email, authed.trigger_or_fallback(None), @@ -9461,6 +9600,18 @@ async fn run_bundle_preview_script( scope_ceiling.as_deref(), ) .await?; + let mut ntx = ntx; + if let Some(obo) = obo.as_ref() { + audit_preview_on_behalf_of( + &mut ntx, + &authed, + &w_id, + uuid, + audited_path.as_deref(), + obo, + ) + .await?; + } job_id = Some(uuid); tx = Some(ntx); } @@ -10138,7 +10289,9 @@ async fn run_preview_flow_job( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(raw_flow.tag.clone()); - let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let obo = preview_on_behalf_of(&authed, raw_flow.on_behalf_of.as_deref(), &w_id, &db).await?; + let (email, permissioned_as, push_authed, tx) = + preview_push_identity(&authed, &user_db, &db, obo.as_ref()); let chat_input_enabled = raw_flow.value.chat_input_enabled.unwrap_or(false); let flow_path = raw_flow.path.clone().unwrap_or_default(); @@ -10171,8 +10324,8 @@ async fn run_preview_flow_job( }, PushArgs::from(&flow_args), authed.display_username(), - &authed.email, - username_to_permissioned_as(&authed.username), + &email, + permissioned_as, authed.token_prefix.as_deref(), authed.username_override.as_deref(), scheduled_for, @@ -10189,7 +10342,7 @@ async fn run_preview_flow_job( None, None, None, - Some(&authed.clone().into()), + push_authed.as_ref(), false, end_user_email, authed.trigger_or_fallback(None), @@ -10198,6 +10351,11 @@ async fn run_preview_flow_job( ) .await?; + if let Some(obo) = obo.as_ref() { + let audited_path = Some(flow_path.as_str()).filter(|p| !p.is_empty()); + audit_preview_on_behalf_of(&mut tx, &authed, &w_id, uuid, audited_path, obo).await?; + } + // Set memory_id if provided (for agent memory) if let Some(memory_id) = run_query.memory_key(&w_id, &flow_path) { set_flow_memory_id(&mut tx, uuid, memory_id).await?; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 92291bd1ea..9163256997 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -209,6 +209,7 @@ mod workspaces; #[cfg(feature = "private")] pub mod workspaces_ee; mod workspaces_export; +mod workspace_settings_document; #[cfg(feature = "mcp")] mod mcp_tools; diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 73b1fd1792..dd3c83a08d 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -56,6 +56,46 @@ pub fn all_tools() -> Vec { query_field_renames: None, body_field_renames: None, }, + EndpointTool { + name: Cow::Borrowed("getWorkspaceSettingsDocument"), + description: Cow::Borrowed("get the workspace settings document: The workspace's settings in the layout of the settings.yaml that `wmill sync` writes. Admin only"), + instructions: Cow::Borrowed(""), + path: Cow::Borrowed("/w/{workspace}/workspaces/settings_document"), + method: Cow::Borrowed("GET"), + path_params_schema: None, + query_params_schema: None, + body_schema: None, + query_field_renames: None, + body_field_renames: None, + }, + EndpointTool { + name: Cow::Borrowed("applyWorkspaceSettingsDocument"), + description: Cow::Borrowed("apply a workspace settings document: Applies a settings document, each setting through the endpoint that owns it. Admin only"), + instructions: Cow::Borrowed("Send only the top-level keys to change, each whole as getWorkspaceSettingsDocument returns it: a key the document omits is left as it is, and a key set to null clears it. Keep `__SENSITIVE_AND_UNCHANGED__` for a secret you were not given. Call with dry_run true first, show the user the changes and apply only once they agree."), + path: Cow::Borrowed("/w/{workspace}/workspaces/settings_document"), + method: Cow::Borrowed("POST"), + path_params_schema: None, + query_params_schema: None, + body_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "document": { + "type": "object", + "description": "The settings to apply, in the layout of the document. A secret given as `__SENSITIVE_AND_UNCHANGED__` keeps its stored value; one prefixed with `$encrypted:` is decrypted with the workspace key." + }, + "dry_run": { + "description": "Only compute the changes, without applying them.", + "type": "boolean" + } + }, + "required": [ + "document" + ], + "minProperties": 1 +})), + query_field_renames: None, + body_field_renames: None, + }, EndpointTool { name: Cow::Borrowed("listDataMetrics"), description: Cow::Borrowed("list declared measures and dimensions on DuckLake tables: Call this before writing any aggregate query over a DuckLake table. A declared measure is the canonical definition of that number, and reproducing it yourself will silently disagree with it (a `revenue` measure typically excludes refunds or test rows). Filter by `table` for one table's declarations, or by `path_prefix` (e.g. `f/analytics`) for everything declared under a folder; omit both to browse the whole catalog. Results are keyset-paged: a full page may mean more remain, so continue with the `cursor_*` params rather than assuming a measure does not exist. Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates can share one GROUP BY. If a number you need has no declared measure, write your own aggregate as usual. Results are limited to declarations whose producing script the caller can read"), @@ -559,6 +599,10 @@ pub fn all_tools() -> Vec { "include_draft_only": { "type": "boolean", "description": "When true, append per-user draft resources whose path has\nno deployed resource. Synthesized rows carry\n`draft_only: true`.\n" + }, + "starred_only": { + "type": "boolean", + "description": "Only the `ai_agent` resources the caller starred" } }, "required": [] @@ -1296,7 +1340,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write). Every such token can call whoami without consent, whether `users:read` is declared or not.\n" } } } @@ -1413,7 +1457,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write). Every such token can call whoami without consent, whether `users:read` is declared or not.\n" } } }, diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index 7e64b48a1a..754221dac0 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -7,15 +7,17 @@ */ //! Unified, keyset-paginated listing of a workspace's runnables (scripts, -//! flows, apps) merged into one globally-ordered stream. The homepage uses it -//! so a chosen order (recently updated / oldest / name) is correct and complete -//! across all three kinds at any workspace size, instead of client-sorting a +//! flows, apps, and the `ai_agent` resources listed as agents) merged into one +//! globally-ordered stream. The homepage uses it so a chosen order (recently +//! updated / oldest / name) is correct and complete across every kind at any +//! workspace size, instead of client-sorting a //! per-kind capped window. //! //! Efficiency: each kind is a UNION ALL branch ordered by an index on //! `(workspace_id, archived, )` (created_at / edited_at, or the lowered //! summary-or-path expression for name orders); Postgres merges the ordered -//! branches and stops at the page limit. Pagination is keyset — a +//! branches and stops at the page limit. Agents, which are few, are instead +//! reached through the partial `idx_resource_agent_owner_prefix` and sorted whole. Pagination is keyset — a //! `(sort_key, path, kind, tiebreak)` cursor, where `tiebreak` (a script's hash, //! 0 for flow/app) is a stable final key that keeps the order total even when rows //! tie on (sort_key, path, kind) — so deep pages don't re-scan. Visibility is @@ -51,7 +53,7 @@ struct ListRunnablesQuery { order_by: Option, /// Descending when true (default true). order_desc: Option, - /// Comma-separated subset of `script,flow,app`; omitted means all. + /// Comma-separated subset of `script,flow,app,agent`; omitted means all. kinds: Option, show_archived: Option, /// Include library scripts (no runnable main). Ignored for flows/apps. @@ -80,7 +82,7 @@ struct ListRunnablesQuery { #[derive(Serialize, sqlx::FromRow)] struct RunnableItem { #[serde(rename = "type")] - kind: String, // 'script' | 'flow' | 'app' + kind: String, // 'script' | 'flow' | 'app' | 'agent' path: String, #[serde(skip_serializing_if = "Option::is_none")] summary: Option, @@ -130,6 +132,11 @@ struct RunnableItem { id: Option, #[serde(skip_serializing_if = "Option::is_none")] version: Option, + // agent-only: the `memory` setting, which decides whether the agent opens as a chat. + #[serde(skip_serializing_if = "Option::is_none")] + agent_memory: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ws_specific: Option, // sort keys, echoed into the cursor (not serialized to the client) #[serde(skip)] sort_time: chrono::DateTime, @@ -247,7 +254,7 @@ fn scope_path_predicate( } } -/// The three UNION-ALL branch SELECTs, each projecting the shared `RunnableItem` +/// The UNION-ALL branch SELECTs, each projecting the shared `RunnableItem` /// column set (NULL for columns that don't apply to that kind). `$1`=workspace, /// `$2`=username (favorites), `$3`=email (drafts). Kind-specific and per-request /// WHERE fragments are appended by the caller. @@ -255,6 +262,7 @@ struct Branches { script: String, flow: String, app: String, + agent: String, } fn branch_sqls() -> Branches { @@ -281,6 +289,7 @@ fn branch_sqls() -> Branches { o.codebase IS NOT NULL as use_codebase, \ (o.lock_error_logs IS NOT NULL) as has_deploy_errors, NULL::bool as chat_input_enabled, \ NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ + NULL::jsonb as agent_memory, NULL::bool as ws_specific, \ o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, o.hash as tiebreak \ FROM script o \ LEFT JOIN favorite ON favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ @@ -298,6 +307,7 @@ fn branch_sqls() -> Branches { NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \ (o.value->>'chat_input_enabled')::bool as chat_input_enabled, \ NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ + NULL::jsonb as agent_memory, NULL::bool as ws_specific, \ o.edited_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ FROM flow o \ LEFT JOIN favorite ON favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ @@ -315,6 +325,7 @@ fn branch_sqls() -> Branches { NULL::bool as use_codebase, NULL::bool as has_deploy_errors, NULL::bool as chat_input_enabled, \ av.raw_app, o.policy->>'execution_mode' as execution_mode, o.id, \ o.versions[array_upper(o.versions, 1)] as version, \ + NULL::jsonb as agent_memory, NULL::bool as ws_specific, \ COALESCE(av.created_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ FROM app o \ LEFT JOIN favorite ON favorite.favorite_kind = 'app' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ @@ -323,25 +334,68 @@ fn branch_sqls() -> Branches { draft_users = draft_users("d.typ IN ('app', 'raw_app')") ); - Branches { script, flow, app } + // An agent is an `ai_agent` resource; the subquery renames its description to the + // `summary` the shared search and name sort read. + let agent = format!( + "SELECT 'agent' as kind, o.path, o.summary, o.workspace_id, o.extra_perms, \ + favorite.path IS NOT NULL as starred, false as archived, \ + draft.email IS NOT NULL as is_draft, NULL::bool as draft_only, NULL::text as draft_path, \ + {draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \ + NULL::bool as ws_error_handler_muted, o.edited_at, \ + NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ + NULL::bool as use_codebase, NULL::bool as has_deploy_errors, NULL::bool as chat_input_enabled, \ + NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ + o.agent_memory, ws_specific.path IS NOT NULL as ws_specific, \ + COALESCE(o.edited_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ + FROM (SELECT workspace_id, path, description as summary, extra_perms, labels, edited_at, \ + value->'memory' as agent_memory \ + FROM resource WHERE resource_type = 'ai_agent') o \ + LEFT JOIN favorite ON favorite.favorite_kind = 'agent' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ + LEFT JOIN draft ON draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'resource' AND draft.email = $3 \ + LEFT JOIN ws_specific ON ws_specific.path = o.path AND ws_specific.workspace_id = o.workspace_id AND ws_specific.item_kind = 'resource'", + draft_users = draft_users("d.typ = 'resource'") + ); + + Branches { script, flow, app, agent } +} + +/// The draft rows a kind lists and the table whose rows make a draft not draft-only. +fn draft_kind_source(kind: &str) -> (&'static str, &'static str) { + match kind { + "script" => ("d.typ = 'script'", "script"), + "flow" => ("d.typ = 'flow'", "flow"), + // Typed by the draft itself, as the resource listing filters its draft-only rows. + "agent" => ( + "d.typ = 'resource' AND d.value->>'resource_type' = 'ai_agent'", + "resource", + ), + _ => ("d.typ IN ('app', 'raw_app')", "app"), + } +} + +/// Where a draft records the path its editor has staged. Scripts and resources bind the +/// Path widget to the item's own `path`, so it round-trips through the draft JSON's +/// `path`; flows and apps write a separate `draft_path` only when it differs from the +/// deployed one. See scripts.rs. +fn draft_typed_path(kind: &str) -> &'static str { + if kind == "script" || kind == "agent" { + "path" + } else { + "draft_path" + } } /// The draft-only branch for a kind: the caller's drafts at paths carrying no /// deployed row, projected into the same column set as `branch_sqls` so they /// sort, search and paginate as ordinary rows. Same `$1`/`$2`/`$3` contract. fn draft_branch_sql(kind: &str) -> String { - let (typ_pred, deployed) = match kind { - "script" => ("d.typ = 'script'", "script"), - "flow" => ("d.typ = 'flow'", "flow"), - _ => ("d.typ IN ('app', 'raw_app')", "app"), - }; - // Scripts bind the Path widget to `script.path`, so the typed path round-trips - // through the draft JSON's own `path`; flows and apps write a separate - // `draft_path` only when it differs from the deployed one. See scripts.rs. - let typed_path = if kind == "script" { - "path" + let (typ_pred, deployed) = draft_kind_source(kind); + let typed_path = draft_typed_path(kind); + // A resource draft carries the description where the other kinds carry a summary. + let summary_field = if kind == "agent" { + "description" } else { - "draft_path" + "summary" }; // `auto_kind` is only what the editor stamped into the draft — a `// pipeline` // annotation is not re-derived from the content here, unlike the per-kind @@ -351,7 +405,8 @@ fn draft_branch_sql(kind: &str) -> String { "script" => { "d.value->>'language' as language, d.value->>'kind' as script_kind, \ d.value->>'auto_kind' as auto_kind, false as raw_app, \ - NULL::bool as chat_input_enabled" + NULL::bool as chat_input_enabled, \ + NULL::jsonb as agent_memory, NULL::bool as ws_specific" } // Type-guarded rather than a bare `::bool` cast: draft JSON is stored // unvalidated, so a malformed value must yield NULL, not abort the list, @@ -360,11 +415,20 @@ fn draft_branch_sql(kind: &str) -> String { "NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ false as raw_app, \ CASE WHEN json_typeof(d.value->'value'->'chat_input_enabled') = 'boolean' \ - THEN (d.value->'value'->>'chat_input_enabled')::bool END as chat_input_enabled" + THEN (d.value->'value'->>'chat_input_enabled')::bool END as chat_input_enabled, \ + NULL::jsonb as agent_memory, NULL::bool as ws_specific" + } + "agent" => { + "NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ + false as raw_app, NULL::bool as chat_input_enabled, \ + (d.value->'args'->'memory')::jsonb as agent_memory, \ + CASE WHEN json_typeof(d.value->'wsSpecific') = 'boolean' \ + THEN (d.value->>'wsSpecific')::bool END as ws_specific" } _ => { "NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ - (d.typ = 'raw_app') as raw_app, NULL::bool as chat_input_enabled" + (d.typ = 'raw_app') as raw_app, NULL::bool as chat_input_enabled, \ + NULL::jsonb as agent_memory, NULL::bool as ws_specific" } }; format!( @@ -378,11 +442,12 @@ fn draft_branch_sql(kind: &str) -> String { NULL::bigint as hash, o.language, o.script_kind, o.auto_kind, \ NULL::bool as use_codebase, NULL::bool as has_deploy_errors, o.chat_input_enabled, \ o.raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ + o.agent_memory, o.ws_specific, \ o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.draft_path, o.path)) as sort_name, 0::bigint as tiebreak \ FROM ( \ SELECT DISTINCT ON (d.path) d.workspace_id, d.path, d.created_at, \ d.email IS NULL as legacy, \ - COALESCE(d.value->>'summary', '') as summary, \ + COALESCE(d.value->>'{summary_field}', '') as summary, \ NULLIF(NULLIF(d.value->>'{typed_path}', ''), d.path) as draft_path, \ {kind_cols} \ FROM draft d \ @@ -397,6 +462,8 @@ fn draft_branch_sql(kind: &str) -> String { ) } +const ALL_KINDS: [&str; 4] = ["script", "flow", "app", "agent"]; + async fn list_runnables( authed: ApiAuthed, Extension(user_db): Extension, @@ -416,16 +483,16 @@ async fn list_runnables( }; let mut kinds: Vec<&str> = match q.kinds.as_deref() { - None | Some("") => vec!["script", "flow", "app"], + None | Some("") => ALL_KINDS.to_vec(), Some(csv) => csv .split(',') .map(|s| s.trim()) - .filter(|s| ["script", "flow", "app"].contains(s)) + .filter(|s| ALL_KINDS.contains(s)) .collect(), }; - // Apps carry no `archived` column and are never listed as archived. + // Apps and agents carry no `archived` column and are never listed as archived. if show_archived { - kinds.retain(|k| *k != "app"); + kinds.retain(|k| *k != "app" && *k != "agent"); } let branches = branch_sqls(); @@ -533,6 +600,11 @@ async fn list_runnables( } else { None }; + let agent_scope = if kinds.contains(&"agent") { + scope_path_predicate(&authed, "resources", "o", 3, &mut binds) + } else { + None + }; // Per-kind archived predicate (scripts/flows have the column; apps don't and // are excluded from the archived view). @@ -570,6 +642,10 @@ async fn list_runnables( if let Some(s) = &app_scope { app_extras.push(s.clone()); } + let mut agent_extras: Vec = vec![]; + if let Some(s) = &agent_scope { + agent_extras.push(s.clone()); + } // Draft-only rows are the caller's own work in progress: never archived, so they // have no place in the archived view, and carrying no labels of their own they are @@ -589,6 +665,7 @@ async fn list_runnables( let scope = match kind { "script" => &script_scope, "flow" => &flow_scope, + "agent" => &agent_scope, _ => &app_scope, }; // The lib filter reads the same projected `auto_kind`, so a draft-only library @@ -649,6 +726,7 @@ async fn list_runnables( "script" => (&branches.script, &script_extras), "flow" => (&branches.flow, &flow_extras), "app" => (&branches.app, &app_extras), + "agent" => (&branches.agent, &agent_extras), _ => return None, }; Some(build_branch( @@ -708,7 +786,7 @@ async fn list_runnables( // No draft branch here: a draft-only path has no favorite row (the UI won't let // you star one), so it would scan the caller's whole draft slice per kind to // return nothing. The main stream below takes them unfiltered instead. - let starred_branches: Vec = ["script", "flow", "app"] + let starred_branches: Vec = ALL_KINDS .iter() .filter_map(|k| branch_for(k, Some(true), None, None)) .collect(); @@ -728,10 +806,10 @@ async fn list_runnables( // Main paged stream: non-starred rows (starred were pinned on the first page above). let main_fav = Some(false); let ns_branches: Vec = - ["script", "flow", "app"] + ALL_KINDS .iter() .filter_map(|k| branch_for(k, main_fav, keyset_sql.as_deref(), Some(per_page))) - .chain(["script", "flow", "app"].iter().filter_map(|k| { + .chain(ALL_KINDS.iter().filter_map(|k| { draft_branch_for(k, main_fav, keyset_sql.as_deref(), Some(per_page)) })) .collect(); @@ -762,7 +840,7 @@ async fn list_runnables( #[derive(Deserialize)] struct CountRunnablesQuery { - /// Comma-separated subset of `script,flow,app`; omitted means all. + /// Comma-separated subset of `script,flow,app,agent`; omitted means all. kinds: Option, /// Include library scripts (no runnable main). Ignored for flows/apps. include_without_main: Option, @@ -789,7 +867,7 @@ struct OwnerCount { /// `owner/` is >= `owner || '/'` and, since '/' (0x2F) is immediately /// followed by '0' (0x30), < `owner || '0'`. `~>=~` / `~<~` are the /// text_pattern_ops operators, which is what lets `idx__owner_prefix` -/// answer the count with an index-only scan — the default opclass sorts by the +/// (`idx_resource_agent_owner_prefix` for agents) answer the count with an index-only scan — the default opclass sorts by the /// database collation and cannot serve a byte-prefix range. fn owner_prefix_range(alias: &str) -> String { format!("{alias}.path ~>=~ (o.owner || '/') AND {alias}.path ~<~ (o.owner || '0')") @@ -818,13 +896,13 @@ async fn count_runnables_by_owner( Query(q): Query, ) -> JsonResult { let kinds: Vec<&str> = match q.kinds.as_deref() { - None | Some("") => vec!["script", "flow", "app"], + None | Some("") => ALL_KINDS.to_vec(), // Deduplicated: every kind becomes its own count subquery, so a repeated // entry would both double that kind's count and multiply the scans. Some(csv) => { let mut ks: Vec<&str> = vec![]; for k in csv.split(',').map(|s| s.trim()) { - if ["script", "flow", "app"].contains(&k) && !ks.contains(&k) { + if ALL_KINDS.contains(&k) && !ks.contains(&k) { ks.push(k); } } @@ -866,6 +944,13 @@ async fn count_runnables_by_owner( w.push(s); } } + "agent" => { + w.push(format!("{alias}.resource_type = 'ai_agent'")); + if let Some(s) = scope_path_predicate(&authed, "resources", alias, base, binds) + { + w.push(s); + } + } _ => { if let Some(s) = scope_path_predicate(&authed, "apps", alias, base, binds) { w.push(s); @@ -877,6 +962,7 @@ async fn count_runnables_by_owner( let table_of = |kind: &str| match kind { "script" => "script", "flow" => "flow", + "agent" => "resource", _ => "app", }; @@ -1048,16 +1134,17 @@ async fn add_draft_counts( let branches: Vec = kinds .iter() .map(|kind| { - let (typ_pred, deployed, domain) = match *kind { - "script" => ("d.typ = 'script'", "script", "scripts"), - "flow" => ("d.typ = 'flow'", "flow", "flows"), - _ => ("d.typ IN ('app', 'raw_app')", "app", "apps"), + let (typ_pred, deployed) = draft_kind_source(kind); + let domain = match *kind { + "script" => "scripts", + "flow" => "flows", + "agent" => "resources", + _ => "apps", }; // The owner a draft counts under is where it says it will live, not the // `u//draft_` it is parked at — same path `/list` groups and - // filters on. Scripts round-trip the typed path through the draft JSON's - // own `path`; flows and apps write `draft_path`. See scripts.rs. - let typed_path = if *kind == "script" { "path" } else { "draft_path" }; + // filters on. + let typed_path = draft_typed_path(kind); let effective_path = format!("COALESCE(NULLIF(d.value->>'{typed_path}', ''), d.path) as path"); let mut w = vec![ diff --git a/backend/windmill-api/src/workspace_settings_document.rs b/backend/windmill-api/src/workspace_settings_document.rs new file mode 100644 index 0000000000..22ba1de6c6 --- /dev/null +++ b/backend/windmill-api/src/workspace_settings_document.rs @@ -0,0 +1,1951 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The workspace settings document: the workspace's settings in the layout of the `settings.yaml` +//! that `wmill sync` writes, read and applied as one. `wmill sync push`, the settings page's YAML +//! mode and the AI tools all apply through [`apply_settings_document`], so the file layout and +//! the rules for applying it exist once. + +use axum::{ + extract::{Path, Query}, + Extension, Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use sqlx::PgConnection; +use std::collections::BTreeMap; +use windmill_api_workspaces::workspaces::{apply_settings_edit, SettingsEdit}; +use windmill_common::{ + datatable_roles::{ + read_role_catalog, DatatableRoleCluster, ADMIN_DATATABLE_ROLE, NO_DEFAULT_DATATABLE_ROLE, + }, + error::{Error, JsonResult, Result}, + utils::require_admin, + variables::build_crypt, + workspaces::{ + DataTable, DataTablePermissions, ProtectionRuleKind, ProtectionRules, + DATATABLE_TENANT_WILDCARD, DEV_WORKSPACE_LOCK_RULE_NAME, + }, + DB, +}; + +use crate::db::ApiAuthed; + +/// Stands for a secret a reader may not see. Sent back unchanged, it keeps the stored value. +pub const SENSITIVE_UNCHANGED: &str = "__SENSITIVE_AND_UNCHANGED__"; +use windmill_common::variables::SETTINGS_ENCRYPTED_PREFIX as ENCRYPTED_PREFIX; + +const SECRET_KEYS: &[&str] = &["slack_oauth_client_secret"]; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum SecretMode { + /// Secrets as [`SENSITIVE_UNCHANGED`], for the UI and AI tools. + #[default] + Redacted, + /// Secrets encrypted with the workspace key, deterministically so that an unchanged secret + /// exports to the same text and a synced file does not change on every pull. + Encrypted, + Plain, +} + +#[derive(sqlx::FromRow)] +struct SettingsRow { + auto_invite: Option, + webhook: Option, + error_handler: Option, + success_handler: Option, + ai_config: Option, + large_file_storage: Option, + git_sync: Option, + default_app: Option, + default_scripts: Option, + name: Option, + mute_critical_alerts: Option, + color: Option, + operator_settings: Option, + datatable: Option, + slack_team_id: Option, + slack_name: Option, + slack_command_script: Option, + slack_oauth_client_id: Option, + slack_oauth_client_secret: Option, + ducklake: Option, + dbt_warehouses: Option, + deploy_ui: Option, + teams_command_script: Option, + public_app_execution_limit_per_minute: Option, + guest_access_enabled: Option, + guest_jwt_public_key: Option, + guest_jwt_jwks_url: Option, + error_handler_fallback_to_instance_alerts: Option, + add_admins_and_developers_to_forks: Option, +} + +/// Server-owned git sync state (the webhook secret and hook id, the synced sha, the admin +/// automatic pulls run as, what the credential check observed) never leaves the server: a +/// re-imported workspace must not inherit another install's hook, sync state or pull identity, +/// and in a synced file it would churn for no reason. +pub(crate) fn redact_git_sync(git_sync: Option) -> Option { + let mut git_sync = git_sync?; + if let Some(repos) = git_sync + .get_mut("repositories") + .and_then(|r| r.as_array_mut()) + { + for repo in repos { + if let Some(auto_pull) = repo.get_mut("auto_pull").and_then(|a| a.as_object_mut()) { + for field in [ + "webhook_secret", + "webhook_id", + "webhook_error", + "last_synced_sha", + "last_pull_status", + "enabled_by", + ] { + auto_pull.remove(field); + } + } + if let Some(repo) = repo.as_object_mut() { + repo.remove("credential"); + } + } + } + Some(git_sync) +} + +/// What `datatable_permissions` is resolved against. Role names by catalog id, per cluster: the +/// document names a data table role, the stored +/// permissions its catalog id, which differs from one instance to the next. `None` for a catalog +/// that could not be read, as on an edition without data table roles. +#[derive(Default)] +struct DatatableNames { + instance: Option>, + external_instance: Option>, + /// The `u/`, `g/` and `f/` tenants of the desired document that name no user, group or folder + /// of the workspace. The endpoint stores such a tenant as a grant that matches nobody, so the + /// plan is where a typo in a pushed file gets caught. + unknown_tenants: std::collections::BTreeSet, +} + +impl DatatableNames { + async fn load(db: &DB, w_id: &str, datatable_permissions: &Value) -> Result { + let read = |cluster| async move { + read_role_catalog(db, cluster) + .await + .ok() + .map(|c| c.into_iter().map(|(id, r)| (id, r.name)).collect()) + }; + let tenants = datatable_permissions + .as_object() + .into_iter() + .flat_map(|m| m.values()) + .filter_map(|entry| entry.get("roles")?.as_object()) + .flat_map(|roles| roles.values()) + .filter_map(Value::as_array) + .flatten() + .filter_map(Value::as_str); + let (mut users, mut groups, mut folders) = (vec![], vec![], vec![]); + for tenant in tenants { + match tenant.split_once('/') { + Some(("u", n)) => users.push(n.to_string()), + Some(("g", n)) => groups.push(n.to_string()), + Some(("f", n)) => folders.push(n.to_string()), + _ => {} + } + } + let mut unknown_tenants = std::collections::BTreeSet::new(); + for (prefix, named, query) in [ + ( + "u", + users, + "SELECT username FROM usr WHERE workspace_id = $1 AND username = ANY($2)", + ), + ( + "g", + groups, + "SELECT name FROM group_ WHERE workspace_id = $1 AND name = ANY($2)", + ), + ( + "f", + folders, + "SELECT name FROM folder WHERE workspace_id = $1 AND name = ANY($2)", + ), + ] { + if named.is_empty() { + continue; + } + let existing: std::collections::HashSet = + sqlx::query_scalar::<_, String>(query) + .bind(w_id) + .bind(&named) + .fetch_all(db) + .await? + .into_iter() + .collect(); + unknown_tenants.extend( + named + .into_iter() + .filter(|n| !existing.contains(n)) + .map(|n| format!("{prefix}/{n}")), + ); + } + Ok(Self { + instance: read(DatatableRoleCluster::Instance).await, + external_instance: read(DatatableRoleCluster::ExternalInstance).await, + unknown_tenants, + }) + } + + fn of(&self, cluster: DatatableRoleCluster) -> Option<&BTreeMap> { + match cluster { + DatatableRoleCluster::Instance => self.instance.as_ref(), + DatatableRoleCluster::ExternalInstance => self.external_instance.as_ref(), + } + } +} + +/// The cluster whose roles a data table entry can be put under: only an entry that owns a +/// database on a cluster Windmill manages, and is not a clone, which its origin governs. +fn role_cluster_of(entry: &Value) -> Option { + let entry: DataTable = serde_json::from_value(entry.clone()).ok()?; + if entry.governed_by.is_some() { + return None; + } + DatatableRoleCluster::of(entry.database?.resource_type) +} + +/// Each data table's stored `permissions`, with roles named rather than given by catalog id. A +/// role the catalog no longer has keeps its id, so the document still shows it. +fn datatable_permissions_document(datatable: Option<&Value>, roles: &DatatableNames) -> Value { + let mut doc = Map::new(); + let entries = datatable + .and_then(|d| d.get("datatables")) + .and_then(Value::as_object); + for (name, entry) in entries.into_iter().flatten() { + let Some(permissions) = entry + .get("permissions") + .filter(|p| !p.is_null()) + .and_then(|p| serde_json::from_value::(p.clone()).ok()) + else { + continue; + }; + let names = role_cluster_of(entry).and_then(|c| roles.of(c)); + let name_of = |id: &str| { + names + .and_then(|n| n.get(id)) + .cloned() + .unwrap_or_else(|| id.to_string()) + }; + let default_role = match permissions.default_role() { + NO_DEFAULT_DATATABLE_ROLE => Value::Null, + id => Value::String(name_of(id)), + }; + let roles: Map = permissions + .roles + .iter() + .map(|(id, t)| (name_of(id), json!(t.tenants))) + .collect(); + doc.insert( + name.clone(), + json!({ "default_role": default_role, "roles": roles }), + ); + } + Value::Object(doc) +} + +/// Protection rules by name, their rules in a fixed order so a reordered file is no change. The +/// dev workspace lock is left out: the dev workspace setting creates and removes it. +async fn protection_rules_document(conn: &mut PgConnection, w_id: &str) -> Result { + let rows = sqlx::query_as::<_, (String, i32, Vec, Vec)>( + "SELECT name, rules, bypass_groups, bypass_users FROM workspace_protection_rule + WHERE workspace_id = $1 AND name <> $2 ORDER BY name", + ) + .bind(w_id) + .bind(DEV_WORKSPACE_LOCK_RULE_NAME) + .fetch_all(&mut *conn) + .await?; + Ok(Value::Object( + rows.into_iter() + .map(|(name, bits, bypass_groups, bypass_users)| { + let rules = ProtectionRules::from_bits_truncate(bits); + let rule = json!({ + "rules": rule_kinds(rules), + "bypass_groups": bypass_groups, + "bypass_users": bypass_users, + }); + (name, rule) + }) + .collect(), + )) +} + +fn rule_kinds(rules: ProtectionRules) -> Value { + json!(rules.kinds()) +} + +/// Reads no permission itself: the caller must have checked that the requester is an admin of +/// `w_id`, since `SecretMode::Plain` returns the workspace's secrets in plain text. +pub async fn load_settings_document( + conn: &mut PgConnection, + db: &DB, + w_id: &str, + secrets: SecretMode, +) -> Result> { + let row = sqlx::query_as::<_, SettingsRow>( + r#"SELECT + auto_invite, webhook, error_handler, success_handler, ai_config, large_file_storage, + git_sync, default_app, default_scripts, workspace.name AS name, mute_critical_alerts, + color, operator_settings, datatable, slack_team_id, slack_name, slack_command_script, + slack_oauth_client_id, slack_oauth_client_secret, ducklake, dbt_warehouses, deploy_ui, + teams_command_script, public_app_execution_limit_per_minute, guest_access_enabled, + guest_jwt_public_key, guest_jwt_jwks_url, error_handler_fallback_to_instance_alerts, + add_admins_and_developers_to_forks + FROM workspace_settings + LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id + WHERE workspace_id = $1"#, + ) + .bind(w_id) + .fetch_one(&mut *conn) + .await?; + + let slack_oauth_client_secret = match (row.slack_oauth_client_secret, secrets) { + (Some(s), SecretMode::Redacted) if !s.is_empty() => Some(SENSITIVE_UNCHANGED.to_string()), + (Some(s), SecretMode::Encrypted) if !s.is_empty() => { + let mc = build_crypt(db, w_id).await?; + Some(format!( + "{ENCRYPTED_PREFIX}{}", + mc.encrypt_str_deterministic(&s) + )) + } + (s, _) => s, + }; + + let has_permissions = row + .datatable + .as_ref() + .and_then(|d| d.get("datatables")) + .and_then(Value::as_object) + .is_some_and(|e| { + e.values() + .any(|e| e.get("permissions").is_some_and(|p| !p.is_null())) + }); + let role_names = if has_permissions { + DatatableNames::load(db, w_id, &Value::Null).await? + } else { + DatatableNames::default() + }; + let datatable_permissions = datatable_permissions_document(row.datatable.as_ref(), &role_names); + let protection_rules = protection_rules_document(&mut *conn, w_id).await?; + + // In the order `wmill sync pull` has always written settings.yaml. A key that predates the + // document and is not marked `always` is written only when set, as it always was; a key + // added since is always written, so a pulled file states it and from then on manages it. + let entries: [(&str, Option, bool); 31] = [ + ("auto_invite", row.auto_invite, false), + ("webhook", row.webhook.map(Value::String), false), + ("error_handler", row.error_handler, true), + ("success_handler", row.success_handler, true), + ("ai_config", row.ai_config, false), + ("large_file_storage", row.large_file_storage, false), + ("git_sync", redact_git_sync(row.git_sync), false), + ("default_app", row.default_app.map(Value::String), false), + ("default_scripts", row.default_scripts, false), + ("name", Some(json!(row.name.unwrap_or_default())), true), + ( + "mute_critical_alerts", + row.mute_critical_alerts.map(Value::Bool), + false, + ), + ("color", row.color.map(Value::String), false), + ("operator_settings", row.operator_settings, false), + ( + "datatable", + windmill_common::workspaces::strip_datatable_permissions(row.datatable), + false, + ), + ("slack_team_id", row.slack_team_id.map(Value::String), false), + ("slack_name", row.slack_name.map(Value::String), false), + ( + "slack_command_script", + row.slack_command_script.map(Value::String), + false, + ), + ( + "slack_oauth_client_id", + row.slack_oauth_client_id.map(Value::String), + true, + ), + ( + "slack_oauth_client_secret", + slack_oauth_client_secret.map(Value::String), + true, + ), + ("ducklake", row.ducklake, true), + ("dbt_warehouses", row.dbt_warehouses, true), + ("deploy_ui", row.deploy_ui, true), + ( + "teams_command_script", + row.teams_command_script.map(Value::String), + true, + ), + ( + "public_app_execution_limit_per_minute", + row.public_app_execution_limit_per_minute.map(|n| json!(n)), + true, + ), + ( + "guest_access_enabled", + Some(json!(row.guest_access_enabled.unwrap_or(false))), + true, + ), + ( + "guest_jwt_public_key", + row.guest_jwt_public_key.map(Value::String), + true, + ), + ( + "guest_jwt_jwks_url", + row.guest_jwt_jwks_url.map(Value::String), + true, + ), + ( + "error_handler_fallback_to_instance_alerts", + Some(json!(row + .error_handler_fallback_to_instance_alerts + .unwrap_or(false))), + true, + ), + ( + "add_admins_and_developers_to_forks", + Some(json!(row + .add_admins_and_developers_to_forks + .unwrap_or(false))), + true, + ), + ("datatable_permissions", Some(datatable_permissions), true), + ("protection_rules", Some(protection_rules), true), + ]; + let mut doc = Map::new(); + for (key, value, always) in entries { + if always || value.is_some() { + doc.insert(key.to_string(), value.unwrap_or(Value::Null)); + } + } + Ok(doc) +} + +#[derive(Deserialize)] +pub struct GetSettingsDocumentQuery { + #[serde(default)] + secrets: SecretMode, +} + +pub async fn get_settings_document( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(q): Query, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + if q.secrets == SecretMode::Plain { + return Err(Error::BadRequest( + "Plain secrets are only exported by the workspace archive".to_string(), + )); + } + let mut conn = db.acquire().await?; + Ok(Json( + load_settings_document(&mut conn, &db, &w_id, q.secrets).await?, + )) +} + +#[derive(Deserialize)] +pub struct ApplySettingsDocument { + document: Map, + /// Clear the settings the document omits that `wmill sync push` has always cleared when + /// settings.yaml omits them. Otherwise a setting the document omits is left as it is. + #[serde(default)] + replace: bool, + #[serde(default)] + dry_run: bool, +} + +#[derive(Serialize, Debug)] +pub struct SettingsChange { + key: String, + before: Value, + after: Value, +} + +#[derive(Serialize, Debug, Default)] +pub struct ApplySettingsDocumentResult { + changes: Vec, + applied: bool, + #[serde(skip_serializing_if = "Vec::is_empty")] + warnings: Vec, +} + +/// Where one planned write goes: a settings endpoint of the workspaces crate, or one of the few +/// that live in this crate. +#[derive(Debug)] +enum Target { + Edit(SettingsEdit), + AiConfig, + MuteCriticalAlerts, + TeamsCommand, +} + +#[derive(Debug)] +struct Write { + target: Target, + body: Value, + /// The document keys this write applies, for error messages. + keys: Vec, +} + +fn is_null(v: Option<&Value>) -> bool { + v.is_none_or(Value::is_null) +} + +/// The endpoints drop a null member rather than store it, so a document stating one would +/// otherwise differ from the stored value on every apply. A handler's `extra_args` is stored +/// verbatim and its nulls are passed to the job as arguments, so it is compared as is. +fn without_null_members(v: &Value) -> Value { + match v { + Value::Object(m) => Value::Object( + m.iter() + .filter(|(_, v)| !v.is_null()) + .map(|(k, v)| { + let v = if k == "extra_args" { + v.clone() + } else { + without_null_members(v) + }; + (k.clone(), v) + }) + .collect(), + ), + Value::Array(a) => Value::Array(a.iter().map(without_null_members).collect()), + v => v.clone(), + } +} + +fn same(a: Option<&Value>, b: Option<&Value>) -> bool { + match (a, b) { + (a, b) if is_null(a) && is_null(b) => true, + (Some(a), Some(b)) => without_null_members(a) == without_null_members(b), + _ => false, + } +} + +/// Switches the endpoint stores as `false` when cleared, compared as `false` when unset. +const FALSE_WHEN_UNSET: &[&str] = &[ + "mute_critical_alerts", + "guest_access_enabled", + "add_admins_and_developers_to_forks", + "error_handler_fallback_to_instance_alerts", +]; + +fn null_as_false(v: Option) -> Option { + v.map(|v| if v.is_null() { Value::Bool(false) } else { v }) +} + +fn redact(key: &str, v: Value) -> Value { + if SECRET_KEYS.contains(&key) && !v.is_null() { + json!(SENSITIVE_UNCHANGED) + } else { + v + } +} + +/// The auto-invite object without the instance group keys, which a separate endpoint owns. +fn domain_invite(v: Option<&Value>) -> Value { + let mut m = v.and_then(Value::as_object).cloned().unwrap_or_default(); + m.remove("instance_groups"); + m.remove("instance_groups_roles"); + Value::Object(m) +} + +struct Plan { + current: Map, + desired: Map, + replace: bool, + changes: Vec, + writes: Vec, + warnings: Vec, +} + +impl Plan { + /// The document's value for a key that predates the document: under `replace`, an omitted + /// one counts as `null`, which clears it. + fn legacy(&self, key: &str) -> Option { + match self.desired.get(key) { + Some(v) => Some(v.clone()), + None if self.replace => Some(Value::Null), + None => None, + } + } + + /// The document's value for a key that is only managed when present. + fn managed(&self, key: &str) -> Option { + self.desired.get(key).cloned() + } + + fn differs(&self, key: &str, desired: &Value) -> bool { + let current = self.current.get(key).cloned(); + let current = if FALSE_WHEN_UNSET.contains(&key) { + null_as_false(Some(current.unwrap_or(Value::Null))) + } else { + current + }; + !same(current.as_ref(), Some(desired)) + } + + fn record(&mut self, key: &str, after: Value) { + let before = self.current.get(key).cloned().unwrap_or(Value::Null); + self.record_change(key, before, after); + } + + fn record_change(&mut self, key: &str, before: Value, after: Value) { + self.changes.push(SettingsChange { + key: key.to_string(), + before: redact(key, before), + after: redact(key, after), + }); + } + + /// A write applying the keys recorded since the previous one. + fn write(&mut self, target: Target, body: Value) { + let from = self.writes.iter().map(|w| w.keys.len()).sum::(); + let keys = self.changes[from..].iter().map(|c| c.key.clone()).collect(); + self.writes.push(Write { target, body, keys }); + } + + /// The common case: one key, one endpoint, a body built from the desired value. + fn simple( + &mut self, + key: &str, + desired: Option, + target: Target, + body: impl FnOnce(Value) -> Value, + ) { + if let Some(desired) = desired { + if self.differs(key, &desired) { + self.record(key, desired.clone()); + self.write(target, body(desired)); + } + } + } +} + +/// The plain value of a secret the document states, or `None` to keep the stored one: for the +/// placeholder, and for a value encrypted under another workspace's key, which a repo synced to +/// several workspaces carries. That one is reported rather than failing every other setting. +fn resolve_secret( + key: &str, + v: Value, + mc: &windmill_common::variables::WorkspaceCrypt, + warnings: &mut Vec, +) -> Option { + match v.as_str() { + Some(SENSITIVE_UNCHANGED) => None, + Some(s) if s.starts_with(ENCRYPTED_PREFIX) => { + match mc.decrypt_str(&s[ENCRYPTED_PREFIX.len()..]) { + Ok(plain) => Some(Value::String(plain)), + Err(_) => { + warnings.push(format!( + "{key} was left unchanged: it is encrypted with another workspace's key. \ + Give this workspace its own settings file, export secrets in plain text \ + (plainSecrets), or share the workspace key (includeKey)." + )); + None + } + } + } + _ => Some(v), + } +} + +const KNOWN_KEYS: &[&str] = &[ + "auto_invite", + "webhook", + "error_handler", + "success_handler", + "ai_config", + "large_file_storage", + "git_sync", + "default_app", + "default_scripts", + "name", + "mute_critical_alerts", + "color", + "operator_settings", + "datatable", + "slack_team_id", + "slack_name", + "slack_command_script", + "slack_oauth_client_id", + "slack_oauth_client_secret", + "ducklake", + "dbt_warehouses", + "deploy_ui", + "teams_command_script", + "public_app_execution_limit_per_minute", + "guest_access_enabled", + "guest_jwt_public_key", + "guest_jwt_jwks_url", + "error_handler_fallback_to_instance_alerts", + "add_admins_and_developers_to_forks", + "datatable_permissions", + "protection_rules", +]; + +fn string_list(v: Option<&Value>, what: &str) -> Result { + match v { + None | Some(Value::Null) => Ok(json!([])), + Some(Value::Array(a)) if a.iter().all(Value::is_string) => Ok(Value::Array(a.clone())), + _ => Err(Error::BadRequest(format!( + "{what} must be a list of strings" + ))), + } +} + +/// `datatable_permissions` as the endpoint stores it: an omitted default role is `admin`, and +/// `admin` is always one of the roles. A data table stated as null is not under roles, as an +/// omitted one is. A data table's error goes to `errors` and leaves it out of the result, so every +/// data table is checked before the document is refused. +fn normalize_datatable_permissions(v: &Value, errors: &mut Vec) -> Result { + let entries = match v { + Value::Null => return Ok(json!({})), + Value::Object(m) => m, + _ => { + return Err(Error::BadRequest( + "datatable_permissions must map data tables to their roles".to_string(), + )) + } + }; + let mut out = Map::new(); + for (name, entry) in entries { + if entry.is_null() { + continue; + } + match normalize_datatable_entry(name, entry) { + Ok(entry) => { + out.insert(name.clone(), entry); + } + Err(e) => errors.push(error_message(e)), + } + } + Ok(Value::Object(out)) +} + +fn normalize_datatable_entry(name: &str, entry: &Value) -> Result { + let what = format!("datatable_permissions.{name}"); + let entry = entry + .as_object() + .ok_or_else(|| Error::BadRequest(format!("{what} must be an object or null")))?; + let default_role = match entry.get("default_role") { + None => json!(ADMIN_DATATABLE_ROLE), + Some(r) if r.is_null() || r.is_string() => r.clone(), + Some(_) => { + return Err(Error::BadRequest(format!( + "{what}.default_role must be a role name or null" + ))) + } + }; + let mut roles = Map::new(); + match entry.get("roles") { + None | Some(Value::Null) => {} + Some(Value::Object(r)) => { + for (role, tenants) in r { + let tenants = string_list(Some(tenants), &format!("{what}.roles.{role}"))?; + // The endpoint's own check, here so a dry run refuses it too. + for tenant in tenants.as_array().into_iter().flatten() { + let tenant = tenant.as_str().unwrap_or_default(); + let valid = tenant == DATATABLE_TENANT_WILDCARD + || matches!(tenant.split_once('/'), + Some(("u" | "g" | "f", rest)) if !rest.is_empty()); + if !valid { + return Err(Error::BadRequest(format!( + "Invalid tenant '{tenant}' in {what}.roles.{role}: expected \ + 'u/', 'g/', 'f/' or '*'" + ))); + } + } + roles.insert(role.clone(), tenants); + } + } + Some(_) => { + return Err(Error::BadRequest(format!( + "{what}.roles must map role names to their tenants" + ))) + } + } + roles + .entry(ADMIN_DATATABLE_ROLE.to_string()) + .or_insert(json!([])); + if let Some(role) = default_role.as_str() { + if !roles.contains_key(role) { + return Err(Error::BadRequest(format!( + "The default role '{role}' of {what} is not among its roles" + ))); + } + } + Ok(json!({ "default_role": default_role, "roles": roles })) +} + +fn error_message(e: Error) -> String { + match e { + Error::BadRequest(msg) => msg, + e => e.to_string(), + } +} + +/// Refuses the document with every data table's error at once. +fn datatable_permissions_errors(errors: Vec) -> Result<()> { + match errors.len() { + 0 => Ok(()), + 1 => Err(Error::BadRequest(errors.into_iter().next().unwrap())), + _ => Err(Error::BadRequest(format!( + "Invalid datatable_permissions, nothing was applied:\n- {}", + errors.join("\n- ") + ))), + } +} + +/// `protection_rules` as it is stored: rules in their fixed order, the bypass lists stated. The +/// dev workspace lock is dropped with a warning, since the dev workspace setting owns it. +fn normalize_protection_rules(v: &Value, warnings: &mut Vec) -> Result { + let rules = match v { + Value::Null => return Ok(json!({})), + Value::Object(m) => m, + _ => { + return Err(Error::BadRequest( + "protection_rules must map rule names to rules".to_string(), + )) + } + }; + let mut out = Map::new(); + for (name, rule) in rules { + if name == DEV_WORKSPACE_LOCK_RULE_NAME { + warnings.push(format!( + "protection_rules.{name} was left unchanged: the dev workspace setting manages it" + )); + continue; + } + if rule.is_null() { + continue; + } + let what = format!("protection_rules.{name}"); + let rule = rule + .as_object() + .ok_or_else(|| Error::BadRequest(format!("{what} must be an object or null")))?; + let kinds: Vec = + serde_json::from_value(rule.get("rules").cloned().unwrap_or(json!([]))) + .map_err(|e| Error::BadRequest(format!("Invalid {what}.rules: {e}")))?; + out.insert( + name.clone(), + json!({ + "rules": rule_kinds(ProtectionRules::from(&kinds)), + "bypass_groups": string_list(rule.get("bypass_groups"), &format!("{what}.bypass_groups"))?, + "bypass_users": string_list(rule.get("bypass_users"), &format!("{what}.bypass_users"))?, + }), + ); + } + Ok(Value::Object(out)) +} + +async fn plan(db: &DB, w_id: &str, desired: Map, replace: bool) -> Result { + let current = { + let mut conn = db.acquire().await?; + load_settings_document(&mut conn, db, w_id, SecretMode::Plain).await? + }; + let mc = build_crypt(db, w_id).await?; + let names = match desired.get("datatable_permissions") { + Some(v) if !v.is_null() => DatatableNames::load(db, w_id, v).await?, + _ => DatatableNames::default(), + }; + plan_document(current, desired, replace, &mc, &names) +} + +fn plan_document( + current: Map, + mut desired: Map, + replace: bool, + mc: &windmill_common::variables::WorkspaceCrypt, + names: &DatatableNames, +) -> Result { + let mut warnings = vec![]; + let unknown: Vec<&str> = desired + .keys() + .map(String::as_str) + .filter(|k| !KNOWN_KEYS.contains(k)) + .collect(); + if !unknown.is_empty() { + warnings.push(format!("Ignored unknown settings: {}", unknown.join(", "))); + } + for key in ["auto_invite", "error_handler", "success_handler"] { + if desired + .get(key) + .is_some_and(|v| !v.is_null() && !v.is_object()) + { + return Err(Error::BadRequest(format!( + "{key} must be an object or null" + ))); + } + } + // The bodies built below coerce a mistyped value to `false` or to a delete, so a wrong type + // must be refused here rather than reach an endpoint as a destructive write. + let typed = |key: &str, v: Option<&Value>, ok: fn(&Value) -> bool, ty: &str| match v { + Some(v) if !v.is_null() && !ok(v) => { + Err(Error::BadRequest(format!("{key} must be {ty} or null"))) + } + _ => Ok(()), + }; + for key in FALSE_WHEN_UNSET { + typed(key, desired.get(*key), Value::is_boolean, "a boolean")?; + } + // The rest are refused by their endpoint anyway, but only after the writes before them + // went through, and after a dry run reported them as changes. + for key in [ + "slack_oauth_client_id", + "slack_oauth_client_secret", + "webhook", + "color", + "default_app", + "slack_command_script", + "teams_command_script", + "guest_jwt_public_key", + "guest_jwt_jwks_url", + ] { + typed(key, desired.get(key), Value::is_string, "a string")?; + } + typed( + "public_app_execution_limit_per_minute", + desired.get("public_app_execution_limit_per_minute"), + |v| v.as_i64().is_some_and(|n| i32::try_from(n).is_ok()), + "a 32-bit integer", + )?; + for (parent, key, ok, ty) in [ + ( + "auto_invite", + "enabled", + Value::is_boolean as fn(&Value) -> bool, + "a boolean", + ), + ("auto_invite", "operator", Value::is_boolean, "a boolean"), + ("auto_invite", "mode", Value::is_string, "a string"), + ( + "error_handler", + "muted_on_cancel", + Value::is_boolean, + "a boolean", + ), + ( + "error_handler", + "muted_on_user_path", + Value::is_boolean, + "a boolean", + ), + ] { + let v = desired.get(parent).and_then(|p| p.get(key)); + typed(&format!("{parent}.{key}"), v, ok, ty)?; + } + let mut datatable_errors = vec![]; + if let Some(v) = desired.get("datatable_permissions") { + let v = normalize_datatable_permissions(v, &mut datatable_errors)?; + desired.insert("datatable_permissions".to_string(), v); + } + if let Some(v) = desired.get("protection_rules") { + let v = normalize_protection_rules(v, &mut warnings)?; + desired.insert("protection_rules".to_string(), v); + } + // The endpoint stores an omitted mute flag as `false`. + if let Some(handler) = desired + .get_mut("error_handler") + .and_then(Value::as_object_mut) + { + for key in ["muted_on_cancel", "muted_on_user_path"] { + if is_null(handler.get(key)) { + handler.insert(key.to_string(), Value::Bool(false)); + } + } + } + for key in SECRET_KEYS { + if let Some(v) = desired.remove(*key) { + if let Some(v) = resolve_secret(key, v, mc, &mut warnings) { + desired.insert(key.to_string(), v); + } else if let Some(cur) = current.get(*key) { + desired.insert(key.to_string(), cur.clone()); + } + } + } + + let stated: Vec = desired.keys().cloned().collect(); + let mut p = Plan { current, desired, replace, changes: vec![], writes: vec![], warnings }; + + let webhook = p.legacy("webhook"); + p.simple( + "webhook", + webhook, + Target::Edit(SettingsEdit::Webhook), + |v| json!({ "webhook": v }), + ); + + if let Some(auto_invite) = p.legacy("auto_invite") { + let desired_domain = domain_invite(Some(&auto_invite)); + if desired_domain != domain_invite(p.current.get("auto_invite")) { + if let Some(mode) = desired_domain.get("mode").and_then(Value::as_str) { + if mode != "invite" && mode != "add" { + return Err(Error::BadRequest( + r#"Invalid value for auto_invite.mode. Valid values are "invite" and "add""# + .to_string(), + )); + } + } + let mut after = if auto_invite.is_null() { + Value::Null + } else { + desired_domain.clone() + }; + if let (Some(after), Some(cur)) = ( + after.as_object_mut(), + p.current.get("auto_invite").and_then(Value::as_object), + ) { + for k in ["instance_groups", "instance_groups_roles"] { + if let Some(v) = cur.get(k) { + after.insert(k.to_string(), v.clone()); + } + } + } + p.record("auto_invite", after); + let enabled = desired_domain.get("enabled").and_then(Value::as_bool) == Some(true); + let body = if enabled { + json!({ + "operator": desired_domain.get("operator").and_then(Value::as_bool).unwrap_or(false), + "invite_all": true, + "auto_add": desired_domain.get("mode").and_then(Value::as_str) == Some("add"), + }) + } else { + json!({}) + }; + p.write(Target::Edit(SettingsEdit::AutoInvite), body); + } + // Clearing an instance group removes the members it granted, so only a document that + // states the groups manages them. + if let Some(groups) = auto_invite.get("instance_groups").filter(|g| !g.is_null()) { + let roles = auto_invite + .get("instance_groups_roles") + .filter(|r| !r.is_null()) + .cloned() + .unwrap_or(json!({})); + let cur = p.current.get("auto_invite"); + let cur_groups = cur.and_then(|c| c.get("instance_groups")); + let cur_roles = cur + .and_then(|c| c.get("instance_groups_roles")) + .filter(|r| !r.is_null()) + .cloned() + .unwrap_or(json!({})); + if !same(Some(groups), cur_groups) || roles != cur_roles { + p.record( + "auto_invite.instance_groups", + json!({ "instance_groups": groups, "instance_groups_roles": roles }), + ); + p.write( + Target::Edit(SettingsEdit::InstanceGroups), + json!({ "groups": groups, "roles": roles }), + ); + } + } + } + + let ai_config = p.legacy("ai_config"); + p.simple("ai_config", ai_config, Target::AiConfig, |v| { + if v.is_null() { + json!({}) + } else { + v + } + }); + + let fallback = null_as_false(p.managed("error_handler_fallback_to_instance_alerts")); + let fallback_changes = fallback + .as_ref() + .is_some_and(|f| p.differs("error_handler_fallback_to_instance_alerts", f)); + let error_handler = p.legacy("error_handler"); + let error_handler_changes = error_handler + .as_ref() + .is_some_and(|e| p.differs("error_handler", e)); + if error_handler_changes || fallback_changes { + let handler = if error_handler_changes { + error_handler.clone().unwrap_or(Value::Null) + } else { + p.current + .get("error_handler") + .cloned() + .unwrap_or(Value::Null) + }; + if error_handler_changes { + p.record("error_handler", handler.clone()); + } + if fallback_changes { + p.record( + "error_handler_fallback_to_instance_alerts", + fallback.clone().unwrap_or(Value::Null), + ); + } + p.write( + Target::Edit(SettingsEdit::ErrorHandler), + json!({ + "path": handler.get("path"), + "extra_args": handler.get("extra_args"), + "muted_on_cancel": handler.get("muted_on_cancel").and_then(Value::as_bool).unwrap_or(false), + "muted_on_user_path": handler.get("muted_on_user_path").and_then(Value::as_bool).unwrap_or(false), + "fallback_to_instance_alerts": if fallback_changes { fallback.and_then(|f| f.as_bool()) } else { None }, + }), + ); + } + + let success_handler = p.legacy("success_handler"); + p.simple( + "success_handler", + success_handler, + Target::Edit(SettingsEdit::SuccessHandler), + |v| json!({ "path": v.get("path"), "extra_args": v.get("extra_args") }), + ); + + let lfs = p.legacy("large_file_storage"); + p.simple( + "large_file_storage", + lfs, + Target::Edit(SettingsEdit::LargeFileStorage), + |v| json!({ "large_file_storage": v }), + ); + + let git_sync = p.legacy("git_sync"); + p.simple( + "git_sync", + git_sync, + Target::Edit(SettingsEdit::GitSync), + |v| json!({ "git_sync_settings": v }), + ); + + let default_scripts = p.legacy("default_scripts"); + p.simple( + "default_scripts", + default_scripts, + Target::Edit(SettingsEdit::DefaultScripts), + |v| v, + ); + + let default_app = p.legacy("default_app"); + p.simple( + "default_app", + default_app, + Target::Edit(SettingsEdit::DefaultApp), + |v| json!({ "default_app_path": v }), + ); + + // The display name is never applied: settings.yaml is shared across a repo's branches, so + // applying it would let one workspace's name overwrite another's. + + let mute = null_as_false(p.legacy("mute_critical_alerts")); + p.simple( + "mute_critical_alerts", + mute, + Target::MuteCriticalAlerts, + |v| json!({ "mute_critical_alerts": v }), + ); + + // `sync pull` omits the key for a workspace without a color, so an absent or null color + // means "not managed", never "clear". + let color = p.managed("color").filter(|c| !c.is_null()); + p.simple( + "color", + color, + Target::Edit(SettingsEdit::Color), + |v| json!({ "color": v }), + ); + + if let Some(desired) = p.managed("operator_settings").filter(|o| !o.is_null()) { + // The endpoint keeps the stored value of a write right the document omits, so an + // older document must not count as differing on those keys. + let mut compared = desired.clone(); + if let (Some(compared), Some(cur)) = ( + compared.as_object_mut(), + p.current + .get("operator_settings") + .and_then(Value::as_object), + ) { + for key in [ + "builder_flows", + "builder_apps", + "manage_schedules", + "manage_triggers", + ] { + if !compared.contains_key(key) { + if let Some(v) = cur.get(key) { + compared.insert(key.to_string(), v.clone()); + } + } + } + } + if p.differs("operator_settings", &compared) { + p.record("operator_settings", compared); + p.write(Target::Edit(SettingsEdit::OperatorSettings), desired); + } + } + + let datatable = p.legacy("datatable"); + p.simple( + "datatable", + datatable, + Target::Edit(SettingsEdit::Datatable), + |v| json!({ "settings": if v.is_null() { json!({ "datatables": {} }) } else { v } }), + ); + + // Stated as a whole: a data table the map omits is not under roles, a rule it omits is + // deleted. + if let Some(desired) = p.managed("datatable_permissions") { + plan_datatable_permissions(&mut p, desired, names, &mut datatable_errors); + } + datatable_permissions_errors(datatable_errors)?; + if let Some(desired) = p.managed("protection_rules") { + plan_protection_rules(&mut p, desired); + } + + let slack_command = p.legacy("slack_command_script"); + p.simple( + "slack_command_script", + slack_command, + Target::Edit(SettingsEdit::SlackCommand), + |v| json!({ "slack_command_script": v }), + ); + + let (client_id, client_secret) = ( + p.legacy("slack_oauth_client_id"), + p.legacy("slack_oauth_client_secret"), + ); + if client_id.is_some() || client_secret.is_some() { + let id = client_id.unwrap_or_else(|| { + p.current + .get("slack_oauth_client_id") + .cloned() + .unwrap_or(Value::Null) + }); + let secret = client_secret.unwrap_or_else(|| { + p.current + .get("slack_oauth_client_secret") + .cloned() + .unwrap_or(Value::Null) + }); + let id_changes = p.differs("slack_oauth_client_id", &id); + let secret_changes = p.differs("slack_oauth_client_secret", &secret); + if id_changes || secret_changes { + if id_changes { + p.record("slack_oauth_client_id", id.clone()); + } + if secret_changes { + p.record("slack_oauth_client_secret", secret.clone()); + } + let set = |v: &Value| v.as_str().is_some_and(|s| !s.is_empty()); + if set(&id) && set(&secret) { + p.write( + Target::Edit(SettingsEdit::SetSlackOAuth), + json!({ "slack_oauth_client_id": id, "slack_oauth_client_secret": secret }), + ); + } else { + p.write(Target::Edit(SettingsEdit::DeleteSlackOAuth), Value::Null); + } + } + } + + // Settings added to the document after `wmill sync` shipped are managed only when the + // document states them, so a settings.yaml written before them never clears them. + let ducklake = p.managed("ducklake"); + p.simple( + "ducklake", + ducklake, + Target::Edit(SettingsEdit::Ducklake), + |v| json!({ "settings": if v.is_null() { json!({ "ducklakes": {} }) } else { v } }), + ); + + let dbt = p.managed("dbt_warehouses"); + p.simple( + "dbt_warehouses", + dbt, + Target::Edit(SettingsEdit::DbtWarehouses), + |v| json!({ "dbt_warehouses": v }), + ); + + let deploy_ui = p.managed("deploy_ui"); + p.simple( + "deploy_ui", + deploy_ui, + Target::Edit(SettingsEdit::DeployUi), + |v| json!({ "deploy_ui_settings": v }), + ); + + let teams = p.managed("teams_command_script"); + p.simple( + "teams_command_script", + teams, + Target::TeamsCommand, + |v| json!({ "teams_command_script": v }), + ); + + let rate = p.managed("public_app_execution_limit_per_minute"); + p.simple( + "public_app_execution_limit_per_minute", + rate, + Target::Edit(SettingsEdit::PublicAppRateLimit), + |v| json!({ "public_app_execution_limit_per_minute": v }), + ); + + let guest_access = null_as_false(p.managed("guest_access_enabled")); + p.simple( + "guest_access_enabled", + guest_access, + Target::Edit(SettingsEdit::GuestAccess), + |v| json!({ "guest_access_enabled": v.as_bool().unwrap_or(false) }), + ); + + let (public_key, jwks_url) = ( + p.managed("guest_jwt_public_key"), + p.managed("guest_jwt_jwks_url"), + ); + if public_key.is_some() || jwks_url.is_some() { + // At most one of the two is set, so stating one clears the other. + let other = |stated: &Option, key: &str| { + if stated.as_ref().is_some_and(|v| !v.is_null()) { + Value::Null + } else { + p.current.get(key).cloned().unwrap_or(Value::Null) + } + }; + let (public_key, jwks_url) = ( + public_key + .clone() + .unwrap_or_else(|| other(&jwks_url, "guest_jwt_public_key")), + jwks_url + .clone() + .unwrap_or_else(|| other(&public_key, "guest_jwt_jwks_url")), + ); + let key_changes = p.differs("guest_jwt_public_key", &public_key); + let url_changes = p.differs("guest_jwt_jwks_url", &jwks_url); + if key_changes || url_changes { + if key_changes { + p.record("guest_jwt_public_key", public_key.clone()); + } + if url_changes { + p.record("guest_jwt_jwks_url", jwks_url.clone()); + } + p.write( + Target::Edit(SettingsEdit::GuestJwtKey), + json!({ "public_key": public_key, "jwks_url": jwks_url }), + ); + } + } + + let forks = null_as_false(p.managed("add_admins_and_developers_to_forks")); + p.simple( + "add_admins_and_developers_to_forks", + forks, + Target::Edit(SettingsEdit::AddAdminsAndDevelopersToForks), + |v| json!({ "add_admins_and_developers_to_forks": v.as_bool().unwrap_or(false) }), + ); + + // As `wmill sync push` always did: a document whose stated settings all match changes + // nothing, so a settings.yaml pulled before a setting was turned on in the UI does not + // clear it just for omitting it. + let changes_stated = p.changes.iter().any(|c| { + stated + .iter() + .any(|k| c.key == *k || c.key.starts_with(&format!("{k}."))) + }); + if replace && !changes_stated { + p.changes.clear(); + p.writes.clear(); + } + + // Refused here rather than by the endpoint, so a dry run already says so. + let unavailable: Vec = p + .writes + .iter() + .filter(|w| !available_in_this_edition(&w.target)) + .flat_map(|w| w.keys.clone()) + .collect(); + if !unavailable.is_empty() { + return Err(Error::BadRequest(format!( + "Only available on Windmill Enterprise Edition: {}", + unavailable.join(", ") + ))); + } + + Ok(p) +} + +fn object_of(v: Option<&Value>) -> Map { + v.and_then(Value::as_object).cloned().unwrap_or_default() +} + +fn plan_datatable_permissions( + p: &mut Plan, + desired: Value, + names: &DatatableNames, + errors: &mut Vec, +) { + let desired = object_of(Some(&desired)); + let current = object_of(p.current.get("datatable_permissions")); + // The data tables as they will be once the `datatable` write before this one has gone through. + let entries = object_of( + p.legacy("datatable") + .as_ref() + .or(p.current.get("datatable")) + .and_then(|d| d.get("datatables")), + ); + let tables: std::collections::BTreeSet<&String> = + desired.keys().chain(current.keys()).collect(); + for name in tables { + let (before, after) = (current.get(name), desired.get(name)); + if same(before, after) { + continue; + } + let entry = entries.get(name); + // A removed data table takes its roles with it. + if after.is_none() && entry.is_none() { + continue; + } + let body = match after { + None => json!({ "datatable": name, "permissioned": false }), + Some(after) => match datatable_permissions_write(name, entry, before, after, names) { + Ok(body) => body, + Err(e) => { + errors.extend(e); + continue; + } + }, + }; + p.record_change( + &format!("datatable_permissions.{name}"), + before.cloned().unwrap_or(Value::Null), + after.cloned().unwrap_or(Value::Null), + ); + p.write(Target::Edit(SettingsEdit::DatatablePermissions), body); + } +} + +/// The endpoint body that puts `name` under the roles of `after`, or everything wrong with it. +fn datatable_permissions_write( + name: &str, + entry: Option<&Value>, + before: Option<&Value>, + after: &Value, + names: &DatatableNames, +) -> std::result::Result> { + let Some(cluster) = entry.and_then(role_cluster_of) else { + return Err(vec![format!( + "Data table '{name}' cannot be put under roles: only a data table this workspace \ + governs, on a cluster Windmill manages, can" + )]); + }; + let Some(catalog) = names.of(cluster) else { + return Err(vec![ + "Data table roles are unavailable on this instance".to_string() + ]); + }; + let ids: BTreeMap<&str, &str> = catalog + .iter() + .map(|(id, name)| (name.as_str(), id.as_str())) + .collect(); + let mut errors = vec![]; + let mut id_of = |role: &str| -> String { + if role == ADMIN_DATATABLE_ROLE { + return role.to_string(); + } + ids.get(role).map(|id| id.to_string()).unwrap_or_else(|| { + errors.push(format!( + "Data table '{name}': '{role}' is not a data table role of the {} cluster", + cluster.as_str() + )); + String::new() + }) + }; + let default_role = match after.get("default_role").and_then(Value::as_str) { + None => NO_DEFAULT_DATATABLE_ROLE.to_string(), + Some(role) => id_of(role), + }; + let roles: Vec = object_of(after.get("roles")) + .iter() + .map(|(role, tenants)| json!({ "id": id_of(role), "tenants": tenants })) + .collect(); + // Only a tenant this change grants: one already stored that has since stopped naming anybody + // must not block every later push. + let granted = |v: Option<&Value>| -> std::collections::BTreeSet { + object_of(v.and_then(|v| v.get("roles"))) + .values() + .filter_map(Value::as_array) + .flatten() + .filter_map(|t| Some(t.as_str()?.to_string())) + .collect() + }; + let already = granted(before); + for tenant in granted(Some(after)).difference(&already) { + if names.unknown_tenants.contains(tenant) { + let kind = match tenant.split_once('/') { + Some(("u", _)) => "user", + Some(("g", _)) => "group", + _ => "folder", + }; + errors.push(format!( + "Data table '{name}': tenant '{tenant}' names no {kind} of this workspace" + )); + } + } + if !errors.is_empty() { + return Err(errors); + } + Ok(json!({ + "datatable": name, + "permissioned": true, + "default_role": default_role, + "roles": roles, + })) +} + +fn plan_protection_rules(p: &mut Plan, desired: Value) { + let desired = object_of(Some(&desired)); + let current = object_of(p.current.get("protection_rules")); + let names: std::collections::BTreeSet<&String> = desired.keys().chain(current.keys()).collect(); + for name in names { + let (before, after) = (current.get(name), desired.get(name)); + if same(before, after) { + continue; + } + p.record_change( + &format!("protection_rules.{name}"), + before.cloned().unwrap_or(Value::Null), + after.cloned().unwrap_or(Value::Null), + ); + let rule = |r: &Value, name_key: &str| { + json!({ + name_key: name, + "rules": r.get("rules"), + "bypass_groups": r.get("bypass_groups"), + "bypass_users": r.get("bypass_users"), + }) + }; + match (before, after) { + (None, Some(after)) => p.write( + Target::Edit(SettingsEdit::CreateProtectionRule), + rule(after, "name"), + ), + (Some(_), Some(after)) => p.write( + Target::Edit(SettingsEdit::UpdateProtectionRule), + rule(after, "rule_name"), + ), + _ => p.write( + Target::Edit(SettingsEdit::DeleteProtectionRule), + json!({ "name": name }), + ), + } + } +} + +fn available_in_this_edition(target: &Target) -> bool { + match target { + Target::MuteCriticalAlerts + | Target::Edit(SettingsEdit::DefaultApp) + | Target::Edit(SettingsEdit::DeployUi) => cfg!(feature = "enterprise"), + Target::Edit(SettingsEdit::DatatablePermissions) => { + cfg!(all(feature = "private", feature = "enterprise")) + } + Target::TeamsCommand | Target::Edit(SettingsEdit::InstanceGroups) => { + cfg!(feature = "private") + } + _ => true, + } +} + +async fn execute(authed: &ApiAuthed, db: &DB, w_id: &str, write: Write) -> Result { + let (a, d, p) = ( + authed.clone(), + Extension(db.clone()), + Path(w_id.to_string()), + ); + let body_err = |e: serde_json::Error| { + Error::BadRequest(format!("Invalid value for {:?}: {e}", write.target)) + }; + match write.target { + Target::Edit(SettingsEdit::AutoInvite) => { + match apply_settings_edit( + a.clone(), + db.clone(), + w_id.to_string(), + SettingsEdit::AutoInvite, + write.body.clone(), + ) + .await + { + // Inviting every domain is refused on cloud, where only the workspace's own + // domain can be auto-invited. + Err(_) if write.body.get("invite_all") == Some(&json!(true)) => { + let mut body = write.body; + body["invite_all"] = json!(false); + apply_settings_edit( + a, + db.clone(), + w_id.to_string(), + SettingsEdit::AutoInvite, + body, + ) + .await + } + r => r, + } + } + Target::Edit(edit) => { + apply_settings_edit(a, db.clone(), w_id.to_string(), edit, write.body).await + } + Target::AiConfig => { + let body = serde_json::from_value(write.body).map_err(body_err)?; + let _ = + crate::workspaces::edit_copilot_config(a, d, p, authed.clone(), Json(body)).await?; + Ok(Value::Null) + } + #[cfg(feature = "enterprise")] + Target::MuteCriticalAlerts => { + let body = serde_json::from_value(write.body).map_err(body_err)?; + crate::workspaces::mute_critical_alerts(d, p, a, Json(body)).await?; + Ok(Value::Null) + } + #[cfg(not(feature = "enterprise"))] + Target::MuteCriticalAlerts => Err(crate::workspaces::mute_critical_alerts().await), + #[cfg(feature = "private")] + Target::TeamsCommand => { + let body = serde_json::from_value(write.body).map_err(body_err)?; + crate::teams_ee::edit_teams_command(a, d, p, authed.clone(), Json(body)).await?; + Ok(Value::Null) + } + #[cfg(not(feature = "private"))] + Target::TeamsCommand => { + crate::teams_oss::edit_teams_command().await?; + Ok(Value::Null) + } + } +} + +/// Applies `document` as a list of writes, each through the endpoint that owns the setting, in +/// the order `wmill sync push` has always written them. The writes are not one transaction: a +/// failing one stops the apply, and the error names the settings already written. +pub async fn apply_settings_document( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + let p = plan(&db, &w_id, req.document, req.replace).await?; + let mut result = ApplySettingsDocumentResult { + changes: p.changes, + warnings: p.warnings, + ..Default::default() + }; + if req.dry_run || p.writes.is_empty() { + return Ok(Json(result)); + } + let mut written: Vec = vec![]; + for write in p.writes { + let keys = write.keys.clone(); + let response = execute(&authed, &db, &w_id, write).await.map_err(|e| { + if written.is_empty() { + return e; + } + Error::Generic( + e.status_code(), + format!( + "{e} (settings already applied before this error: {})", + written.join(", ") + ), + ) + })?; + if let Some(stranded) = response + .get("stranded_references") + .and_then(Value::as_array) + .filter(|s| !s.is_empty()) + { + let refs = stranded + .iter() + .filter_map(|r| { + Some(format!( + "{}/{}", + r.get("workspace_id")?.as_str()?, + r.get("datatable")?.as_str()? + )) + }) + .collect::>() + .join(", "); + result.warnings.push(format!( + "The removed data tables governed data tables in other workspaces, which no longer resolve: {refs}. A superadmin can point them somewhere else." + )); + } + written.extend(keys); + } + result.applied = true; + Ok(Json(result)) +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::variables::WorkspaceCrypt; + + fn doc(v: Value) -> Map { + v.as_object().unwrap().clone() + } + + fn keys(p: &Plan) -> Vec<&str> { + p.changes.iter().map(|c| c.key.as_str()).collect() + } + + fn current() -> Map { + doc(json!({ + "name": "ws", + "webhook": "https://a", + "large_file_storage": { "type": "S3Storage", "s3_resource_path": "f/s3" }, + "slack_oauth_client_id": "cid", + "slack_oauth_client_secret": "sekret", + "guest_jwt_public_key": "PEM", + "guest_jwt_jwks_url": null, + "public_app_execution_limit_per_minute": 5, + })) + } + + #[test] + fn replace_clears_an_omitted_setting_only_when_a_stated_one_changes() { + let mc = WorkspaceCrypt::new("k"); + let unchanged = doc(json!({ "name": "other", "webhook": "https://a" })); + let p = plan_document(current(), unchanged, true, &mc, &DatatableNames::default()).unwrap(); + assert!(p.writes.is_empty(), "{:?}", keys(&p)); + + let changed = doc(json!({ "webhook": "https://b" })); + let p = plan_document(current(), changed, true, &mc, &DatatableNames::default()).unwrap(); + assert_eq!( + keys(&p), + [ + "webhook", + "large_file_storage", + "slack_oauth_client_id", + "slack_oauth_client_secret" + ] + ); + assert!(!keys(&p).contains(&"public_app_execution_limit_per_minute")); + } + + #[test] + fn secrets_keep_their_value_for_the_placeholder_and_a_foreign_key() { + let mc = WorkspaceCrypt::new("k"); + let ours = format!( + "{ENCRYPTED_PREFIX}{}", + mc.encrypt_str_deterministic("sekret") + ); + let theirs = format!( + "{ENCRYPTED_PREFIX}{}", + WorkspaceCrypt::new("other").encrypt_str_deterministic("x") + ); + for secret in [SENSITIVE_UNCHANGED.to_string(), ours, theirs.clone()] { + let p = plan_document( + current(), + doc(json!({ "slack_oauth_client_secret": secret })), + false, + &mc, + &DatatableNames::default(), + ) + .unwrap(); + assert!(p.changes.is_empty(), "{secret}: {:?}", keys(&p)); + assert_eq!(p.warnings.len(), usize::from(secret == theirs)); + } + } + + #[test] + fn unset_switches_compare_as_false() { + let mc = WorkspaceCrypt::new("k"); + let d = doc(json!({ "mute_critical_alerts": false, "guest_access_enabled": null })); + assert!( + plan_document(current(), d, false, &mc, &DatatableNames::default()) + .unwrap() + .changes + .is_empty() + ); + } + + #[test] + fn stating_one_guest_key_clears_the_other() { + let mc = WorkspaceCrypt::new("k"); + let d = doc(json!({ "guest_jwt_jwks_url": "https://jwks" })); + let p = plan_document(current(), d, false, &mc, &DatatableNames::default()).unwrap(); + assert_eq!(p.writes.len(), 1); + assert_eq!( + p.writes[0].body, + json!({ "public_key": null, "jwks_url": "https://jwks" }) + ); + } + + #[test] + fn mistyped_values_are_refused_rather_than_coerced() { + let mc = WorkspaceCrypt::new("k"); + for d in [ + json!({ "guest_access_enabled": "true" }), + json!({ "add_admins_and_developers_to_forks": 1 }), + json!({ "slack_oauth_client_id": 123 }), + json!({ "error_handler": { "path": "f/x", "muted_on_cancel": "yes" } }), + json!({ "auto_invite": { "enabled": "true" } }), + json!({ "webhook": 123 }), + json!({ "public_app_execution_limit_per_minute": "5" }), + json!({ "public_app_execution_limit_per_minute": 1.5 }), + json!({ "public_app_execution_limit_per_minute": 2147483648i64 }), + ] { + assert!( + plan_document( + current(), + doc(d.clone()), + false, + &mc, + &DatatableNames::default() + ) + .is_err(), + "{d}" + ); + } + } + + #[test] + fn a_null_handler_argument_is_a_change() { + let mc = WorkspaceCrypt::new("k"); + let mut cur = current(); + cur.insert( + "error_handler".into(), + json!({ "path": "f/x", "extra_args": {}, "muted_on_cancel": false, "muted_on_user_path": false }), + ); + let d = doc(json!({ "error_handler": { "path": "f/x", "extra_args": { "input": null } } })); + let p = plan_document(cur.clone(), d, false, &mc, &DatatableNames::default()).unwrap(); + assert_eq!(keys(&p), ["error_handler"]); + + let d = doc(json!({ "error_handler": { "path": "f/x", "extra_args": {} } })); + let p = plan_document(cur, d, false, &mc, &DatatableNames::default()).unwrap(); + assert!(p.changes.is_empty(), "{:?}", keys(&p)); + } + + #[cfg(all(feature = "private", feature = "enterprise"))] + #[test] + fn datatable_roles_are_named_in_the_document_and_written_by_id() { + let mc = WorkspaceCrypt::new("k"); + let roles = DatatableNames { + instance: Some([("r1".to_string(), "readers".to_string())].into()), + ..Default::default() + }; + let mut cur = current(); + cur.insert( + "datatable".into(), + json!({ "datatables": { "main": { "database": { "resource_type": "instance", "resource_path": "main" } } } }), + ); + cur.insert( + "datatable_permissions".into(), + json!({ "main": { "default_role": "admin", "roles": { "admin": [], "readers": ["g/x"] } } }), + ); + + let d = doc( + json!({ "datatable_permissions": { "main": { "default_role": "readers", "roles": { "readers": ["g/x", "u/bob"] } } } }), + ); + let p = plan_document(cur.clone(), d, false, &mc, &roles).unwrap(); + assert_eq!(keys(&p), ["datatable_permissions.main"]); + assert_eq!( + p.writes[0].body, + json!({ "datatable": "main", "permissioned": true, "default_role": "r1", + "roles": [{ "id": "r1", "tenants": ["g/x", "u/bob"] }, { "id": "admin", "tenants": [] }] }) + ); + + let d = doc(json!({ "datatable_permissions": {} })); + let p = plan_document(cur.clone(), d, false, &mc, &roles).unwrap(); + assert_eq!( + p.writes[0].body, + json!({ "datatable": "main", "permissioned": false }) + ); + + let d = doc(json!({ "datatable_permissions": { "main": { "roles": { "writers": [] } } } })); + assert!(plan_document(cur.clone(), d, false, &mc, &roles).is_err()); + + let d = doc(json!({ "datatable_permissions": { "main": { "default_role": "readers" } } })); + assert!(plan_document(cur.clone(), d, false, &mc, &roles).is_err()); + + // A tenant that names nobody is refused when granted, not when already stored, and every + // data table's errors come back together. + let strict = DatatableNames { + instance: roles.instance.clone(), + unknown_tenants: ["g/x".to_string(), "u/bob".to_string()].into(), + ..Default::default() + }; + let d = doc( + json!({ "datatable_permissions": { "main": { "default_role": "readers", "roles": { "readers": ["g/x"] } } } }), + ); + assert_eq!( + keys(&plan_document(cur.clone(), d, false, &mc, &strict).unwrap()), + ["datatable_permissions.main"] + ); + let d = doc(json!({ "datatable_permissions": { + "main": { "roles": { "readers": ["g/x", "u/bob"], "writers": [] } }, + "other": { "roles": { "readers": ["x/y"] } }, + } })); + let e = plan_document(cur.clone(), d, false, &mc, &strict) + .err() + .unwrap() + .to_string(); + for part in [ + "tenant 'u/bob' names no user", + "'writers' is not a data table role", + "Invalid tenant 'x/y'", + ] { + assert!(e.contains(part), "{e}"); + } + + // Under replace an omitted `datatable` is cleared, taking its tables' roles with it. + let d = doc(json!({ "datatable_permissions": {} })); + let p = plan_document(cur.clone(), d, true, &mc, &roles).unwrap(); + assert!(!p + .writes + .iter() + .any(|w| matches!(w.target, Target::Edit(SettingsEdit::DatatablePermissions)))); + let d = doc(json!({ "datatable_permissions": { "main": {} } })); + assert!(plan_document(cur, d, true, &mc, &roles).is_err()); + } + + #[test] + fn protection_rules_are_diffed_by_name() { + let mc = WorkspaceCrypt::new("k"); + let mut cur = current(); + cur.insert( + "protection_rules".into(), + json!({ "old": { "rules": ["DisableWorkspaceForking"], "bypass_groups": [], "bypass_users": [] } }), + ); + let d = doc(json!({ "protection_rules": { + "new": { "rules": ["RestrictPublicRunSharing", "DisableDirectDeployment"] }, + "dev_workspace_lock": { "rules": [] }, + } })); + let p = plan_document(cur, d, false, &mc, &DatatableNames::default()).unwrap(); + assert_eq!(keys(&p), ["protection_rules.new", "protection_rules.old"]); + assert_eq!( + p.writes[0].body, + json!({ "name": "new", "rules": ["DisableDirectDeployment", "RestrictPublicRunSharing"], + "bypass_groups": [], "bypass_users": [] }) + ); + assert!(matches!( + p.writes[1].target, + Target::Edit(SettingsEdit::DeleteProtectionRule) + )); + assert_eq!(p.warnings.len(), 1); + } +} diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 186cfbc291..e5b5cf3e72 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -61,6 +61,11 @@ pub fn workspaced_service() -> Router { ) .route("/tarball", get(crate::workspaces_export::tarball_workspace)) .route("/edit_copilot_config", post(edit_copilot_config)) + .route( + "/settings_document", + get(crate::workspace_settings_document::get_settings_document) + .post(crate::workspace_settings_document::apply_settings_document), + ) .route("/get_copilot_info", get(get_copilot_info)) .route("/critical_alerts", get(get_critical_alerts)) .route( @@ -85,7 +90,7 @@ pub fn workspaced_service() -> Router { router } -async fn edit_copilot_config( +pub(crate) async fn edit_copilot_config( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, @@ -190,7 +195,7 @@ async fn edit_copilot_config( } #[derive(Serialize)] -struct EditCopilotConfigResponse { +pub(crate) struct EditCopilotConfigResponse { effective_ai_config: AIConfig, has_instance_ai_config: bool, uses_instance_ai_config: bool, @@ -321,7 +326,7 @@ pub struct MuteCriticalAlertRequest { } #[cfg(feature = "enterprise")] -async fn mute_critical_alerts( +pub(crate) async fn mute_critical_alerts( Extension(db): Extension, Path(w_id): Path, authed: ApiAuthed, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 3c3e965248..bff261093a 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -359,6 +359,9 @@ pub(crate) struct ArchiveQueryParams { include_users: Option, include_groups: Option, include_settings: Option, + /// Opt-in, with `settings_version=v2`: state the data tables' roles in settings.json. Their + /// tenants name this workspace's users and groups, and their roles this instance's catalog. + include_datatable_permissions: Option, include_key: Option, include_workspace_dependencies: Option, /// Default `false`: data table migrations ship in the tarball unless the repo @@ -513,49 +516,6 @@ struct SimplifiedGroup { admins: Vec, } -// V2 format: New grouped format -#[derive(Serialize)] -struct SimplifiedSettings { - #[serde(skip_serializing_if = "Option::is_none")] - auto_invite: Option, - #[serde(skip_serializing_if = "Option::is_none")] - webhook: Option, - // Always serialize (including as `null`) so that `wmill sync pull` emits - // these fields in settings.yaml unconditionally. Makes round-trip - // bijective: YAML is the source of truth, absence/null = "clear remote", - // mirroring every other workspace setting. - error_handler: Option, - success_handler: Option, - #[serde(skip_serializing_if = "Option::is_none")] - ai_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - large_file_storage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - git_sync: Option, - #[serde(skip_serializing_if = "Option::is_none")] - default_app: Option, - #[serde(skip_serializing_if = "Option::is_none")] - default_scripts: Option, - name: String, - #[serde(skip_serializing_if = "Option::is_none")] - mute_critical_alerts: Option, - #[serde(skip_serializing_if = "Option::is_none")] - color: Option, - #[serde(skip_serializing_if = "Option::is_none")] - operator_settings: Option, - #[serde(skip_serializing_if = "Option::is_none")] - datatable: Option, - #[serde(skip_serializing_if = "Option::is_none")] - slack_team_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - slack_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - slack_command_script: Option, - // Always serialize (see note above on error_handler / success_handler). - slack_oauth_client_id: Option, - slack_oauth_client_secret: Option, -} - // V1 format: Legacy flat format for backward compatibility (matches main branch exactly) #[derive(Serialize)] struct SimplifiedSettingsLegacy { @@ -602,7 +562,6 @@ struct SettingsRow { auto_invite: Option, webhook: Option, error_handler: Option, - success_handler: Option, ai_config: Option, large_file_storage: Option, git_sync: Option, @@ -616,8 +575,6 @@ struct SettingsRow { slack_team_id: Option, slack_name: Option, slack_command_script: Option, - slack_oauth_client_id: Option, - slack_oauth_client_secret: Option, } pub(crate) async fn tarball_workspace( @@ -638,6 +595,7 @@ pub(crate) async fn tarball_workspace( include_users, include_groups, include_settings, + include_datatable_permissions, include_key, include_workspace_dependencies, skip_datatable_migrations, @@ -1593,7 +1551,6 @@ pub(crate) async fn tarball_workspace( auto_invite, webhook, error_handler, - success_handler, ai_config, large_file_storage, git_sync, @@ -1606,9 +1563,7 @@ pub(crate) async fn tarball_workspace( datatable, slack_team_id, slack_name, - slack_command_script, - slack_oauth_client_id, - slack_oauth_client_secret + slack_command_script FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, @@ -1617,73 +1572,23 @@ pub(crate) async fn tarball_workspace( .fetch_one(&mut *tx) .await?; - // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility - // Server-owned state (the HMAC webhook secret + hook id/error, the - // synced-sha / last-pull status, the admin automatic pulls run as, and what - // the credential check observed) must never leave the server: keep it out of - // export archives and synced repos, and don't let a re-imported workspace - // inherit another install's hook/sync state or pull identity. - fn redact_git_sync_for_export(git_sync: Option) -> Option { - let mut git_sync = git_sync?; - if let Some(repos) = git_sync - .get_mut("repositories") - .and_then(|r| r.as_array_mut()) - { - for repo in repos { - if let Some(auto_pull) = - repo.get_mut("auto_pull").and_then(|a| a.as_object_mut()) - { - for field in [ - "webhook_secret", - "webhook_id", - "webhook_error", - "last_synced_sha", - "last_pull_status", - "enabled_by", - ] { - auto_pull.remove(field); - } - } - // What this install observed about its own credential: a token - // id and expiry, and a `checked_at` that moves on its own. - // None of it describes the workspace, and in a git-synced - // `wmill.yaml` it would churn the file for no reason. - if let Some(repo) = repo.as_object_mut() { - repo.remove("credential"); - } - } - } - Some(git_sync) - } - let settings_str = if settings_version.as_deref() == Some("v2") { - let settings = SimplifiedSettings { - auto_invite: row.auto_invite, - webhook: row.webhook, - error_handler: row.error_handler, - success_handler: row.success_handler, - ai_config: row.ai_config, - large_file_storage: row.large_file_storage, - git_sync: redact_git_sync_for_export(row.git_sync), - default_app: row.default_app, - default_scripts: row.default_scripts, - name: row.name.clone().unwrap_or_default(), - mute_critical_alerts: row.mute_critical_alerts, - color: row.color.clone(), - operator_settings: row.operator_settings.clone(), - datatable: windmill_common::workspaces::strip_datatable_permissions( - row.datatable.clone(), - ), - slack_team_id: row.slack_team_id.clone(), - slack_name: row.slack_name.clone(), - slack_command_script: row.slack_command_script.clone(), - slack_oauth_client_id: row.slack_oauth_client_id.clone(), - slack_oauth_client_secret: row.slack_oauth_client_secret.clone(), + let secrets = if export_plain_secrets { + crate::workspace_settings_document::SecretMode::Plain + } else { + crate::workspace_settings_document::SecretMode::Encrypted }; - serde_json::to_value(settings) - .map(|v| serde_json::to_string_pretty(&v).ok()) - .ok() - .flatten() + let mut doc = crate::workspace_settings_document::load_settings_document( + &mut *tx, &db, &w_id, secrets, + ) + .await?; + // A repo syncs its rules through protection-rules.yaml, so settings.yaml leaving + // them out keeps one writer for them. + doc.remove("protection_rules"); + if !include_datatable_permissions.unwrap_or(false) { + doc.remove("datatable_permissions"); + } + serde_json::to_string_pretty(&doc).ok() } else { // V1 (legacy) format: convert JSONB to flat fields (matches main branch exactly) let (auto_invite_enabled, auto_invite_as, auto_invite_mode) = @@ -1730,7 +1635,7 @@ pub(crate) async fn tarball_workspace( error_handler_muted_on_cancel, ai_config: row.ai_config, large_file_storage: row.large_file_storage, - git_sync: redact_git_sync_for_export(row.git_sync), + git_sync: crate::workspace_settings_document::redact_git_sync(row.git_sync), default_app: row.default_app, default_scripts: row.default_scripts, name: row.name.unwrap_or_default(), diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index 1686c20d2b..4e25cec17d 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -71,6 +71,24 @@ pub fn app_value_runnable_paths(value: &Value) -> error::Result( + key: &str, + value: &'a Value, + container_id: Option<&String>, +) -> Option<&'a serde_json::Map> { + match value { + Value::Object(runnables) if container_id.is_none() && key == RAW_APP_RUNNABLES => { + Some(runnables) + } + _ => None, + } +} + /// Traverse FlowValue while invoking provided by caller callback on leafs // #[async_recursion::async_recursion(?Send)] pub fn traverse_app_inline_scripts< @@ -109,7 +127,15 @@ pub fn traverse_app_inline_scripts< )?; } } else { - for (_, value) in object { + for (key, value) in object { + if let Some(runnables) = + raw_app_runnables_map(key, value, container_id.as_ref()) + { + for (id, runnable) in runnables { + traverse_app_inline_scripts(runnable, Some(id.clone()), cb)?; + } + continue; + } traverse_app_inline_scripts( value, object diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index a54e9eb399..dd8ec49f92 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -289,9 +289,9 @@ pub fn error_source_chain(e: &dyn std::error::Error) -> String { msg } -impl IntoResponse for Error { - fn into_response(self) -> axum::response::Response { - let status = match self { +impl Error { + pub fn status_code(&self) -> axum::http::StatusCode { + match self { Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND, Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED, Self::RequireAdmin(_) | Self::PermissionDenied(_) => axum::http::StatusCode::FORBIDDEN, @@ -301,9 +301,15 @@ impl IntoResponse for Error { | Self::JsonErr(_) | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST, Self::BadGateway(_) => axum::http::StatusCode::BAD_GATEWAY, - Self::Generic(status_code, _) => status_code, + Self::Generic(status_code, _) => *status_code, _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, - }; + } + } +} + +impl IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let status = self.status_code(); let e = &self; diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index ebd0ce363f..cd7d6e0959 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -18,6 +18,7 @@ pub use windmill_types::user_drafts::DraftUserRef; use crate::db::DB; use crate::error::Result; +use crate::workspaces::ManageKind; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -160,6 +161,21 @@ impl UserDraftItemKind { } } + /// The withdrawable operator right that governs deploying this kind, which is + /// therefore also the right to draft it. Every trigger kind maps here, native ones + /// included: `gate_operator_writes` covers their routers as well. + pub fn manage_kind(&self) -> Option { + use UserDraftItemKind::*; + match self { + TriggerSchedule => Some(ManageKind::Schedules), + TriggerWebhook | TriggerDefaultEmail | TriggerEmail | TriggerHttp + | TriggerWebsocket | TriggerPostgres | TriggerKafka | TriggerNats | TriggerMqtt + | TriggerAmqp | TriggerSqs | TriggerGcp | TriggerAzure | TriggerPoll | TriggerCli + | TriggerNextcloud | TriggerGoogle | TriggerGithub => Some(ManageKind::Triggers), + Script | Flow | App | RawApp | Resource | Variable | DataPipeline => None, + } + } + /// The `draft.value` key holding the user-typed target path — where a /// deploy of this draft would land when the user staged a rename. A script /// draft round-trips its own `path`; flows and apps write a separate diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index b7a2f19d86..ac9c43269a 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -373,6 +373,10 @@ pub async fn get_secret_value_as_admin( Ok(r) } +/// Marks a workspace setting's secret written into the settings document encrypted with the +/// workspace key, as `wmill sync pull` writes it. +pub const SETTINGS_ENCRYPTED_PREFIX: &str = "$encrypted:"; + pub fn encrypt(mc: &WorkspaceCrypt, value: &str) -> String { mc.encrypt_str(value) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index b8e4ce50e6..f5743782e3 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2223,14 +2223,22 @@ pub async fn fetch_raw_script_from_app_query( loc: &'static Location<'_>, ) -> error::Result { sqlx::query!( - "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + "SELECT lock, code, modules AS \"modules: serde_json::Value\" + FROM app_script WHERE id = $1 LIMIT 1", id, ) .fetch_optional(db) .await .map_err(Into::into) .and_then(unwrap_or_error(&loc, "Application script", id)) - .map(|r| RawScript { content: r.code, lock: r.lock, meta: None, modules: None }) + .and_then(|r| { + Ok(RawScript { + content: r.code, + lock: r.lock, + meta: None, + modules: r.modules.map(serde_json::from_value).transpose()?, + }) + }) } /// Returns the number of jobs the row already accounted for: non-zero when a worker of the diff --git a/backend/windmill-common/src/workspace_crypt.rs b/backend/windmill-common/src/workspace_crypt.rs index 6bdea28391..b8114c3abb 100644 --- a/backend/windmill-common/src/workspace_crypt.rs +++ b/backend/windmill-common/src/workspace_crypt.rs @@ -21,6 +21,7 @@ use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng}; use aes_gcm::{Aes256Gcm, Key, Nonce}; use base64::Engine; use hkdf::Hkdf; +use hmac::{Hmac, Mac}; use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use sha2::Sha256; @@ -36,6 +37,7 @@ const NARROW_VARIABLE_VALUE_WIDTH: usize = 15000; pub const MAX_BYTES_OVERHEAD: usize = V2_MAGIC.len() + NONCE_LEN + BLOCK + TAG_LEN; const AEAD_KEY_INFO: &[u8] = b"windmill workspace secret aes-256-gcm v1"; +const SYNTHETIC_NONCE_KEY_INFO: &[u8] = b"windmill workspace secret synthetic nonce v1"; const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; @@ -79,6 +81,7 @@ impl CipherFormat { pub struct WorkspaceCrypt { legacy: MagicCrypt256, aead: Aes256Gcm, + synthetic_nonce_key: [u8; 32], } /// Code that drives `magic_crypt` directly on this cipher (app link secrets) keeps the legacy @@ -99,9 +102,14 @@ impl WorkspaceCrypt { Hkdf::::new(None, crypt_key.as_bytes()) .expand(AEAD_KEY_INFO, &mut aead_key) .expect("32 bytes is a valid HKDF-SHA256 output length"); + let mut synthetic_nonce_key = [0u8; 32]; + Hkdf::::new(None, crypt_key.as_bytes()) + .expand(SYNTHETIC_NONCE_KEY_INFO, &mut synthetic_nonce_key) + .expect("32 bytes is a valid HKDF-SHA256 output length"); Self { legacy: magic_crypt::new_magic_crypt!(crypt_key, 256), aead: Aes256Gcm::new(Key::::from_slice(&aead_key)), + synthetic_nonce_key, } } @@ -132,6 +140,18 @@ impl WorkspaceCrypt { } } + /// A v2 value whose nonce is an HMAC of the plaintext, so the same plaintext always gives + /// the same ciphertext. For values written into synced files, where a fresh nonce on every + /// export would show as a change. It reveals that two values are equal, nothing more; it + /// decrypts with [`Self::decrypt_str`] like any v2 value. + pub fn encrypt_str_deterministic(&self, plaintext: &str) -> String { + let mut mac = as Mac>::new_from_slice(&self.synthetic_nonce_key) + .expect("HMAC accepts any key length"); + mac.update(plaintext.as_bytes()); + let tag = mac.finalize().into_bytes(); + B64.encode(self.seal_with_nonce(plaintext.as_bytes(), Nonce::from_slice(&tag[..NONCE_LEN]))) + } + pub fn decrypt_str(&self, ciphertext: &str) -> Result { let bytes = B64 .decode(ciphertext) @@ -175,18 +195,21 @@ impl WorkspaceCrypt { } fn seal(&self, plaintext: &[u8]) -> Vec { - let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + self.seal_with_nonce(plaintext, &Aes256Gcm::generate_nonce(&mut OsRng)) + } + + fn seal_with_nonce(&self, plaintext: &[u8], nonce: &Nonce<::NonceSize>) -> Vec { let pad = BLOCK - plaintext.len() % BLOCK; let mut padded = Vec::with_capacity(plaintext.len() + pad); padded.extend_from_slice(plaintext); padded.resize(plaintext.len() + pad, pad as u8); let sealed = self .aead - .encrypt(&nonce, padded.as_slice()) + .encrypt(nonce, padded.as_slice()) .expect("AES-GCM encryption fails only past 64 GiB of plaintext"); let mut out = Vec::with_capacity(V2_MAGIC.len() + NONCE_LEN + sealed.len()); out.extend_from_slice(&V2_MAGIC); - out.extend_from_slice(&nonce); + out.extend_from_slice(nonce); out.extend_from_slice(&sealed); out } @@ -234,6 +257,17 @@ mod tests { } } + #[test] + fn deterministic_is_stable_per_plaintext_and_key() { + let wc = WorkspaceCrypt::new(KEY); + let a = wc.encrypt_str_deterministic("secret"); + assert_eq!(a, wc.encrypt_str_deterministic("secret")); + assert_ne!(a, wc.encrypt_str_deterministic("secret2")); + assert_ne!(a, WorkspaceCrypt::new("other-key").encrypt_str_deterministic("secret")); + assert!(wc.is_v2(&a)); + assert_eq!(wc.decrypt_str(&a).unwrap(), "secret"); + } + #[test] fn legacy_cbc_still_decrypts() { let wc = WorkspaceCrypt::new(KEY); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 25d15ae973..b821ffa744 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -150,6 +150,15 @@ impl From<&Vec> for ProtectionRules { } } +impl ProtectionRules { + /// The rules set, in declaration order. + pub fn kinds(self) -> Vec { + ::iter() + .filter(|k| self.contains(k.flag())) + .collect() + } +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct WorkspaceGitSyncSettings { #[serde(skip_serializing_if = "Option::is_none")] @@ -1362,6 +1371,29 @@ pub async fn operator_builder_rights(db: &DB, workspace_id: &str) -> Result Result { + use crate::user_drafts::UserDraftItemKind; + if matches!( + kind, + UserDraftItemKind::Resource | UserDraftItemKind::Variable + ) { + return Ok(true); + } + let rights = operator_rights(db, workspace_id).await?; + Ok(match kind { + UserDraftItemKind::Flow => rights.builder.has(BuilderKind::Flows), + UserDraftItemKind::RawApp => rights.builder.has(BuilderKind::Apps), + kind => kind.manage_kind().is_some_and(|m| rights.manage.has(m)), + }) +} + /// Gate for a write only a workspace that granted the builder right covering `kind` lets operators /// perform. `action` completes "Operators cannot {action} for security reasons". pub async fn check_operator_can_build( @@ -1578,9 +1610,10 @@ pub struct DataTable { /// which is how data tables behaved before roles existed. Only meaningful on a terminal entry; /// a pointer is governed by what it points at. /// - /// Never leaves the instance: stripped from the workspace export and ignored on import, since - /// tenants are workspace-scoped names and syncing them would make repo write access a second - /// door onto the access decision. + /// Stripped from the `datatable` setting wherever it is read or written as a whole. It leaves + /// the instance only as the settings document's `datatable_permissions`, which `wmill sync` + /// carries only when the repo opts in: tenants are workspace-scoped names, and syncing them + /// makes repo write access a second door onto the access decision. #[serde(default, skip_serializing_if = "Option::is_none")] pub permissions: Option, } @@ -1806,7 +1839,23 @@ pub async fn managed_database_uses( dbname: &str, exempt: Option<(&str, &str)>, ) -> Result> { - let (exempt_workspace, exempt_name) = exempt.unzip(); + match exempt { + Some((workspace_id, name)) => { + managed_database_uses_except(conn, kind, dbname, workspace_id, &[name]).await + } + None => managed_database_uses_except(conn, kind, dbname, "", &[]).await, + } +} + +/// [`managed_database_uses`], exempting every entry of `workspace_id` in `names`: the data tables +/// of a fork that share one copy stop using it together. +pub async fn managed_database_uses_except( + conn: &mut sqlx::PgConnection, + kind: DataTableCatalogResourceType, + dbname: &str, + workspace_id: &str, + names: &[&str], +) -> Result> { Ok(sqlx::query_scalar::<_, String>( "WITH entries AS ( SELECT ws.workspace_id::text AS workspace_id, dt.key AS name, dt.value @@ -1820,7 +1869,7 @@ pub async fn managed_database_uses( AND value->'database'->>'resource_path' = $2 ) SELECT format('data table ''%s'' in workspace ''%s''', name, workspace_id) FROM naming - WHERE $3::text IS NULL OR NOT (workspace_id = $3 AND name = $4) + WHERE NOT (workspace_id = $3 AND name = ANY($4)) UNION ALL SELECT format('data table ''%s'' in workspace ''%s'', which points at the one in ''%s''', e.name, e.workspace_id, n.workspace_id) @@ -1843,8 +1892,8 @@ pub async fn managed_database_uses( ) .bind(kind.as_ref()) .bind(dbname) - .bind(exempt_workspace) - .bind(exempt_name) + .bind(workspace_id) + .bind(names) .fetch_all(&mut *conn) .await?) } diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index aab50795a9..1d47056cf9 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -2,6 +2,7 @@ pub mod ci_tests; #[cfg(feature = "private")] pub mod ci_tests_ee; pub mod lock_hash; +pub mod raw_app_runnables; pub mod scoped_dependency_map; pub mod trigger_dependents; pub mod workspace_dependencies; diff --git a/backend/windmill-dep-map/src/raw_app_runnables.rs b/backend/windmill-dep-map/src/raw_app_runnables.rs new file mode 100644 index 0000000000..87b148d53c --- /dev/null +++ b/backend/windmill-dep-map/src/raw_app_runnables.rs @@ -0,0 +1,160 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use serde_json::{from_value, Value}; +use windmill_common::scripts::{ScriptLang, ScriptModule}; + +use crate::{extract_referenced_paths, extract_relative_imports}; + +/// A raw app's inline runnables, keyed by the path a relative import of one resolves to: +/// `/`, the path the runnable itself runs under. A runnable ships +/// the ones it imports as its modules, so they resolve from the job dir at run time. +#[derive(Default)] +pub struct RawAppRunnables(HashMap); + +impl RawAppRunnables { + pub fn from_value(app_path: &str, value: &Value) -> Self { + let mut runnables = HashMap::new(); + if let Some(Value::Object(by_id)) = value.get("runnables") { + for (id, runnable) in by_id { + let Some(script) = runnable.get("inlineScript") else { + continue; + }; + let content = script.get("content").and_then(Value::as_str); + let language = script + .get("language") + .and_then(|l| from_value::(l.clone()).ok()); + if let (Some(content), Some(language)) = (content, language) { + // `main.ts` is the Bun entrypoint's own file in the job dir, so a + // runnable with that id cannot ship as a module next to it. + let ext = module_extension(language); + if ext.is_some() && !(ext == Some("ts") && id == "main") { + runnables.insert( + format!("{app_path}/{id}"), + (id.clone(), language, content.to_string()), + ); + } + } + } + } + Self(runnables) + } + + /// Whether an import of `path` by `language` code resolves to a runnable shipped as a + /// module, rather than to the workspace script at that path. + pub fn resolves(&self, path: &str, language: ScriptLang) -> bool { + self.0.get(path).is_some_and(|(_, sibling_language, _)| { + module_extension(*sibling_language) == module_extension(language) + }) + } + + /// The runnables `content` imports, directly or through one another. + pub fn modules_of( + &self, + app_path: &str, + content: &str, + language: ScriptLang, + ) -> Option> { + let ext = module_extension(language)?; + let mut queue = + extract_relative_imports(content, &format!("{app_path}/app"), &Some(language)) + .unwrap_or_default(); + let mut visited = HashSet::new(); + let mut modules = BTreeMap::new(); + while let Some(path) = queue.pop() { + if !visited.insert(path.clone()) { + continue; + } + let Some((id, sibling_language, sibling_content)) = self.0.get(&path) else { + continue; + }; + if module_extension(*sibling_language) != Some(ext) { + continue; + } + queue.extend( + extract_relative_imports(sibling_content, &path, &Some(*sibling_language)) + .unwrap_or_default(), + ); + modules.insert( + format!("{id}.{ext}"), + ScriptModule { + content: sibling_content.clone(), + language: *sibling_language, + lock: None, + }, + ); + } + (!modules.is_empty()).then_some(modules) + } + + /// What an app inline script depends on, as its dependency-map node records it. An import + /// satisfied by another runnable is not a dependency on the workspace script at that path, + /// but what that runnable imports is: the script's lock covers its modules, so a change + /// there must relock it and fail its skip check. + pub fn referenced_paths( + &self, + app_path: &str, + content: &str, + language: ScriptLang, + ) -> Option> { + let base = format!("{app_path}/app"); + let own = extract_referenced_paths(content, &base, Some(language)); + let Some(modules) = self.modules_of(app_path, content, language) else { + return own.map(|paths| { + paths + .into_iter() + .filter(|p| !self.resolves(p, language)) + .collect() + }); + }; + let mut paths = own.unwrap_or_default(); + for module in modules.values() { + paths.extend( + extract_referenced_paths(&module.content, &base, Some(module.language)) + .unwrap_or_default(), + ); + } + paths.retain(|p| !self.resolves(p, language)); + paths.sort(); + paths.dedup(); + Some(paths) + } +} + +/// The languages whose executors resolve a relative import against a module file. +pub fn module_extension(language: ScriptLang) -> Option<&'static str> { + match language { + ScriptLang::Bun | ScriptLang::Bunnative => Some("ts"), + ScriptLang::Python3 => Some("py"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn modules_follow_imports_between_runnables_of_one_language() { + let value = json!({ "runnables": { + "main": { "inlineScript": { "language": "bun", "content": "import { a } from './a'" } }, + "a": { "inlineScript": { "language": "bun", "content": "import { b } from './b.ts'" } }, + "b": { "inlineScript": { "language": "bun", "content": "export const b = 1" } }, + "unused": { "inlineScript": { "language": "bun", "content": "export const u = 1" } }, + "py": { "inlineScript": { "language": "python3", "content": "x = 1" } }, + }}); + let runnables = RawAppRunnables::from_value("u/admin/app", &value); + + let modules = runnables + .modules_of( + "u/admin/app", + "import { a } from './a'\nimport { p } from './py'", + ScriptLang::Bun, + ) + .unwrap(); + assert_eq!(modules.keys().collect::>(), vec!["a.ts", "b.ts"]); + assert!(runnables + .modules_of("u/admin/app", "export const x = 1", ScriptLang::Bun) + .is_none()); + } +} diff --git a/backend/windmill-dep-map/src/scoped_dependency_map.rs b/backend/windmill-dep-map/src/scoped_dependency_map.rs index ad1093bb76..e57273e43f 100644 --- a/backend/windmill-dep-map/src/scoped_dependency_map.rs +++ b/backend/windmill-dep-map/src/scoped_dependency_map.rs @@ -11,6 +11,8 @@ use windmill_common::{ use std::collections::HashSet; +use crate::raw_app_runnables::RawAppRunnables; + lazy_static::lazy_static! { pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok(); } @@ -506,15 +508,19 @@ SELECT importer_node_id, imported_path, imported_lockfile_hash let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "app", db).await?; let mut tx = db.begin().await?; let mut to_process = vec![]; + let runnables = RawAppRunnables::from_value(&r.path, &value); traverse_app_inline_scripts(&value, None, &mut |ais, id| { - to_process.push(( - crate::extract_referenced_paths( + let referenced_paths = match ais.language { + Some(language) => { + runnables.referenced_paths(&r.path, &ais.content, language) + } + None => crate::extract_referenced_paths( &ais.content, &(r.path.clone() + "/app"), - ais.language, + None, ), - id, - )); + }; + to_process.push((referenced_paths, id)); Ok(()) })?; diff --git a/backend/windmill-dep-map/src/trigger_dependents.rs b/backend/windmill-dep-map/src/trigger_dependents.rs index 48e151736a..45671bb132 100644 --- a/backend/windmill-dep-map/src/trigger_dependents.rs +++ b/backend/windmill-dep-map/src/trigger_dependents.rs @@ -12,6 +12,11 @@ use windmill_queue::PushIsolationLevel; use crate::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; +/// A `components_to_relock` entry asking an app relock to relock every component of the version +/// it runs on. It travels in the list itself so that debouncing, which concatenates the lists of +/// the relocks it merges, cannot drop it. +pub const RELOCK_ALL_APP_COMPONENTS: &str = "*"; + lazy_static::lazy_static! { static ref DEPENDENCY_JOB_DEBOUNCE_DELAY: usize = std::env::var("DEPENDENCY_JOB_DEBOUNCE_DELAY").ok().and_then(|flag| flag.parse().ok()).unwrap_or( if cfg!(test) { 15 } else { 5 } @@ -141,12 +146,13 @@ pub async fn trigger_dependents_to_recompute_dependencies( }, "app" => match sqlx::query_scalar!( - "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2", importer_path, w_id ) .fetch_optional(&mut *tx) .await? + .flatten() { Some(version) => { tracing::debug!("Handling app dependency update for: {}", importer_path); diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index f92bb6fda1..74cd7b8a18 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -460,19 +460,11 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "diffy" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10aec8f7f9393bd6a4f2762be0ceb012d3cbe2478987258cc9960de148561914" -dependencies = [ - "hashbrown 0.17.1", -] - [[package]] name = "duckdb" -version = "1.10505.0" -source = "git+https://github.com/windmill-labs/duckdb-rs?rev=7190adfcf58624e7e9a7a2a1000bac18fed64c19#7190adfcf58624e7e9a7a2a1000bac18fed64c19" +version = "1.10506.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3472b9152d879c5534353b20854e43525b2c78b2dbba156711e9939d30278b4" dependencies = [ "arrow", "cast", @@ -547,12 +539,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "funty" version = "2.0.0" @@ -644,7 +630,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash 0.1.5", + "foldhash", ] [[package]] @@ -652,9 +638,6 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "foldhash 0.2.0", -] [[package]] name = "hashlink" @@ -813,11 +796,11 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libduckdb-sys" -version = "1.10505.0" -source = "git+https://github.com/windmill-labs/duckdb-rs?rev=7190adfcf58624e7e9a7a2a1000bac18fed64c19#7190adfcf58624e7e9a7a2a1000bac18fed64c19" +version = "1.10506.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aac3459f2f7f0221d5be9114fa77ff5877040c8de456d3b986a680579fe202b" dependencies = [ "cc", - "diffy", "flate2", "pkg-config", "serde", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 73e132a2d7..ba78e3522d 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -11,10 +11,9 @@ chrono = "0.4.41" # identical DECIMAL. Building one by hand would mean reimplementing that # f64 → (mantissa, scale) inference. # -# Not crates.io: a fork of duckdb-rs 1.10505.0 carrying two patches to the bundled engine — -# see docs/duckdb-isolation.md for why, and for the exit path back to the crates.io crate. -# Pinned by rev, not branch, so `cargo update` cannot move the engine under us. -duckdb = { git = "https://github.com/windmill-labs/duckdb-rs", rev = "7190adfcf58624e7e9a7a2a1000bac18fed64c19", features = [ +# Pinned exactly, so `cargo update` cannot move the engine (and the prebuilt extensions it +# must match) under us. +duckdb = { version = "=1.10506.0", features = [ "bundled", "rust_decimal", ] } @@ -25,5 +24,6 @@ serde = { version = "1.0", features = ["derive"] } # to avoid re-parsing every row. serde_json = { version = "^1.0.151", features = ["preserve_order", "raw_value"] } -[lib] -crate-type = ["cdylib"] +[[bin]] +name = "windmill_duckdb_runner" +path = "src/bin/windmill_duckdb_runner.rs" diff --git a/backend/windmill-duckdb-ffi-internal/README_DEV.md b/backend/windmill-duckdb-ffi-internal/README_DEV.md index 0c34ebc069..f5c5d61083 100644 --- a/backend/windmill-duckdb-ffi-internal/README_DEV.md +++ b/backend/windmill-duckdb-ffi-internal/README_DEV.md @@ -1,8 +1,8 @@ -This crate is compiled separately because it causes nasty issues when compiled with the deno_core feature flag enabled (lib c++ interactions). +This crate is compiled separately from the backend workspace because the bundled DuckDB's libc++ +clashes with `deno_core`'s when both live in one process. -The main issue was : -Errors in DuckDB always worked correctly, except when attached to a Ducklake and when the deno_core feature flag was set. -For example: +The symptom was opaque errors. DuckDB errors worked, except when attached to a DuckLake with the +`deno_core` feature on: ```sql ATTACH 'ducklake' AS dl; USE dl; @@ -10,6 +10,9 @@ CREATE TABLE IF NOT EXISTS t (x string not null); INSERT INTO t VALUES (NULL); ``` -causes `Constraint Error: NOT NULL constraint failed: t.x` normally, but here we see `Unknown exception in ExecutorTask::Execute`. This opaque errors comes directly from the C++ DuckDB library : https://github.com/duckdb/duckdb/blob/f99fed1e0b16a842573f9dad529f6c170a004f6e/src/parallel/executor_task.cpp#L58 +gave `Unknown exception in ExecutorTask::Execute` instead of `Constraint Error: NOT NULL +constraint failed: t.x`. The opaque error comes from the C++ DuckDB library: +https://github.com/duckdb/duckdb/blob/f99fed1e0b16a842573f9dad529f6c170a004f6e/src/parallel/executor_task.cpp#L58 -To solve this, we compile duckdb separately from the main backend crate and call it with FFI. It has to be loaded dynamically, if it is loaded statically it will still share lib c++ with deno_core and cause issues. +The worker never links this crate: it runs `windmill_duckdb_runner` (`src/bin`) as a separate +process for every job. See `docs/duckdb-execution.md`. diff --git a/backend/windmill-duckdb-ffi-internal/build_dev.sh b/backend/windmill-duckdb-ffi-internal/build_dev.sh index dab70398a8..c334baa6ae 100755 --- a/backend/windmill-duckdb-ffi-internal/build_dev.sh +++ b/backend/windmill-duckdb-ffi-internal/build_dev.sh @@ -31,22 +31,21 @@ else echo "duckdb-ffi: shared cache $CARGO_TARGET_DIR" # The cache key covers only the dependency inputs (Cargo.lock + build.rs), so # the expensive bundled-DuckDB build is shared even when this crate's own - # source differs between checkouts. But the shared dir's uplifted cdylib is a + # source differs between checkouts. But the shared dir's runner binary is a # single fixed-name artifact written by whichever worktree built last. Touch - # our sources so cargo re-links THIS checkout's cdylib and re-uplifts it + # our sources so cargo re-links THIS checkout's runner and re-uplifts it # before we copy — the copied artifact then always matches this worktree, # never a sibling's. The bundled dependency is a separate crate and stays # cached, so this only costs a ~1s relink. find src -type f -exec touch {} + fi -CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release -p windmill_duckdb_ffi_internal +cargo build --release -p windmill_duckdb_ffi_internal mkdir -p ../target/debug/ -# Install through a rename rather than writing over the destination: a running worker -# has the cdylib mapped, and rewriting those pages under it kills the process with -# SIGBUS. A rename swaps the directory entry and leaves the old inode alone. -for artifact in "$CARGO_TARGET_DIR/release/"libwindmill_duckdb_ffi_internal.*; do - dest="../target/debug/$(basename "$artifact")" - cp "$artifact" "$dest.tmp" - mv -f "$dest.tmp" "$dest" -done +# The worker looks for the runner next to its own binary. Install through a rename rather +# than writing over the destination: a runner may be executing, and rewriting its pages under +# it kills the process with SIGBUS. A rename swaps the directory entry and leaves the old +# inode alone. +dest="../target/debug/windmill_duckdb_runner" +cp "$CARGO_TARGET_DIR/release/windmill_duckdb_runner" "$dest.tmp" +mv -f "$dest.tmp" "$dest" diff --git a/backend/windmill-duckdb-ffi-internal/src/bin/windmill_duckdb_runner.rs b/backend/windmill-duckdb-ffi-internal/src/bin/windmill_duckdb_runner.rs new file mode 100644 index 0000000000..38052a9537 --- /dev/null +++ b/backend/windmill-duckdb-ffi-internal/src/bin/windmill_duckdb_runner.rs @@ -0,0 +1,122 @@ +//! Runs one DuckDB job, so the engine lives in a process of its own under the worker's job +//! isolation rather than in the worker. +//! +//! The worker writes one JSON [`Request`] to stdin and reads the answer from +//! `Request::result_path`: a first line holding the column order as JSON (empty when there is +//! none), then the collected results, or `ERROR `. +//! +//! stdout/stderr are not protocol: the worker keeps only their tail, and reports it when the +//! runner dies without writing a result. + +use std::io::{Read, Write}; + +use serde::Deserialize; +use windmill_duckdb_ffi_internal::{prepare, run, Arg, Connection}; + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +enum Mode { + Run, + Prepare, + /// `INSTALL` only, into the extension cache the jail later mounts read-only. Never loads + /// anything, so it runs no extension code. + Install, +} + +#[derive(Deserialize)] +struct Request { + mode: Mode, + blocks: Vec, + #[serde(default)] + job_args: Vec, + #[serde(default)] + token: String, + #[serde(default)] + base_internal_url: String, + #[serde(default)] + w_id: String, + #[serde(default)] + memory_limit: String, + #[serde(default)] + temp_directory: String, + #[serde(default)] + max_result_size: usize, + #[serde(default)] + collect_last_only: bool, + #[serde(default)] + collect_first_row_only: bool, + result_path: String, +} + +fn non_empty(s: &str) -> Option { + (!s.is_empty()).then(|| s.to_string()) +} + +fn install(blocks: &[String]) -> Result { + let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; + for b in blocks { + conn.execute_batch(b).map_err(|e| e.to_string())?; + } + Ok("[]".to_string()) +} + +/// The result string and the column order as JSON. +fn execute(req: Request) -> (String, Option) { + let conn = Connection { + token: &req.token, + base_internal_url: &req.base_internal_url, + w_id: &req.w_id, + memory_limit: non_empty(&req.memory_limit), + temp_directory: non_empty(&req.temp_directory), + }; + let outcome = match req.mode { + Mode::Install => install(&req.blocks).map(|r| (r, None)), + Mode::Prepare => prepare(&req.blocks, conn).map(|r| (r, None)), + Mode::Run => run( + &req.blocks, + req.job_args, + conn, + req.max_result_size, + req.collect_last_only, + req.collect_first_row_only, + ) + .map(|(r, order)| { + let order = order.map(|o| serde_json::to_string(&o).unwrap_or_else(|_| "[]".into())); + (r, order) + }), + }; + outcome.unwrap_or_else(|e| { + let msg = serde_json::to_string(&e).unwrap_or_else(|_| "\"Unknown DuckDB error\"".into()); + (format!("ERROR {msg}"), None) + }) +} + +fn main() { + let mut input = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut input) { + eprintln!("windmill_duckdb_runner: could not read the request: {e}"); + std::process::exit(2); + } + let req: Request = match serde_json::from_str(&input) { + Ok(r) => r, + Err(e) => { + eprintln!("windmill_duckdb_runner: invalid request: {e}"); + std::process::exit(2); + } + }; + drop(input); + + let result_path = req.result_path.clone(); + let (result, column_order) = execute(req); + let write = || -> std::io::Result<()> { + let mut f = std::io::BufWriter::new(std::fs::File::create(&result_path)?); + f.write_all(column_order.as_deref().unwrap_or("").as_bytes())?; + f.write_all(b"\n")?; + f.write_all(result.as_bytes())?; + f.flush() + }; + if let Err(e) = write() { + eprintln!("windmill_duckdb_runner: could not write the result to {result_path}: {e}"); + std::process::exit(2); + } +} diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index 69820b285f..0de7d3bd79 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,9 +1,9 @@ -use std::{ - collections::HashMap, - ffi::{c_char, c_uint, CStr, CString}, - ptr::null_mut, - sync::LazyLock, -}; +//! DuckDB execution for windmill jobs. The worker never loads this crate: it runs +//! `windmill_duckdb_runner` (src/bin), which calls [`run`] / [`prepare`] in a process of its +//! own, under the worker's job isolation. Kept out of the backend workspace because the +//! bundled engine's libc++ clashes with `deno_core`'s in one process. + +use std::{collections::HashMap, sync::LazyLock}; use duckdb::{core::LogicalTypeId, params_from_iter, types::TimeUnit, Row}; use regex::Regex; @@ -11,22 +11,6 @@ use rust_decimal::{prelude::FromPrimitive, Decimal}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -// Worker passes "" for "no override" — saves an extra C string nullability dance. -// Returns an owned String so the value outlives the raw pointer's lifetime. -fn ptr_to_opt_str(ptr: *const c_char) -> Result, String> { - if ptr.is_null() { - return Ok(None); - } - let s = unsafe { CStr::from_ptr(ptr) } - .to_str() - .map_err(|e| format!("Invalid string in duckdb ffi: {}", e))?; - Ok(if s.is_empty() { - None - } else { - Some(s.to_owned()) - }) -} - #[derive(Deserialize, Clone, Debug, PartialEq, Default)] pub struct Arg { pub name: String, @@ -34,95 +18,69 @@ pub struct Arg { pub json_value: serde_json::Value, } -// Freeing from the caller side crashes the runtime with jemalloc enabled (EXIT CODE 11 SEGFAULT) -#[unsafe(no_mangle)] -pub extern "C" fn free_cstr(string: *mut c_char) -> () { - if string.is_null() { - return; - } - unsafe { - let _ = CString::from_raw(string); - } +/// Everything a connection is set up with before the script's first block runs. +pub struct Connection<'a> { + /// The job token: the S3/GCS secrets against the workspace's S3 proxy are derived from it. + pub token: &'a str, + pub base_internal_url: &'a str, + pub w_id: &'a str, + pub memory_limit: Option, + /// Where DuckDB spills; the job dir, so the spill is cleaned up with the job. + pub temp_directory: Option, } -#[unsafe(no_mangle)] -pub extern "C" fn get_version() -> c_uint { - // Increment when making breaking changes to the FFI interface. - // The windmill worker will check that the version matches or else refuse to call - // the FFI functions to avoid undefined behavior. - return 3; -} - -#[unsafe(no_mangle)] -pub extern "C" fn run_duckdb_ffi( - query_block_list: *const *const c_char, - query_block_list_count: usize, - job_args: *const c_char, - token: *const c_char, - base_internal_url: *const c_char, - w_id: *const c_char, - memory_limit: *const c_char, - temp_directory: *const c_char, - // 0 means unbounded; the worker sends its own budget-derived cap. +/// Runs the blocks in order on one in-memory connection and returns every collected result, +/// serialized, plus the last statement's column order. `max_result_size` 0 is unbounded. +pub fn run( + blocks: &[String], + job_args: Vec, + conn: Connection<'_>, max_result_size: usize, - column_order_ptr: *mut *mut c_char, collect_last_only: bool, collect_first_row_only: bool, -) -> *mut c_char { - let resource_limits = match (ptr_to_opt_str(memory_limit), ptr_to_opt_str(temp_directory)) { - (Ok(m), Ok(t)) => Ok(ResourceLimits::from_ffi(m, t)), - (Err(e), _) | (_, Err(e)) => Err(e), - }; - let (r, column_order) = match convert_args( - query_block_list, - query_block_list_count, - job_args, - token, - base_internal_url, - w_id, +) -> Result<(String, Option>), String> { + let limits = ResourceLimits::new(conn.memory_limit, conn.temp_directory); + run_duckdb_internal( + blocks.iter().map(String::as_str), + blocks.len(), + job_args_to_values(job_args), + conn.token, + conn.base_internal_url, + conn.w_id, + limits, + max_result_size, + collect_last_only, + collect_first_row_only, ) - .and_then(|args| resource_limits.map(|r| (args, r))) - .and_then( - |((query_block_list, job_args, token, base_internal_url, w_id), limits)| { - run_duckdb_internal( - query_block_list, - query_block_list_count, - job_args, - token, - base_internal_url, - w_id, - limits, - max_result_size, - collect_last_only, - collect_first_row_only, - ) - }, - ) { - Ok(result) => result, - Err(err) => { - let err = serde_json::to_string(&err) - .unwrap_or_else(|_| "Unknown error in duckdb ffi lib".to_string()); - (format!("ERROR {}", err), None) - } - }; +} - unsafe { - if let Some(column_order) = column_order { - let column_order = - serde_json::to_string(&column_order).unwrap_or_else(|_| "[]".to_string()); - let c_column_order = - CString::new(column_order).unwrap_or_else(|_| CString::new("[]").unwrap()); - *column_order_ptr = c_column_order.into_raw(); - } else { - *column_order_ptr = null_mut(); - } - } - // CString::into_raw because it needs to outlive this function call. - // The caller is responsible for freeing the memory through CString::from_raw. - CString::new(r).map(|s| s.into_raw()).unwrap_or_else(|e| { - println!("Failed to allocate error string in duckdb ffi lib: {:?}", e); - null_mut() - }) +/// Describes each non-setup block's result columns without running it. +pub fn prepare(blocks: &[String], conn: Connection<'_>) -> Result { + let limits = ResourceLimits::new(conn.memory_limit, conn.temp_directory); + prepare_duckdb_internal( + blocks.iter().map(String::as_str).collect(), + conn.token, + conn.base_internal_url, + conn.w_id, + limits, + ) +} + +fn job_args_to_values(job_args: Vec) -> HashMap { + job_args + .into_iter() + .map(|arg| { + let duckdb_value = json_value_to_duckdb_value(&arg.json_value, &arg.arg_type) + .unwrap_or_else(|e| { + println!( + "Error converting job_arg {} to duckdb value: {}", + arg.name, e + ); + duckdb::types::Value::Null + }); + (arg.name, duckdb_value) + }) + .collect() } #[derive(Serialize, Debug)] @@ -187,47 +145,6 @@ fn replace_params_with_null(query: &str) -> String { PARAM_RE.replace_all(query, "NULL").to_string() } -#[unsafe(no_mangle)] -pub extern "C" fn prepare_duckdb_ffi( - query_block_list: *const *const c_char, - query_block_list_count: usize, - token: *const c_char, - base_internal_url: *const c_char, - w_id: *const c_char, - memory_limit: *const c_char, - temp_directory: *const c_char, -) -> *mut c_char { - let resource_limits = match (ptr_to_opt_str(memory_limit), ptr_to_opt_str(temp_directory)) { - (Ok(m), Ok(t)) => Ok(ResourceLimits::from_ffi(m, t)), - (Err(e), _) | (_, Err(e)) => Err(e), - }; - let r = match convert_prepare_args( - query_block_list, - query_block_list_count, - token, - base_internal_url, - w_id, - ) - .and_then(|args| resource_limits.map(|r| (args, r))) - .and_then( - |((query_block_list, token, base_internal_url, w_id), limits)| { - prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id, limits) - }, - ) { - Ok(result) => result, - Err(err) => { - let err = serde_json::to_string(&err) - .unwrap_or_else(|_| "Unknown error in duckdb ffi lib".to_string()); - format!("ERROR {}", err) - } - }; - - CString::new(r).map(|s| s.into_raw()).unwrap_or_else(|e| { - println!("Failed to allocate error string in duckdb ffi lib: {:?}", e); - null_mut() - }) -} - #[derive(Clone, Default)] struct ResourceLimits { memory_limit: Option, @@ -236,11 +153,10 @@ struct ResourceLimits { } impl ResourceLimits { - // The spill cap comes from the environment rather than from an FFI parameter: this - // cdylib is versioned separately from the worker and agent workers ship it out of band, - // so a signature change costs an ABI bump and a coordinated redeploy for what is a - // config value. Unset keeps DuckDB's default of 90% of the temp volume's free space. - fn from_ffi(memory_limit: Option, temp_directory: Option) -> Self { + // The spill cap is the worker's DUCKDB_MAX_TEMP_DIRECTORY_SIZE, passed through to the + // runner's environment. Unset keeps DuckDB's default of 90% of the temp volume's free + // space. A script can raise it, as it can write the job dir any other way. + fn new(memory_limit: Option, temp_directory: Option) -> Self { let max_temp_directory_size = std::env::var("DUCKDB_MAX_TEMP_DIRECTORY_SIZE") .ok() .map(|v| v.trim().to_string()) @@ -253,10 +169,9 @@ fn sql_single_quote(s: &str) -> String { s.replace('\'', "''") } -// Bounds memory so DuckDB spills before blowing the cgroup cap and getting the worker -// SIGKILLed, and bounds the spill so one query cannot fill its disk. Spill goes to the job -// dir when set, so it is cleaned up with the job; either way the directory and its cap are -// then locked for the rest of the connection. +// Bounds memory so DuckDB spills before blowing the cgroup cap and getting the job +// SIGKILLed, and sets a default bound on the spill. Spill goes to the job dir when set, so it +// is cleaned up with the job. fn configure_duckdb_resource_limits( conn: &duckdb::Connection, limits: &ResourceLimits, @@ -301,11 +216,7 @@ fn configure_duckdb_resource_limits( ) })?; } - // Freeze both before any script statement can touch them. A remote temp directory makes - // `~TemporaryDirectoryHandle` throw out of a `noexcept` destructor — `std::terminate`, - // taking this in-process worker and every job on it down, uncatchable from Rust — and a - // cap a script can raise bounds nothing. - run("SET lock_temp_directory=true;\n") + Ok(()) } fn setup_duckdb_connection( @@ -353,39 +264,6 @@ fn setup_duckdb_connection( .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string())) } -fn convert_prepare_args<'a>( - query_block_list: *const *const c_char, - query_block_list_count: usize, - token: *const c_char, - base_internal_url: *const c_char, - w_id: *const c_char, -) -> Result<(Vec<&'a str>, &'a str, &'a str, &'a str), String> { - let query_block_list = unsafe { - std::slice::from_raw_parts(query_block_list, query_block_list_count) - .iter() - .map(|q| { - CStr::from_ptr(*q).to_str().unwrap_or_else(|e| { - println!( - "Invalid query_block string pointer in duckdb ffi: {}", - e.to_string() - ); - "Invalid query_block string pointer in duckdb ffi" - }) - }) - .collect::>() - }; - let token = unsafe { CStr::from_ptr(token) } - .to_str() - .map_err(|e| format!("Invalid token string: {}", e.to_string()))?; - let base_internal_url = unsafe { CStr::from_ptr(base_internal_url) } - .to_str() - .map_err(|e| format!("Invalid base_internal_url string: {}", e.to_string()))?; - let w_id = unsafe { CStr::from_ptr(w_id) } - .to_str() - .map_err(|e| format!("Invalid w_id string: {}", e.to_string()))?; - Ok((query_block_list, token, base_internal_url, w_id)) -} - fn prepare_duckdb_internal( query_block_list: Vec<&str>, token: &str, @@ -453,67 +331,6 @@ fn prepare_duckdb_internal( serde_json::to_string(&results).map_err(|e| e.to_string()) } -fn convert_args<'a>( - query_block_list: *const *const c_char, - query_block_list_count: usize, - job_args: *const c_char, - token: *const c_char, - base_internal_url: *const c_char, - w_id: *const c_char, -) -> Result< - ( - impl Iterator, - HashMap, - &'a str, - &'a str, - &'a str, - ), - String, -> { - let query_block_list = unsafe { - std::slice::from_raw_parts(query_block_list, query_block_list_count) - .iter() - .map(|q| { - CStr::from_ptr(*q).to_str().unwrap_or_else(|e| { - println!( - "Invalid query_block string pointer in duckdb ffi: {}", - e.to_string() - ); - "Invalid query_block string pointer in duckdb ffi" - }) - }) - }; - let job_args_str = unsafe { CStr::from_ptr(job_args) } - .to_str() - .map_err(|e| format!("Invalid job_args string: {}", e.to_string()))?; - let job_args: Vec = serde_json::from_str(job_args_str) - .map_err(|e| format!("Invalid job_args JSON: {}", e.to_string()))?; - let job_args: HashMap = job_args - .into_iter() - .map(|arg| { - let duckdb_value = json_value_to_duckdb_value(&arg.json_value, &arg.arg_type) - .unwrap_or_else(|e| { - println!( - "Error converting job_arg {} to duckdb value: {}", - arg.name, e - ); - duckdb::types::Value::Null - }); - (arg.name, duckdb_value) - }) - .collect(); - let token = unsafe { CStr::from_ptr(token) } - .to_str() - .map_err(|e| format!("Invalid token string: {}", e.to_string()))?; - let base_internal_url = unsafe { CStr::from_ptr(base_internal_url) } - .to_str() - .map_err(|e| format!("Invalid base_internal_url string: {}", e.to_string()))?; - let w_id = unsafe { CStr::from_ptr(w_id) } - .to_str() - .map_err(|e| format!("Invalid w_id string: {}", e.to_string()))?; - Ok((query_block_list, job_args, token, base_internal_url, w_id)) -} - // TODO : Better error return to leverage different error kinds on the worker side fn run_duckdb_internal<'a>( query_block_list: impl Iterator, @@ -1230,8 +1047,8 @@ mod temporal_json_tests { // Numbers too wide for a JSON number are rendered as strings. DECIMAL runs to // 38 digits and UHUGEINT to 2^128-1; rendering either through a type that - // cannot hold it aborts the whole worker rather than erroring, because the - // panic escapes an `extern "C"` frame and those cannot unwind. + // cannot hold it panics, which kills the runner and fails the job with no + // engine message to show for it. #[test] fn wide_numbers_render_as_strings_without_losing_precision() { let conn = duckdb::Connection::open_in_memory().unwrap(); @@ -1677,214 +1494,3 @@ fn string_to_duckdb_time(s: &str) -> Result { time.num_seconds_from_midnight() as i64, )) } - -// The bundled engine is not stock (docs/duckdb-isolation.md). These guard the ways an engine -// bump can drop the patches without anything else noticing. -#[cfg(test)] -mod patched_engine_tests { - use super::*; - - /// Too large to be answered within the `memory_limit` `spill_attempt` sets, so the engine - /// has to reach the temp directory to finish it. - const SPILLING_QUERY: &str = "CREATE TABLE t AS SELECT * FROM range(1000000)"; - - /// Every connection is locked at setup, so no script statement can move the temp - /// directory onto a remote filesystem — where the spill fails and the - /// `~TemporaryDirectoryHandle` cleaning up throws out of a `noexcept` destructor, - /// aborting the worker and every job colocated on it. - #[test] - fn configured_connections_cannot_have_their_temp_directory_moved() { - let temp_dir = std::env::temp_dir().join(format!( - "windmill_duckdb_locked_{}_{:?}", - std::process::id(), - std::thread::current().id() - )); - std::fs::create_dir_all(&temp_dir).unwrap(); - - let conn = duckdb::Connection::open_in_memory().unwrap(); - configure_duckdb_resource_limits( - &conn, - &ResourceLimits { - memory_limit: Some("2MB".to_string()), - temp_directory: Some(temp_dir.display().to_string()), - max_temp_directory_size: None, - }, - ) - .unwrap(); - - for stmt in [ - "SET temp_directory='s3://bucket/tmp'", - "SET temp_directory TO 's3://bucket/tmp'", - "PRAGMA temp_directory='s3://bucket/tmp'", - "RESET temp_directory", - ] { - let err = conn - .execute_batch(stmt) - .expect_err(&format!("engine accepted `{stmt}` on a locked connection")); - assert!( - err.to_string().contains("temp directory has been locked"), - "unexpected refusal for `{stmt}`: {err}" - ); - } - - // The lock must not cost the spilling it exists to protect. - let spilled = conn.execute_batch(SPILLING_QUERY); - std::fs::remove_dir_all(&temp_dir).ok(); - spilled.expect("a locked temp directory must still be usable for spilling"); - } - - /// `DUCKDB_MAX_TEMP_DIRECTORY_SIZE` is the only thing standing between one query and the - /// worker's whole disk, so pin that a configured connection really does stop at it, names - /// the setting when it does, and leaves nothing behind for the next job to trip over. - #[test] - fn spilling_past_the_configured_cap_fails_the_query_and_cleans_up() { - let temp_dir = std::env::temp_dir().join(format!( - "windmill_duckdb_capped_{}_{:?}", - std::process::id(), - std::thread::current().id() - )); - std::fs::create_dir_all(&temp_dir).unwrap(); - - let spill_under_cap = |cap: &str| { - let conn = duckdb::Connection::open_in_memory().unwrap(); - configure_duckdb_resource_limits( - &conn, - &ResourceLimits { - memory_limit: Some("2MB".to_string()), - temp_directory: Some(temp_dir.display().to_string()), - max_temp_directory_size: Some(cap.to_string()), - }, - ) - .unwrap(); - conn.execute_batch(SPILLING_QUERY) - .map_err(|e| e.to_string()) - }; - - let err = spill_under_cap("1MB").expect_err("the cap did not stop the spill"); - // Verbatim because the worker matches on it (`SPILL_CAP_ENGINE_MARKER` in - // `duckdb_executor.rs`) to correct the engine's advice to raise the cap, which the - // lock refuses. An engine bump that rewords this has to update that too. - assert!( - err.contains("This limit was set by the 'max_temp_directory_size' setting"), - "the refusal must name the setting an operator can act on, got: {err}" - ); - let leftover: Vec<_> = std::fs::read_dir(&temp_dir) - .unwrap() - .filter_map(|e| e.ok().map(|e| e.path())) - .collect(); - assert!( - leftover.is_empty(), - "a query killed by the cap must not leave its spill behind: {leftover:?}" - ); - - // DuckDB compresses temp files, so the failure above is only evidence of the cap if - // the same query gets through once the cap is out of the way. - let generous = spill_under_cap("1GB"); - std::fs::remove_dir_all(&temp_dir).ok(); - generous.expect("the query must still spill and finish under a cap that fits it"); - } - - fn spill_attempt(temp_dir: &std::path::Path, lock: bool) -> Result<(), String> { - let conn = duckdb::Connection::open_in_memory().unwrap(); - conn.execute_batch(&format!( - "SET threads=2; SET memory_limit='2MB'; SET temp_directory='{}';", - temp_dir.display() - )) - .unwrap(); - if lock { - conn.execute_batch("SET lock_temp_directory=true;") - .map_err(|e| format!("lock_temp_directory rejected: {e}"))?; - } - conn.execute_batch("SET disabled_filesystems='LocalFileSystem';") - .unwrap(); - conn.execute_batch(SPILLING_QUERY) - .map_err(|e| e.to_string()) - } - - #[test] - fn locking_the_temp_directory_keeps_spilling_available() { - let temp_dir = std::env::temp_dir().join(format!( - "windmill_duckdb_spill_{}_{:?}", - std::process::id(), - std::thread::current().id() - )); - std::fs::create_dir_all(&temp_dir).unwrap(); - - // Without the lock the fence reaches the buffer manager too, which is both the - // stock engine's behaviour and the proof that this query really does spill — - // otherwise the assertion below would pass on an unpatched engine. - let unlocked = spill_attempt(&temp_dir, false).expect_err("query did not need to spill"); - assert!( - unlocked.contains("LocalFileSystem has been disabled"), - "expected the fence to stop the spill, got: {unlocked}" - ); - - let locked = spill_attempt(&temp_dir, true); - std::fs::remove_dir_all(&temp_dir).ok(); - locked.expect("a locked temp directory must stay usable behind disabled_filesystems"); - } - - /// The exemption above is only sound while neither the spill's destination nor its size - /// can be changed: repointing the directory is a write primitive through the very fence - /// it is exempt from, and raising the cap leaves the disk unbounded. A rebase can drop - /// either half while leaving the spill test green, so pin both. - #[test] - fn locking_the_temp_directory_makes_it_immutable() { - let conn = duckdb::Connection::open_in_memory().unwrap(); - conn.execute_batch("SET temp_directory='/tmp/windmill_duckdb_lock_probe';") - .unwrap(); - conn.execute_batch("SET lock_temp_directory=true;") - .expect("the patched engine must accept lock_temp_directory"); - - for stmt in [ - "SET temp_directory='/tmp/windmill_duckdb_elsewhere';", - "RESET temp_directory;", - "SET max_temp_directory_size='100GB';", - "RESET max_temp_directory_size;", - "SET lock_temp_directory=false;", - "RESET lock_temp_directory;", - ] { - let err = conn.execute_batch(stmt).expect_err(&format!( - "`{stmt}` must be refused once the directory is locked" - )); - assert!( - err.to_string().contains("locked"), - "expected a lock refusal for `{stmt}`, got: {err}" - ); - } - } - - // The engine loads the *prebuilt* extensions from extensions.duckdb.org, which are - // compiled against the struct layouts of the release it claims to be. A patch that - // moves a member those extensions reach does not fail — the extension reads the - // neighbouring field and LOAD blocks forever — so this is a timeout, not an assertion. - #[test] - fn prebuilt_extensions_still_load() { - let conn = duckdb::Connection::open_in_memory().unwrap(); - // Fetching the extension needs egress, which loading it back — the part this guards — - // does not. Skipping keeps a developer offline from seeing a failure they can't act on, - // but never in CI, where a skip and a pass would be indistinguishable. - if let Err(e) = conn.execute_batch("INSTALL httpfs;") { - assert!( - std::env::var_os("CI").is_none(), - "could not install httpfs, so the extension-load guard did not run: {e}" - ); - eprintln!("skipping: INSTALL httpfs failed ({e})"); - return; - } - let (tx, rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - tx.send( - conn.execute_batch("LOAD httpfs;") - .map_err(|e| e.to_string()), - ) - }); - match rx.recv_timeout(std::time::Duration::from_secs(120)) { - Ok(r) => r.expect("could not load httpfs"), - Err(_) => panic!( - "LOAD httpfs did not return: the patched engine's struct layout no longer matches \ - the release the prebuilt extensions were built against" - ), - } - } -} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 7c4cce6262..2ed6d034b2 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4661,7 +4661,9 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { // A script gets its new version from the worker, once the generated lock is known // to differ from the live version's: minting one here would deploy, and walk the // importers of, a version whose lock turns out byte-identical to its parent's. - JobKind::Dependencies => *base_hash, + // An app is relocked in place: a raw app's bundle is stored under its version id, + // so a copy minted here could never be published. + JobKind::Dependencies | JobKind::AppDependencies => *base_hash, JobKind::FlowDependencies => { sqlx::query_scalar!( "INSERT INTO flow_version @@ -4679,18 +4681,6 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { .fetch_one(db) .await? } - JobKind::AppDependencies => { - sqlx::query_scalar!( - "INSERT INTO app_version - (app_id, value, created_by, raw_app) - SELECT app_id, value, created_by, raw_app - FROM app_version WHERE id = $1 - RETURNING id", - *base_hash - ) - .fetch_one(db) - .await? - } _ => *base_hash, }; @@ -5590,8 +5580,6 @@ pub async fn get_result_and_success_by_id_from_flow( }, }; - let mut completed = false; - if job_result.is_none() { job_result = match get_result_by_id_from_original_flow_inner(db, w_id, flow_id, node_id).await { @@ -5601,7 +5589,6 @@ pub async fn get_result_and_success_by_id_from_flow( _ => return Err(err), }, }; - completed = job_result.is_some(); } let job_result = not_found_if_none( @@ -5621,32 +5608,37 @@ pub async fn get_result_and_success_by_id_from_flow( .fetch_one(db) .await? } - JobResult::ListJob(_) => { - let query = format!( - r#"WITH modules AS ( - SELECT jsonb_array_elements(flow_status->'modules') AS module - FROM {} - WHERE id = $1 - ) - SELECT module->>'type' = 'Success' - FROM modules - WHERE module->>'id' = $2"#, - if completed { - "v2_job_completed" - } else { - "v2_job_status" + JobResult::ListJob(jobs) => { + // The step's status lives in the flow job that ran it, which is the root only for a + // top-level step: one nested in a branch is a module of that branch's job. An empty + // container has no job to start from, so only a top-level one resolves; a nested one + // falls back to the flow's final result. + let holder = match jobs.first() { + Some(job) => { + sqlx::query_scalar!("SELECT parent_job FROM v2_job WHERE id = $1", job) + .fetch_optional(db) + .await? + .flatten() } - ); - sqlx::query_scalar(&query) - .bind(flow_id) - .bind(node_id) - .fetch_optional(db) - .await? - .ok_or_else(|| { - error::Error::internal_err(format!( - "Could not get success from flow job status" - )) - })? + None => None, + } + .unwrap_or(*flow_id); + // That job can complete before the root does, so its status may be in either table. + sqlx::query_scalar( + r#"SELECT module->>'type' = 'Success' + FROM jsonb_array_elements(COALESCE( + (SELECT flow_status FROM v2_job_completed WHERE id = $1), + (SELECT flow_status FROM v2_job_status WHERE id = $1) + )->'modules') AS module + WHERE module->>'id' = $2"#, + ) + .bind(holder) + .bind(node_id) + .fetch_optional(db) + .await? + .ok_or_else(|| { + error::Error::internal_err(format!("Could not get success from flow job status")) + })? } }; diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 81be3cb7e8..0041dceea0 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -175,15 +175,20 @@ pub async fn push_scheduled_job<'c>( // Scheduled events must be stored in the database in UTC let next = next.with_timezone(&chrono::Utc); let already_exists: bool = sqlx::query_scalar!( - // Query plan: - // - use of the `ix_v2_job_root_by_path` index; hence the `parent_job IS NULL` clause. - // - select from `v2_job` first, then join with `v2_job_queue` to avoid a full table scan - // on `scheduled_for = $3`. + // Must be driven by `v2_job_queue` (via `v2_job_queue_workspace_scheduled_for`), never + // by a `v2_job` path index, which walks the schedule's whole retained history (stats + // underestimate a single path's rows). The lateral reference forces the join order and + // `OFFSET 0` keeps the filters out of the subquery, so `v2_job` is only ever probed by + // primary key. "SELECT EXISTS ( - SELECT 1 FROM v2_job j JOIN v2_job_queue USING (id) - WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2 AND runnable_path = $4 - AND parent_job IS NULL - AND scheduled_for = $3 + SELECT 1 FROM v2_job_queue q + CROSS JOIN LATERAL ( + SELECT trigger_kind, trigger, runnable_path, parent_job + FROM v2_job WHERE id = q.id OFFSET 0 + ) j + WHERE q.workspace_id = $1 AND q.scheduled_for = $3 + AND j.trigger_kind = 'schedule' AND j.trigger = $2 AND j.runnable_path = $4 + AND j.parent_job IS NULL )", &schedule.workspace_id, &schedule.path, diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index eeea790009..0940e91d98 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -20,7 +20,7 @@ use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::ssrf::{ is_private_ip, private_git_host_allowed, private_git_host_hint, GitRemoteCaller, }; -use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; +use windmill_common::workspaces::{check_deploy_rules, operator_can_draft, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; use crate::var_resource_cache::{auth_identity, cache_resource, get_cached_resource}; @@ -261,6 +261,8 @@ pub struct ListResourceQuery { /// When true, append per-user draft-only rows; picker callers leave it off /// to stay deployed-only. See list synthesis in scripts.rs. pub include_draft_only: Option, + /// Only the `ai_agent` resources the authed user starred. + pub starred_only: Option, } #[derive(Serialize, FromRow)] @@ -437,6 +439,14 @@ async fn list_resources( ); } + if lq.starred_only.unwrap_or(false) { + sqlb.and_where( + "EXISTS(SELECT 1 FROM favorite WHERE favorite.workspace_id = resource.workspace_id \ + AND favorite.path = resource.path AND favorite.favorite_kind = 'agent' AND favorite.usr = ?)" + .bind(&authed.username), + ); + } + if let Some(label) = &lq.label { for l in label.split(',') { sqlb.and_where( @@ -464,7 +474,6 @@ async fn list_resources( // list (the resources page always passes `resource_type_exclude`); they're applied // per-row below against the draft JSON's `resource_type` instead. if lq.include_draft_only.unwrap_or(false) - && !authed.is_operator && offset == 0 && lq.path_start.is_none() && lq.path.is_none() @@ -472,6 +481,9 @@ async fn list_resources( && lq.value.is_none() && lq.broad_filter.is_none() && lq.label.is_none() + && !lq.starred_only.unwrap_or(false) + && (!authed.is_operator + || operator_can_draft(&db, &w_id, UserDraftItemKind::Resource).await?) { let rt_filter: Option> = lq .resource_type diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index 151b75bd85..fca29736e6 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -11,7 +11,7 @@ use windmill_api_auth::{ ApiAuthed, }; use windmill_common::db::{Authable, DB}; -use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; +use windmill_common::workspaces::{check_deploy_rules, operator_can_draft, RuleCheckResult}; use crate::secret_backend_ext::{ delete_secret_from_backend, get_secret_value, is_external_stored_value, is_vault_stored_value, @@ -229,7 +229,6 @@ async fn list_variables( // Append the authed user's draft-only variables; see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && !authed.is_operator && offset == 0 && lq.path_start.is_none() && lq.path.is_none() @@ -237,6 +236,8 @@ async fn list_variables( && lq.value.is_none() && lq.broad_filter.is_none() && lq.label.is_none() + && (!authed.is_operator + || operator_can_draft(&db, &w_id, UserDraftItemKind::Variable).await?) { let draft_only_rows = fetch_draft_only_list_rows(&db, &w_id, &authed.email, UserDraftItemKind::Variable) diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index ba40ed2c48..7395a5e8a3 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -695,12 +695,18 @@ async fn list_triggers( // Best-effort: the editor's TriggerData shape overlaps T::Trigger but a per-kind // config can deviate, so drop a row on deserialize failure rather than fail the list. if query.include_draft_only.unwrap_or(false) - && !authed.is_operator && query.page.unwrap_or(0) == 0 && query.path.is_none() && query.is_flow.is_none() && query.path_start.is_none() && query.label.is_none() + && (!authed.is_operator + || windmill_common::workspaces::operator_can_draft( + &db, + &workspace_id, + T::user_draft_item_kind(), + ) + .await?) { let draft_only_rows = fetch_draft_only_list_rows( &db, diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 57b292f8f2..68728a9713 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -39,7 +39,7 @@ nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] ruby = ["dep:windmill-parser-ruby"] rlang = ["dep:windmill-parser-r"] -duckdb = ["dep:libloading"] +duckdb = [] quickjs = ["windmill-jseval/quickjs", "windmill-queue/quickjs"] bedrock = ["windmill-ai/bedrock"] @@ -143,7 +143,6 @@ backon.workspace = true pep440_rs.workspace = true process-wrap.workspace = true async-once-cell.workspace = true -libloading = { workspace = true, optional = true } opentelemetry-proto = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index edfd3375ab..a36e8e136f 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -41,7 +41,10 @@ const p = { imp.path.startsWith("/f/")) && !imp.path.endsWith(".ts") ) { - code = code.replaceAll(imp.path, imp.path + ".ts"); + // Whole quoted specifiers only: `./util` is also a substring of `../util_ws`. + for (const q of ['"', "'"]) { + code = code.replaceAll(q + imp.path + q, q + imp.path + ".ts" + q); + } } } } @@ -55,6 +58,17 @@ const p = { return replaceRelativeImports(code); }); + // Module files (written by write_module_files next to main.ts) need the same `.ts` + // rewrite: an extensionless relative import misses the resolve filter below, and the + // lockfile builder then treats it as an external package named ".". + const filterLoadModule = new RegExp( + `^${cdir}\/(?!node_modules\/)(?!_wm_)(?!main\\.ts$).*\\.ts$` + ); + build.onLoad({ filter: filterLoadModule }, async (args) => { + const code = readFileSync(args.path, "utf8"); + return replaceRelativeImports(code); + }); + // A stalled fetch would otherwise hold the whole build for bun's own 5-minute // default, with nothing naming the script it was waiting on. const RELATIVE_IMPORT_FETCH_TIMEOUT_MS = 120000; @@ -109,18 +123,32 @@ const p = { // Check if the import resolves to a local module file (written by write_module_files). // Only check relative paths — absolute/bare specifiers should fall through to the // remote resolver, matching the Windows loader pattern. - if (args.path.startsWith(".")) { - const localPath = resolve(cdir, args.path); + // Only for local importers (main.ts or a module, both `.ts` in the job dir): a fetched + // workspace script (`.url`) resolves within its own workspace folder, where a module + // file of the same name would shadow its real neighbour. + const importer = args.importer ? resolve(args.importer) : ""; + if ( + args.path.startsWith(".") && + importer.endsWith(".ts") && + importer.startsWith(cdir + "/") + ) { + // From the importing module's own folder: a module may sit in a subfolder (`lib/a.ts`). + const localPath = resolve(dirname(importer), args.path); try { readFileSync(localPath); return { path: localPath }; } catch {} } + const importerRel = args.importer.replace(cdir + "/", ""); + // A module file stands in the job dir for a script in the entrypoint's folder, so its + // own relative imports resolve from that folder's workspace path, not from the job dir. const file_path = args.importer == "./main.ts" || args.importer == resolve("./main.ts") ? current_path - : args.importer.replace(cdir + "/", ""); + : importerRel.endsWith(".ts") + ? join(dirname(current_path), importerRel.replace(/\.ts$/, "")) + : importerRel; const isRelative = !args.path.startsWith("/"); const endExt = args.path.endsWith(".ts") ? "" : ".ts"; diff --git a/backend/windmill-worker/nsjail/download.ruby.config.proto b/backend/windmill-worker/nsjail/download.ruby.config.proto index 4d6b398fda..31eefae766 100644 --- a/backend/windmill-worker/nsjail/download.ruby.config.proto +++ b/backend/windmill-worker/nsjail/download.ruby.config.proto @@ -10,6 +10,10 @@ rlimit_cpu: 1000 rlimit_fsize: 1024 rlimit_nofile: 64 +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} + envar: "HOME=/user" envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 43e4464beb..cd1510adc6 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -20,6 +20,9 @@ clone_newcgroup: false skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount { src: "/bin" diff --git a/backend/windmill-worker/nsjail/run.csharp.config.proto b/backend/windmill-worker/nsjail/run.csharp.config.proto index c624e55c43..cf2d4f7f0e 100644 --- a/backend/windmill-worker/nsjail/run.csharp.config.proto +++ b/backend/windmill-worker/nsjail/run.csharp.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount_proc: true mount { diff --git a/backend/windmill-worker/nsjail/run.duckdb.config.proto b/backend/windmill-worker/nsjail/run.duckdb.config.proto new file mode 100644 index 0000000000..426f3d583d --- /dev/null +++ b/backend/windmill-worker/nsjail/run.duckdb.config.proto @@ -0,0 +1,145 @@ +name: "duckdb run" + +mode: ONCE +hostname: "duckdb" +log_level: ERROR +# The executor's own job timeout kills the jail; nsjail adds none of its own. +time_limit: 0 + +# DuckDB bounds its memory through memory_limit, and the worker bounds its disk use (see the +# work dir mount). rlimit_as would break it: jemalloc reserves address space well past what it +# ever touches. +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/proc/self/fd" + dst: "/dev/fd" + is_symlink: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +# DuckDB sizes its thread pool from the cgroup's CPU quota. +mount { + src: "/sys/fs/cgroup" + dst: "/sys/fs/cgroup" + is_bind: true + mandatory: false +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +{TMP_MOUNT_BLOCK} + +# At its host path, so every path the worker wrote into the blocks (Postgres root +# certificates) resolves unchanged. Read-only: the one writable place on the host is the +# runner's work dir below, which the worker caps at DUCKDB_MAX_TEMP_DIRECTORY_SIZE. +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true +} + +# The spill directory and the result file. +mount { + src: "{JOB_DIR}/duckdb" + dst: "{JOB_DIR}/duckdb" + is_bind: true + rw: true +} + +mount { + src: "{RUNNER_PATH}" + dst: "/tmp/windmill_duckdb_runner" + is_bind: true +} + +# Read-only: the worker installs into it before the jail starts, so a job can load what it +# named but can neither add to nor replace what the next job loads. +mount { + src: "{EXTENSION_DIR}" + dst: "/tmp/.duckdb/extensions" + is_bind: true + mandatory: false +} + +{BIGQUERY_CREDENTIALS_MOUNT} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto index 1e6de06c9c..691707b22f 100644 --- a/backend/windmill-worker/nsjail/run.go.config.proto +++ b/backend/windmill-worker/nsjail/run.go.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount { src: "/bin" diff --git a/backend/windmill-worker/nsjail/run.java.config.proto b/backend/windmill-worker/nsjail/run.java.config.proto index 9ae43f5b52..b53758c3a3 100644 --- a/backend/windmill-worker/nsjail/run.java.config.proto +++ b/backend/windmill-worker/nsjail/run.java.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount_proc: true mount { diff --git a/backend/windmill-worker/nsjail/run.nu.config.proto b/backend/windmill-worker/nsjail/run.nu.config.proto index 6c517a5b39..91174f920c 100644 --- a/backend/windmill-worker/nsjail/run.nu.config.proto +++ b/backend/windmill-worker/nsjail/run.nu.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount_proc: true mount { diff --git a/backend/windmill-worker/nsjail/run.php.config.proto b/backend/windmill-worker/nsjail/run.php.config.proto index 6910cbe1a0..3156d6a71c 100644 --- a/backend/windmill-worker/nsjail/run.php.config.proto +++ b/backend/windmill-worker/nsjail/run.php.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount_proc: true mount { diff --git a/backend/windmill-worker/nsjail/run.r.config.proto b/backend/windmill-worker/nsjail/run.r.config.proto index bc4d74bcba..a0c4c51cf0 100644 --- a/backend/windmill-worker/nsjail/run.r.config.proto +++ b/backend/windmill-worker/nsjail/run.r.config.proto @@ -14,6 +14,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} # mount_proc: true mount { diff --git a/backend/windmill-worker/nsjail/run.ruby.config.proto b/backend/windmill-worker/nsjail/run.ruby.config.proto index 2527e785b6..f1a642e0b8 100644 --- a/backend/windmill-worker/nsjail/run.ruby.config.proto +++ b/backend/windmill-worker/nsjail/run.ruby.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} # mount_proc: true mount { diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto index 070a76e997..2b60847f98 100644 --- a/backend/windmill-worker/nsjail/run.rust.config.proto +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -15,6 +15,9 @@ clone_newuser: {CLONE_NEWUSER} skip_setsid: true keep_caps: false keep_env: true +# The script's env vars, proto-escaped. Set here rather than on the process that +# execs nsjail: the loader honours `LD_PRELOAD` before the jail exists. +{ENVARS} mount { src: "/bin" diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 1b889aea34..01ac2becdd 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -260,6 +260,49 @@ fn keep_authored_memory_id( } } +/// A step's own `model` and `reasoning_effort` inputs replace the provider's, which is how a step +/// linked to a saved agent picks them per run. An empty model keeps the provider's, since a run +/// needs one; an empty effort is no effort, which is how a model that reasons only on request is +/// turned off. A reasoning token is per model, so a different model without an effort of its own +/// runs at its default. +/// Only an input the step authored counts: the arguments of an agent run as a tool also carry +/// whatever the calling model sent, which must not choose the model it runs on. +fn override_provider_model( + provider: &mut ProviderWithResource, + local_args: &HashMap>, + step_input_transforms: &HashMap, +) { + let authored = |key: &str| { + step_input_transforms + .get(key) + .filter(|t| { + matches!( + t, + InputTransform::Javascript { .. } | InputTransform::Static { .. } + ) + }) + .map(|t| match t { + // A static null is not evaluated for a tool, which leaves the caller's argument + // in its place. + InputTransform::Static { value } if value.get() == "null" => String::new(), + _ => local_args + .get(key) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_default(), + }) + }; + if let Some(model) = authored("model").filter(|m| !m.is_empty()) { + if model != provider.model { + provider.model = model; + provider.reasoning_effort = None; + } + } + if let Some(effort) = authored("reasoning_effort") { + provider.reasoning_effort = Some(effort).filter(|e| !e.is_empty()); + } +} + /// Reconciles the step's history inputs, the agent's memory policy and the run's memory id. A step /// holds one of two shapes: an older `auto` or `manual` memory, read as the editor that wrote it /// meant it, or the current setting plus the step's own history inputs. Also returns lines for the @@ -854,6 +897,7 @@ pub async fn handle_ai_agent_job( }; keep_authored_memory_id(&mut args, &step_input_transforms); + override_provider_model(&mut args.provider, &local_args, &step_input_transforms); // Nesting is capped at flow → agent → nested agent. When this job is itself a nested tool, // a linked resource's tool set may still contain AIAgent or AIDecision tools (the editor can't @@ -2940,6 +2984,70 @@ mod tests { assert!(bounded_prompt_cache_key(&long).len() <= 64); } + #[test] + fn step_model_and_effort_replace_the_providers() { + use serde_json::json; + let run = |step: serde_json::Value| { + let mut provider: ProviderWithResource = serde_json::from_value(json!({ + "kind": "openai", + "resource": {}, + "model": "saved-model", + "reasoning_effort": "high" + })) + .unwrap(); + let local_args: HashMap> = serde_json::from_value(step).unwrap(); + let authored = local_args + .keys() + .map(|key| { + ( + key.clone(), + InputTransform::Javascript { expr: String::new() }, + ) + }) + .collect(); + override_provider_model(&mut provider, &local_args, &authored); + (provider.model, provider.reasoning_effort) + }; + let saved = ("saved-model".to_string(), Some("high".to_string())); + + // An argument the step did not author is the calling model's, on an agent run as a tool. + let mut provider: ProviderWithResource = serde_json::from_value( + json!({ "kind": "openai", "resource": {}, "model": "saved-model" }), + ) + .unwrap(); + let sent = serde_json::from_value(json!({ "model": "other" })).unwrap(); + override_provider_model(&mut provider, &sent, &HashMap::new()); + assert_eq!(provider.model, "saved-model"); + let null = HashMap::from([( + "model".to_string(), + InputTransform::Static { value: to_raw_value(&serde_json::Value::Null) }, + )]); + override_provider_model(&mut provider, &sent, &null); + assert_eq!(provider.model, "saved-model"); + + assert_eq!(run(json!({})), saved); + assert_eq!(run(json!({ "model": "" })), saved); + assert_eq!(run(json!({ "model": "saved-model" })), saved); + // An effort the step sets to nothing is no effort, not the saved one. + assert_eq!( + run(json!({ "reasoning_effort": null })), + ("saved-model".to_string(), None) + ); + assert_eq!( + run(json!({ "reasoning_effort": "low" })), + ("saved-model".to_string(), Some("low".to_string())) + ); + // The saved effort names a level of the saved model, so it does not follow to another one. + assert_eq!( + run(json!({ "model": "other" })), + ("other".to_string(), None) + ); + assert_eq!( + run(json!({ "model": "other", "reasoning_effort": "low" })), + ("other".to_string(), Some("low".to_string())) + ); + } + #[test] fn overlay_tool_inputs_binds_matching_flowmodule_tool_only() { fn js(expr: &str) -> InputTransform { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index de33a2b9e6..201219b674 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -21,8 +21,8 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, resolve_nsjail_timeout, - resolve_nsjail_tmp_mount_block, start_child_process, write_file_binary, MaybeLock, - OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, + resolve_nsjail_tmp_mount_block, script_envars, start_child_process, write_file_binary, + MaybeLock, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -1542,6 +1542,26 @@ pub async fn compute_bundle_local_and_remote_path( input_src.push_str(&path); input_src.push_str(&hash.to_string()); } + // Modules are inlined too, and so is every deployed script they import: those are + // reached through the module files, never through the entrypoint's own imports. + let dir = script_path + .rsplit_once('/') + .map(|(dir, _)| dir) + .unwrap_or(""); + let mut module_entries = modules.into_iter().flatten().collect_vec(); + module_entries.sort_by(|a, b| a.0.cmp(b.0)); + for (key, module) in module_entries { + let Some(stem) = key.strip_suffix(".ts") else { + continue; + }; + let module_path = format!("{dir}/{stem}"); + for (path, hash) in + collect_transitive_import_versions(db, w_id, &module_path, &module.content).await + { + input_src.push_str(&path); + input_src.push_str(&hash.to_string()); + } + } }; // Keep temp-script-ref (preview) bundles in a distinct cache slot: their @@ -2474,35 +2494,6 @@ try {{ let child = if is_sandboxing_enabled() || annotation.sandbox { let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_BUN_CONTENT - .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" }) - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{UIDGIDMAP}", if *DISABLE_NUSER { "" } else { - "uidmap {\n inside_id: \"1000\"\n outside_id: \"\"\n count: 1\n}\n\ngidmap {\n inside_id: \"1000\"\n outside_id: \"\"\n count: 1\n}" - }) - .replace( - "{SHARED_MOUNT}", - &shared_mount.replace( - "/tmp/shared", - if annotation.nodejs { - "/tmp/nodejs/shared" - } else { - "/tmp/bun/shared" - }, - ), - ) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); let args = if annotation.nodejs { @@ -2541,7 +2532,6 @@ try {{ nsjail_cmd .current_dir(job_dir) .env_clear() - .envs(envs) .envs(reserved_variables) .envs( get_proxy_envs_for_lang( @@ -2558,6 +2548,36 @@ try {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_BUN_CONTENT + .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" }) + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{UIDGIDMAP}", if *DISABLE_NUSER { "" } else { + "uidmap {\n inside_id: \"1000\"\n outside_id: \"\"\n count: 1\n}\n\ngidmap {\n inside_id: \"1000\"\n outside_id: \"\"\n count: 1\n}" + }) + .replace( + "{SHARED_MOUNT}", + &shared_mount.replace( + "/tmp/shared", + if annotation.nodejs { + "/tmp/nodejs/shared" + } else { + "/tmp/bun/shared" + }, + ), + ) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{ENVARS}", &script_envars(&envs, &nsjail_cmd, &[])), + )?; start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { let cmd = if annotation.nodejs { diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 50af562062..2fa9a7a5ff 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1314,6 +1314,84 @@ pub fn render_nsjail_rlimit_as(env_override: Option<&str>, default_mb: u32) -> S } } +/// Quote a string as a protobuf-text-format string literal for safe inclusion in +/// an nsjail config. Every byte is emitted as a printable ASCII char or a valid +/// protobuf escape (`\"`, `\\`, `\n`/`\r`/`\t`, or 3-digit octal `\NNN` for +/// control/non-ASCII bytes), so a hostile value can neither break the literal nor +/// inject further directives. +/// +/// `{` and `}` are escaped too: profiles are rendered by substituting `{PLACEHOLDER}` +/// tokens one after another, so a value that still contained braces could smuggle in a +/// later placeholder's expansion — and `{ENVARS}` expands to unquoted directives. +pub(crate) fn proto_str(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for &b in s.as_bytes() { + match b { + b'"' => out.push_str("\\\""), + b'\\' => out.push_str("\\\\"), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + b'\t' => out.push_str("\\t"), + b'{' | b'}' => out.push_str(&format!("\\{b:03o}")), + 0x20..=0x7e => out.push(b as char), + _ => out.push_str(&format!("\\{b:03o}")), + } + } + out.push('"'); + out +} + +/// Render env vars as nsjail `envar:` directives (one per line), the only way +/// caller-controlled env may reach a jailed child. Set on the process that execs +/// nsjail instead, `LD_PRELOAD`/`LD_LIBRARY_PATH`/`LD_AUDIT` would be honoured by +/// the loader for nsjail itself, before any namespace or chroot exists. +pub(crate) fn render_envars<'a>(env: impl IntoIterator) -> String { + env.into_iter() + .map(|(k, v)| format!("envar: {}", proto_str(&format!("{k}={v}")))) + .collect::>() + .join("\n") +} + +/// The script's env vars as `envar:` directives for its nsjail run profile. +/// +/// The profile sets `keep_env`, so the child also inherits the launcher's env, and +/// an `envar:` overrides an inherited key. A key the launcher sets therefore keeps +/// the launcher's value, as it did when both were set on the launcher with the +/// script's env first, unless it is listed in `script_overrides` (see +/// [`script_overrides`]). +pub(crate) fn script_envars( + envs: &HashMap, + launcher: &Command, + script_overrides: &[&str], +) -> String { + let launcher_keys: HashSet<&std::ffi::OsStr> = + launcher.as_std().get_envs().map(|(k, _)| k).collect(); + render_envars( + envs.iter() + .filter(|(k, _)| { + script_overrides.contains(&k.as_str()) + || !launcher_keys.contains(std::ffi::OsStr::new(k.as_str())) + }) + .map(|(k, v)| (k.as_str(), v.as_str())), + ) +} + +/// The keys an executor sets on its nsjail launcher before the script's env, so the +/// script's value wins, less those `reserved_variables` (applied after the script's env, +/// and carrying worker-group env vars) sets again, so the worker's value wins. +#[cfg(any(feature = "nu", feature = "java", feature = "ruby", feature = "rlang"))] +pub(crate) fn script_overrides<'a>( + set_before: &[&'a str], + reserved_variables: &HashMap, +) -> Vec<&'a str> { + set_before + .iter() + .copied() + .filter(|k| !reserved_variables.contains_key(*k)) + .collect() +} + /// Default size (in bytes) of the `/tmp` tmpfs mount inside nsjail sandboxes, /// used when the `nsjail_tmpfs_size_mb` instance setting is unset. pub const DEFAULT_NSJAIL_TMPFS_SIZE_BYTES: u64 = 800_000_000; @@ -2074,15 +2152,18 @@ pub struct StreamNotifier { parent_job: uuid::Uuid, root_job: uuid::Uuid, flow_step_id: Option, + flow_scope: Option, } // Helper struct to hold parent job information #[derive(Debug)] struct JobInfo { flow_step_id: Option, + flow_scope: Option, step: Option, len: Option, is_branch_one: Option, + is_branch: Option, next_parent: Option, } @@ -2092,9 +2173,11 @@ async fn get_job_info(db: &DB, job_id: Uuid) -> error::Result { JobInfo, r#"SELECT flow_step_id, + flow_innermost_root_job as flow_scope, (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len, runnable_path ~ '/branchone-\d+$' as is_branch_one, + runnable_path ~ '/branch(one|all)-\d+$' as is_branch, parent_job as next_parent FROM v2_job LEFT JOIN v2_job_status USING (id) @@ -2142,21 +2225,20 @@ async fn check_if_last_step( } // Iterative implementation to avoid stack overflow from async_recursion -// Checks if we're at last step in nested branches AND if any parent's flow_step_id matches early_return_id +// True when the job is the early-return step, or the last step of branches to one that end in it. +// Step ids are only unique within one flow, so an id match counts when the job's innermost root +// is the root flow: a step of a subflow reusing the id has the subflow as its innermost root. async fn check_if_early_return_or_last_in_early_return_parent( db: &DB, mut next_parent: Option, mut step_id: Option, + mut flow_scope: Option, early_return_id: &str, root_job: Uuid, ) -> error::Result { let mut visited = HashSet::new(); loop { - if step_id.is_none() { - return Ok(false); - } - - let Some(parent_job) = next_parent else { + let (Some(step), Some(parent_job)) = (step_id.as_deref(), next_parent) else { return Ok(false); }; @@ -2165,25 +2247,49 @@ async fn check_if_early_return_or_last_in_early_return_parent( return Ok(false); } - // If the parent's flow_step_id matches early_return_id, we found it! - if step_id.as_deref() == Some(early_return_id) && parent_job == root_job { - return Ok(true); - } else { - let parent_info = get_job_info(db, parent_job).await?; - if let Some(step) = parent_info.step { - let step = Step::from_i32_and_len(step, parent_info.len.unwrap_or(0) as usize); - // we only continue if we are at the last step and the parent is a branch one - if step.is_last_step() && parent_info.is_branch_one.unwrap_or(false) { - next_parent = parent_info.next_parent; - step_id = parent_info.flow_step_id; - continue; - } - } + if step == early_return_id && flow_scope == Some(root_job) { + return only_branches_up_to_root(db, parent_job, root_job).await; + } + if parent_job == root_job { return Ok(false); } + + let parent_info = get_job_info(db, parent_job).await?; + if let Some(parent_step) = parent_info.step { + let parent_step = + Step::from_i32_and_len(parent_step, parent_info.len.unwrap_or(0) as usize); + // we only continue if we are at the last step and the parent is a branch one + if parent_step.is_last_step() && parent_info.is_branch_one.unwrap_or(false) { + next_parent = parent_info.next_parent; + step_id = parent_info.flow_step_id; + flow_scope = parent_info.flow_scope; + continue; + } + } + return Ok(false); } } +// A step in a loop shares the root flow's scope but runs once per iteration, so no single run +// holds the flow's early-return result: every container between it and the root must be a branch. +async fn only_branches_up_to_root(db: &DB, mut job: Uuid, root_job: Uuid) -> error::Result { + let mut visited = HashSet::new(); + while job != root_job { + if !visited.insert(job) { + return Ok(false); + } + let info = get_job_info(db, job).await?; + if !info.is_branch.unwrap_or(false) { + return Ok(false); + } + let Some(parent) = info.next_parent else { + return Ok(false); + }; + job = parent; + } + Ok(true) +} + impl StreamNotifier { pub fn new(conn: &Connection, job: &MiniPulledJob) -> Option { let root_job = get_root_job_id(job); @@ -2195,6 +2301,7 @@ impl StreamNotifier { job_id: job.id, root_job, flow_step_id: job.flow_step_id.clone(), + flow_scope: job.flow_innermost_root_job, }), Connection::Http(_) => { tracing::warn!( @@ -2214,6 +2321,7 @@ impl StreamNotifier { job_id: Uuid, root_job: Uuid, flow_step_id: Option, + flow_scope: Option, ) -> Result<(), Error> { // Check if early_return is set at the flow level let early_return_node_id = sqlx::query_scalar!( @@ -2229,26 +2337,33 @@ impl StreamNotifier { .await? .flatten(); - let should_set_stream_job = if let Some(ref early_return_id) = early_return_node_id { + let is_early_return = if let Some(ref early_return_id) = early_return_node_id { check_if_early_return_or_last_in_early_return_parent( &db, Some(parent_job), flow_step_id, + flow_scope, early_return_id, root_job, ) .await? } else { - check_if_last_step(&db, Some(parent_job), root_job).await? + false }; - if should_set_stream_job { + // An early-return step inside a branch that is not taken never runs, so the flow's last + // step streams in its place. One that did run has a leaf job on the root, streamed or not, + // and the last step must not take the stream from it. + let unless_ran = early_return_node_id.filter(|_| !is_early_return); + + if is_early_return || check_if_last_step(&db, Some(parent_job), root_job).await? { sqlx::query!(r#" UPDATE v2_job_status SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT)) - WHERE id = $2"#, + WHERE id = $2 AND NOT COALESCE(flow_leaf_jobs ? $3, false)"#, job_id, - root_job + root_job, + unless_ran ) .execute(&db) .await?; @@ -2263,6 +2378,7 @@ impl StreamNotifier { let job_id = self.job_id; let root_job = self.root_job; let flow_step_id = self.flow_step_id.clone(); + let flow_scope = self.flow_scope; tokio::spawn(async move { if let Err(err) = Self::update_flow_status_with_stream_job_inner( db, @@ -2270,6 +2386,7 @@ impl StreamNotifier { job_id, root_job, flow_step_id, + flow_scope, ) .await { @@ -2636,6 +2753,78 @@ mod tests { assert!(result.is_err()); } + + // A profile that loses `{ENVARS}` silently drops the script's env, since a + // `replace` on a missing token does nothing. It must also precede every fixed + // `envar:`, which overrides it the way it overrode the launcher's env. + #[test] + fn script_profiles_take_envars_before_their_fixed_ones() { + for (name, profile) in [ + ("go", include_str!("../nsjail/run.go.config.proto")), + ("rust", include_str!("../nsjail/run.rust.config.proto")), + ("bun", include_str!("../nsjail/run.bun.config.proto")), + ("nu", include_str!("../nsjail/run.nu.config.proto")), + ("php", include_str!("../nsjail/run.php.config.proto")), + ("csharp", include_str!("../nsjail/run.csharp.config.proto")), + ("java", include_str!("../nsjail/run.java.config.proto")), + ("ruby", include_str!("../nsjail/run.ruby.config.proto")), + ("r", include_str!("../nsjail/run.r.config.proto")), + ( + "gem install", + include_str!("../nsjail/download.ruby.config.proto"), + ), + ] { + let at = profile + .find("\n{ENVARS}\n") + .unwrap_or_else(|| panic!("{name}: no {{ENVARS}}")); + assert!( + !profile[..at].contains("\nenvar:"), + "{name}: a fixed envar precedes {{ENVARS}}" + ); + } + } + + #[test] + fn script_envars_reach_the_jail_not_the_launcher() { + let envs = HashMap::from([ + ("LD_PRELOAD".to_string(), "/tmp/evil.so".to_string()), + ("WM_TOKEN".to_string(), "forged".to_string()), + ("PATH".to_string(), "/custom/bin".to_string()), + ]); + let mut launcher = Command::new("nsjail"); + launcher + .env_clear() + .env("PATH", "/usr/bin") + .env("WM_TOKEN", "real"); + + let out = script_envars(&envs, &launcher, &["PATH"]); + let mut lines = out.lines().collect::>(); + lines.sort(); + // A launcher key keeps the launcher's value unless the executor lets the + // script override it. + assert_eq!( + lines, + vec![ + "envar: \"LD_PRELOAD=/tmp/evil.so\"", + "envar: \"PATH=/custom/bin\"", + ] + ); + assert!(launcher + .as_std() + .get_envs() + .all(|(k, _)| k != std::ffi::OsStr::new("LD_PRELOAD"))); + + let hostile = HashMap::from([( + "X".to_string(), + "v\"\nclone_newuser: false {JOB_DIR}".to_string(), + )]); + let line = script_envars(&hostile, &launcher, &[]); + assert_eq!(line.lines().count(), 1, "{line}"); + assert!( + !line.contains('{') && !line.contains("\nclone_newuser"), + "{line}" + ); + } } lazy_static::lazy_static! { diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 7d77f8703f..87213223e7 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -28,7 +28,7 @@ use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, resolve_nsjail_timeout, - resolve_nsjail_tmp_mount_block, start_child_process, DEV_CONF_NSJAIL, + resolve_nsjail_tmp_mount_block, script_envars, start_child_process, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -729,28 +729,10 @@ pub async fn handle_csharp_job( let child = if is_sandboxing_enabled() { let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_CSHARP_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", &*CSHARP_CACHE_DIR) - .replace("{CACHE_HASH}", &hash) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() - .envs(envs) .envs(reserved_variables) .envs( get_proxy_envs_for_lang( @@ -777,6 +759,24 @@ pub async fn handle_csharp_job( #[cfg(windows)] nsjail_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_CSHARP_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", &*CSHARP_CACHE_DIR) + .replace("{CACHE_HASH}", &hash) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{ENVARS}", &script_envars(&envs, &nsjail_cmd, &[])), + )?; start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), true).await? } else { #[cfg(unix)] diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 90db1b9795..dbef02bec3 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -31,7 +31,7 @@ use windmill_parser_yaml::{ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::common::{ - render_nsjail_rlimit_as, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, + render_envars, render_nsjail_rlimit_as, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, }; use crate::common::{start_child_process, OccupancyMetrics}; use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; @@ -1393,12 +1393,10 @@ pub(crate) async fn prepare_project( // invocation's. Neither may sit on the launcher. .replace( "{ENVARS}", - &jail_envars( + &render_envars( env.iter() - .map(|(k, v)| (k.clone(), v.clone())) - .chain(invocation_env.iter().map(|(k, v)| (k.clone(), v.clone()))) - .collect::>() - .as_slice(), + .map(|(k, v)| (k.as_str(), v.as_str())) + .chain(invocation_env.iter().map(|(k, v)| (k.as_str(), v.as_str()))), ), ) .replace("{SHARED_MOUNT}", "") @@ -2376,16 +2374,6 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command { cmd } -/// The project's environment, as jail directives rather than launcher -/// environment. `keep_env` passes nsjail's own environment through, so these -/// are added on top of the Windmill-controlled ones the launcher carries. -fn jail_envars(env: &[(String, String)]) -> String { - env.iter() - .map(|(k, v)| format!("envar: \"{}={}\"", escape_textproto(k), escape_textproto(v))) - .collect::>() - .join("\n") -} - const SANDBOX_PROFILE_NAME: &str = "dbt.nsjail.config.proto"; /// The written nsjail profile, kept OUTSIDE every path the jailed child can @@ -5862,7 +5850,7 @@ mod tests { }; } - let envars = jail_envars(&[("LD_PRELOAD".to_string(), hostile.to_string())]); + let envars = render_envars([("LD_PRELOAD", hostile)]); assert_eq!(envars.lines().count(), 1, "{envars}"); assert!(envars.starts_with("envar: \"LD_PRELOAD="), "{envars}"); } diff --git a/backend/windmill-worker/src/docker_v2.rs b/backend/windmill-worker/src/docker_v2.rs index 52d1060b67..3f47ab6efc 100644 --- a/backend/windmill-worker/src/docker_v2.rs +++ b/backend/windmill-worker/src/docker_v2.rs @@ -32,8 +32,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - build_args_map, get_reserved_variables, raw_to_string, read_and_check_file, - resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, + build_args_map, get_reserved_variables, proto_str, raw_to_string, read_and_check_file, + render_envars, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -163,36 +163,6 @@ struct OciConfig { working_dir: Option, } -/// Quote a string as a protobuf-text-format string literal for safe inclusion in -/// the nsjail config. Image-controlled values (mount srcs/dsts, symlink targets, -/// WorkingDir) flow into the config, so they MUST be escaped — an unescaped `"` or -/// newline would otherwise let a hostile image config inject arbitrary nsjail -/// directives and break out of the sandbox. Every byte is emitted as a printable -/// ASCII char or a valid protobuf escape (`\"`, `\\`, `\n`/`\r`/`\t`, or 3-digit -/// octal `\NNN` for control/non-ASCII bytes), so the result always parses. -/// -/// `{` and `}` are escaped too: the profile is rendered by substituting `{PLACEHOLDER}` -/// tokens one after another, so a value that still contained braces could smuggle in a -/// later placeholder's expansion — and `{ENVARS}` expands to unquoted directives. -fn proto_str(s: &str) -> String { - let mut out = String::with_capacity(s.len() + 2); - out.push('"'); - for &b in s.as_bytes() { - match b { - b'"' => out.push_str("\\\""), - b'\\' => out.push_str("\\\\"), - b'\n' => out.push_str("\\n"), - b'\r' => out.push_str("\\r"), - b'\t' => out.push_str("\\t"), - b'{' | b'}' => out.push_str(&format!("\\{b:03o}")), - 0x20..=0x7e => out.push(b as char), - _ => out.push_str(&format!("\\{b:03o}")), - } - } - out.push('"'); - out -} - /// Why `./result.json` can't be collected, phrased for the job log. const UNVERIFIABLE_DST: &str = "it cannot be proven to resolve inside the container root (a `..` component, or a \ @@ -270,16 +240,6 @@ fn render_result_mount(job_dir: &str, dst: &str) -> String { ) } -/// Render container env vars as nsjail `envar:` directives (one per line). Each -/// `KEY=VALUE` is proto-escaped, so image-controlled keys/values can neither break -/// the config nor reach nsjail's own process environment. -fn render_envars(env: &[(String, String)]) -> String { - env.iter() - .map(|(k, v)| format!("envar: {}", proto_str(&format!("{k}={v}")))) - .collect::>() - .join("\n") -} - /// Run `crane` with the optional per-job `DOCKER_CONFIG` auth dir. async fn crane(args: &[&str], auth_dir: Option<&str>) -> Result { let mut cmd = Command::new(CRANE_PATH.as_str()); @@ -795,7 +755,7 @@ pub async fn handle_docker_v2_job( if !container_env.iter().any(|(k, _)| k == "HOME") { container_env.push(("HOME".to_string(), "/root".to_string())); } - let envars = render_envars(&container_env); + let envars = render_envars(container_env.iter().map(|(k, v)| (k.as_str(), v.as_str()))); // Render the nsjail profile: dynamic per-entry rootfs binds + image WorkingDir. let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; @@ -926,9 +886,10 @@ pub async fn handle_docker_v2_job( #[cfg(test)] mod tests { use super::{ - digest_key, proto_str, ref_key, registry_qualified, render_envars, render_result_mount, - result_mount_dst, SHADOWED_DST, UNVERIFIABLE_DST, + digest_key, ref_key, registry_qualified, render_result_mount, result_mount_dst, + SHADOWED_DST, UNVERIFIABLE_DST, }; + use crate::common::{proto_str, render_envars}; #[tokio::test] async fn result_mount_dst_targets_working_dir() { @@ -1046,14 +1007,14 @@ mod tests { ("PATH".to_string(), "/usr/bin".to_string()), ("LD_PRELOAD".to_string(), "rootfs/evil.so".to_string()), ]; - let out = render_envars(&env); + let out = render_envars(env.iter().map(|(k, v)| (k.as_str(), v.as_str()))); assert_eq!( out, "envar: \"PATH=/usr/bin\"\nenvar: \"LD_PRELOAD=rootfs/evil.so\"" ); // A value trying to inject extra directives is escaped, not interpreted. let evil = vec![("X".to_string(), "v\"\nclone_newuser: false".to_string())]; - let line = render_envars(&evil); + let line = render_envars(evil.iter().map(|(k, v)| (k.as_str(), v.as_str()))); assert!(line.starts_with("envar: \"")); assert!(!line.contains("\nclone_newuser")); assert!(line.contains("\\n")); diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 513d30d181..05c35fce8d 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -1,10 +1,6 @@ -use std::cell::RefCell; use std::env; -use std::ffi::{c_char, c_uint, CStr, CString}; -use std::ptr::NonNull; use std::sync::{Arc, Mutex}; -use libloading::{Library, Symbol}; use serde::Serialize; use serde_json::value::RawValue; use serde_json::{json, Value}; @@ -14,7 +10,8 @@ use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ get_datatable_resource_from_db, get_ducklake_from_db_unchecked, - strip_fork_reserved_attach_args, DatatableAccess, DatatableConnectAs, DucklakeCatalogResourceType, + strip_fork_reserved_attach_args, DatatableAccess, DatatableConnectAs, + DucklakeCatalogResourceType, }; use windmill_common::PgDatabase; use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE; @@ -24,6 +21,7 @@ use windmill_types::s3::S3Object; use crate::agent_workers::{get_datatable_resource_from_agent_http, get_ducklake_from_agent_http}; use crate::common::{build_args_values, get_reserved_variables, OccupancyMetrics}; +use crate::duckdb_runner::{RequestMode, RunnerMode, RunnerRequest}; use crate::handle_child::run_future_with_polling_update_job_poller; #[cfg(feature = "mysql")] use crate::mysql_executor::MysqlDatabase; @@ -405,23 +403,19 @@ async fn compute_sync_alter_ddl( base_internal_url: String, w_id: String, job_dir: String, + runner: RunnerMode, ) -> Result { - let n = probe_blocks.len(); - let (result, _) = tokio::task::spawn_blocking(move || { - run_duckdb_ffi_safe( - probe_blocks.iter().map(String::as_str), - n, - job_args, - &token, - &base_internal_url, - &w_id, - &job_dir, - SqlResultCollectionStrategy::LastStatementAllRows, - ) - }) + let (result, _) = run_duckdb( + runner, + probe_blocks, + job_args, + token, + base_internal_url, + w_id, + job_dir, + SqlResultCollectionStrategy::LastStatementAllRows, + ) .await - .map_err(|e| Error::from(to_anyhow(e))) - .and_then(|r| r) .map_err(|e| { Error::ExecutionErr(format!( "on_schema_change=sync: the schema pre-pass probe failed ({e}); refusing to write" @@ -1328,6 +1322,7 @@ pub async fn do_duckdb( let token = client.token.clone(); let hidden_passwords = Arc::new(Mutex::new(Vec::::new())); + let runner = crate::duckdb_runner::runner_mode(); let result_f = async { let mut hidden_passwords = hidden_passwords.clone(); @@ -1522,10 +1517,6 @@ pub async fn do_duckdb( probe_blocks, ) .await?; - // The probe opens its own DuckDB connection, so it needs the same - // connection policy as the main query below. - #[cfg(feature = "private")] - let probe_blocks = crate::duckdb_isolation_ee::apply_duckdb_isolation(probe_blocks)?; let alter_ddl = compute_sync_alter_ddl( probe_blocks, job_args.clone(), @@ -1534,6 +1525,7 @@ pub async fn do_duckdb( client.base_internal_url.clone(), job.workspace_id.clone(), job_dir.to_string(), + runner, ) .await?; query = query.replace( @@ -1610,46 +1602,34 @@ pub async fn do_duckdb( ) .await?; - #[cfg(feature = "private")] - let query_block_list = - crate::duckdb_isolation_ee::apply_duckdb_isolation(query_block_list)?; let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); let job_dir = job_dir.to_string(); if annotations.prepare { - let result = tokio::task::spawn_blocking(move || { - prepare_duckdb_ffi_safe( - query_block_list.iter().map(String::as_str), - &token, - &base_internal_url, - &w_id, - &job_dir, - ) - }) - .await - .map_err(|e| Error::from(to_anyhow(e))) - .and_then(|r| r)?; - - return Ok(result); + return prepare_duckdb( + runner, + query_block_list, + token.clone(), + base_internal_url, + w_id, + job_dir, + ) + .await; } - let result = tokio::task::spawn_blocking(move || { - run_duckdb_ffi_safe( - query_block_list.iter().map(String::as_str), - query_block_list.len(), - job_args, - &token, - &base_internal_url, - &w_id, - &job_dir, - collection_strategy, - ) - }) - .await - .map_err(|e| Error::from(to_anyhow(e))) - .and_then(|r| r); + let result = run_duckdb( + runner, + query_block_list, + job_args, + token.clone(), + base_internal_url, + w_id, + job_dir, + collection_strategy, + ) + .await; let (result, column_order) = match result { Ok(r) => r, @@ -1857,8 +1837,6 @@ pub async fn do_duckdb( match result { Ok(result) => Ok(result), Err(e) => { - #[cfg(feature = "private")] - let e = crate::duckdb_isolation_ee::explain_error(e); // Passwords might appear in the error message — and, for the // structured data-test failure, in sampled row data read from an // attached database — so every outgoing error is sanitized here. @@ -1903,111 +1881,6 @@ pub async fn do_duckdb( } } -thread_local! { - static DUCKDB_FFI_LIB_SINGLETON: RefCell<*const DuckDbFfiLib> = RefCell::new(std::ptr::null()); -} - -struct DuckDbFfiLib { - run_duckdb_ffi: Symbol< - 'static, - unsafe extern "C" fn( - query_block_list: *const *const c_char, - query_block_list_count: usize, - job_args: *const c_char, - token: *const c_char, - base_internal_url: *const c_char, - w_id: *const c_char, - memory_limit: *const c_char, - temp_directory: *const c_char, - max_result_size: usize, - column_order_ptr: *mut *mut c_char, - collect_last_only: bool, - collect_first_row_only: bool, - ) -> *mut c_char, - >, - prepare_duckdb_ffi: Option< - Symbol< - 'static, - unsafe extern "C" fn( - query_block_list: *const *const c_char, - query_block_list_count: usize, - token: *const c_char, - base_internal_url: *const c_char, - w_id: *const c_char, - memory_limit: *const c_char, - temp_directory: *const c_char, - ) -> *mut c_char, - >, - >, - free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, -} - -impl DuckDbFfiLib { - fn get_singleton() -> Result<&'static DuckDbFfiLib> { - DUCKDB_FFI_LIB_SINGLETON.with(|cell| unsafe { - let mut singleton = cell.borrow_mut(); - if singleton.is_null() { - let lib = DuckDbFfiLib::init()?; - let boxed_lib = Box::new(lib); - let lib_ptr = Box::leak(boxed_lib); - *singleton = lib_ptr as *const _; - Ok(NonNull::new_unchecked(*singleton as *mut DuckDbFfiLib).as_ref()) - } else { - Ok(&**singleton) - } - }) - } - fn init() -> Result { - let lib = unsafe { - Library::new(if cfg!(target_os = "macos") { - "libwindmill_duckdb_ffi_internal.dylib" - } else if cfg!(target_os = "windows") { - "windmill_duckdb_ffi_internal.dll" - } else { - "libwindmill_duckdb_ffi_internal.so" - }) - .map_err(|e| { - Error::InternalErr(format!( - "Could not init duckdb. Make sure you have the latest windmill_duckdb_ffi_lib.{} alongside your binary : https://github.com/windmill-labs/windmill/releases \n{}", - if cfg!(target_os = "macos") { "dylib" } - else if cfg!(target_os = "windows") { "dll" } - else { "so" }, - e.to_string() - )) - })? - }; - - let lib = Box::leak(Box::new(lib)); - - // Version mismatch should only be possible on Windows agent workers - // We check for it because FFI interface mismatch will cause undefined behavior / crashes - unsafe { - let expected_version: c_uint = 3; - let get_version: Symbol<'static, unsafe extern "C" fn() -> c_uint> = - lib.get(b"get_version") - .map_err(|e| return Error::ExecutionErr(format!("Could not find get_version in the duckdb ffi library. If you are not using docker, consider manually upgrading windmill_duckdb_ffi_lib. {}", e.to_string())))?; - let actual_version = get_version(); - if actual_version < expected_version { - return Err(Error::InternalErr( - format!("Incompatible duckdb ffi library version. Expected: {expected_version}, actual: {actual_version}. Please update to the latest windmill_duckdb_ffi_lib."), - )); - } else if actual_version > expected_version { - return Err(Error::InternalErr( - format!("Incompatible duckdb ffi library version. Expected: {expected_version}, actual: {actual_version}. Please upgrade your worker to the latest windmill version."), - )); - } - } - - let prepare_duckdb_ffi = unsafe { lib.get(b"prepare_duckdb_ffi").ok() }; - - Ok(DuckDbFfiLib { - run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, - prepare_duckdb_ffi, - free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, - }) - } -} - // 20% headroom for Rust runtime + DuckDB's untracked allocations. Mirrors // DuckDB's own default ratio, but applied to the worker's cgroup budget // instead of host RAM. @@ -2037,123 +1910,29 @@ fn cgroup_bytes_to_duckdb_memory_limit(bytes: i64) -> Option { Some(format!("{}MiB", mib.max(64))) } -// Read backend/windmill-duckdb-ffi-internal/README_DEV.md for details about why we use FFI -// The FFI returns errors as `ERROR ` (the message is -// serde_json::to_string'd). Decode the JSON string back so multi-line errors -// render with real newlines, not escaped `\n` inside wrapping quotes; fall back -// to the raw slice if it is somehow not a JSON string. +// The runner reports an engine error as `ERROR `. Decode the JSON string +// back so multi-line errors render with real newlines, not escaped `\n` inside wrapping quotes; +// fall back to the raw slice if it is somehow not a JSON string. fn decode_ffi_error(result_str: &str) -> String { let raw = result_str.strip_prefix("ERROR ").unwrap_or(result_str); - let msg = serde_json::from_str::(raw).unwrap_or_else(|_| raw.to_string()); - // Mirrors the filter the FFI applies to the same variable (`ResourceLimits::from_ffi`); - // they live in separate crates, so the rule is stated twice and must stay one rule. - let configured = env::var("DUCKDB_MAX_TEMP_DIRECTORY_SIZE").is_ok_and(|v| !v.trim().is_empty()); - match spill_cap_hint(&msg, configured) { - Some(hint) => format!("{msg}\n{hint}"), - None => msg, - } + serde_json::from_str::(raw).unwrap_or_else(|_| raw.to_string()) } -/// DuckDB's own wording when a query outgrows the spill cap. Asserted against real engine -/// output by `spilling_past_the_configured_cap_fails_the_query_and_cleans_up` in -/// `windmill-duckdb-ffi-internal`, so an engine bump that rewords it fails there rather than -/// silently dropping the hint. -const SPILL_CAP_ENGINE_MARKER: &str = "This limit was set by the 'max_temp_directory_size' setting"; - -/// Split out of [`decode_ffi_error`] so the branch is testable without mutating the process -/// environment. The engine emits the same text for a configured cap and for its own default, -/// so which limit was hit has to come from the worker's config, not from the message. -fn spill_cap_hint(msg: &str, configured: bool) -> Option<&'static str> { - if !msg.contains(SPILL_CAP_ENGINE_MARKER) { - return None; - } - Some(if configured { - SPILL_CAP_CONFIGURED - } else { - SPILL_CAP_DEFAULT - }) -} - -/// Both hints open by contradicting the engine, whose message ends by suggesting -/// `PRAGMA max_temp_directory_size='10GiB'`: every windmill connection refuses that, since the -/// cap is frozen along with the temp directory, so following it yields a second, -/// unrelated-looking permission error. True either way, because the lock is unconditional. -const SPILL_CAP_CONFIGURED: &str = "\nThe PRAGMA suggested above is refused: this connection's \ - temp directory and the cap on it are frozen, so a script cannot raise the limit. That limit \ - is the worker's DUCKDB_MAX_TEMP_DIRECTORY_SIZE. Spill less (fewer threads, a narrower scan, \ - aggregating before the join), or ask an operator to raise it."; - -/// Nothing configured a cap, so this is DuckDB's own default and the worker is genuinely low on -/// disk. Naming the env var here would send the reader after a knob nobody set. -const SPILL_CAP_DEFAULT: &str = "\nThe PRAGMA suggested above is refused: this connection's temp \ - directory and the cap on it are frozen, so a script cannot raise the limit. No cap is \ - configured on this worker, so that limit is DuckDB's default of 90% of the free space on \ - its temp volume: the worker is low on disk. Free space on it, or spill less (fewer threads, \ - a narrower scan, aggregating before the join)."; - -fn run_duckdb_ffi_safe<'a>( - query_block_list: impl Iterator, +/// Turns what the runner returned into the job result. +fn finish_run( + result_str: String, + column_order: Option, query_block_list_count: usize, - job_args: Vec, - token: &str, - base_internal_url: &str, - w_id: &str, - job_dir: &str, collection_strategy: SqlResultCollectionStrategy, ) -> Result<(Box, Option>)> { - let query_block_list = query_block_list - .map(|s| { - CString::new(s).map_err(|e| { - Error::ExecutionErr(format!("Failed CString conversion: {}", e.to_string())) - }) - }) - .collect::>>()?; - let query_block_list = query_block_list - .iter() - .map(|s| s.as_ptr()) - .collect::>(); - let job_args = serde_json::to_string(&job_args).map_err(to_anyhow)?; - - let job_args = CString::new(job_args).map_err(to_anyhow)?; - let token = CString::new(token).map_err(to_anyhow)?; - let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; - let w_id = CString::new(w_id).map_err(to_anyhow)?; - let memory_limit = - CString::new(resolve_duckdb_memory_limit().unwrap_or_default()).map_err(to_anyhow)?; - let temp_directory = CString::new(job_dir).map_err(to_anyhow)?; - - let run_duckdb_ffi = &DuckDbFfiLib::get_singleton()?.run_duckdb_ffi; - let free_cstr = &DuckDbFfiLib::get_singleton()?.free_cstr; - let mut column_order: *mut c_char = std::ptr::null_mut(); - let result_str = unsafe { - let ptr = run_duckdb_ffi( - query_block_list.as_ptr(), - query_block_list_count, - job_args.as_ptr(), - token.as_ptr(), - base_internal_url.as_ptr(), - w_id.as_ptr(), - memory_limit.as_ptr(), - temp_directory.as_ptr(), - *MAX_SQL_RESULT_SIZE, - &mut column_order, - collection_strategy.collect_last_statement_only(query_block_list_count), - collection_strategy.collect_first_row_only(), - ); - let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); - free_cstr(ptr); - str - }; - - let column_order = if column_order.is_null() - || !collection_strategy.collect_last_statement_only(query_block_list_count) - || collection_strategy.collect_scalar() - { - None - } else { - let str = unsafe { CStr::from_ptr(column_order).to_string_lossy().to_string() }; - unsafe { free_cstr(column_order) }; - Some(serde_json::from_str::>(&str)?) + let column_order = match column_order { + Some(str) + if collection_strategy.collect_last_statement_only(query_block_list_count) + && !collection_strategy.collect_scalar() => + { + Some(serde_json::from_str::>(&str)?) + } + _ => None, }; if result_str.starts_with("ERROR") { @@ -2171,55 +1950,7 @@ fn run_duckdb_ffi_safe<'a>( } } -fn prepare_duckdb_ffi_safe<'a>( - query_block_list: impl Iterator, - token: &str, - base_internal_url: &str, - w_id: &str, - job_dir: &str, -) -> Result> { - let query_block_list = query_block_list - .map(|s| { - CString::new(s).map_err(|e| { - Error::ExecutionErr(format!("Failed CString conversion: {}", e.to_string())) - }) - }) - .collect::>>()?; - let query_block_list = query_block_list - .iter() - .map(|s| s.as_ptr()) - .collect::>(); - - let token = CString::new(token).map_err(to_anyhow)?; - let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; - let w_id = CString::new(w_id).map_err(to_anyhow)?; - let memory_limit = - CString::new(resolve_duckdb_memory_limit().unwrap_or_default()).map_err(to_anyhow)?; - let temp_directory = CString::new(job_dir).map_err(to_anyhow)?; - - let lib = DuckDbFfiLib::get_singleton()?; - let prepare_fn = lib.prepare_duckdb_ffi.as_ref().ok_or_else(|| { - Error::InternalErr( - "prepare_duckdb_ffi not available in duckdb ffi library. Please update to the latest windmill_duckdb_ffi_lib.".to_string(), - ) - })?; - let free_cstr = &lib.free_cstr; - - let result_str = unsafe { - let ptr = prepare_fn( - query_block_list.as_ptr(), - query_block_list.len(), - token.as_ptr(), - base_internal_url.as_ptr(), - w_id.as_ptr(), - memory_limit.as_ptr(), - temp_directory.as_ptr(), - ); - let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); - free_cstr(ptr); - str - }; - +fn finish_prepare(result_str: String) -> Result> { if result_str.starts_with("ERROR") { Err(Error::ExecutionErr(decode_ffi_error(&result_str))) } else { @@ -2227,6 +1958,70 @@ fn prepare_duckdb_ffi_safe<'a>( } } +async fn run_duckdb( + runner: RunnerMode, + query_block_list: Vec, + job_args: Vec, + token: String, + base_internal_url: String, + w_id: String, + job_dir: String, + collection_strategy: SqlResultCollectionStrategy, +) -> Result<(Box, Option>)> { + let n = query_block_list.len(); + let job_args = serde_json::value::to_raw_value(&job_args).map_err(to_anyhow)?; + let memory_limit = resolve_duckdb_memory_limit().unwrap_or_default(); + let out = crate::duckdb_runner::run( + runner, + &RunnerRequest { + mode: RequestMode::Run, + blocks: &query_block_list, + job_args: Some(&job_args), + token: &token, + base_internal_url: &base_internal_url, + w_id: &w_id, + memory_limit: &memory_limit, + temp_directory: &job_dir, + max_result_size: *MAX_SQL_RESULT_SIZE, + collect_last_only: collection_strategy.collect_last_statement_only(n), + collect_first_row_only: collection_strategy.collect_first_row_only(), + }, + &job_dir, + ) + .await?; + finish_run(out.result, out.column_order, n, collection_strategy) +} + +async fn prepare_duckdb( + runner: RunnerMode, + query_block_list: Vec, + token: String, + base_internal_url: String, + w_id: String, + job_dir: String, +) -> Result> { + let memory_limit = resolve_duckdb_memory_limit().unwrap_or_default(); + let out = crate::duckdb_runner::run( + runner, + &RunnerRequest { + mode: RequestMode::Prepare, + blocks: &query_block_list, + job_args: None, + token: &token, + base_internal_url: &base_internal_url, + w_id: &w_id, + memory_limit: &memory_limit, + temp_directory: &job_dir, + max_result_size: 0, + collect_last_only: false, + collect_first_row_only: false, + }, + &job_dir, + ) + .await?; + finish_prepare(out.result) +} + struct ParsedAttachDbResource<'a> { resource_path: &'a str, name: &'a str, @@ -3057,33 +2852,6 @@ mod tests { assert_eq!(decode_ffi_error("ERROR not json"), "not json"); } - /// The engine says the same thing whether the cap came from the worker or is its own - /// default, so the attribution has to come from the config — naming the env var on a - /// worker that never set it would send the reader after the wrong problem. - #[test] - fn spill_cap_hint_attributes_the_limit_to_whoever_set_it() { - let capped = "Out of Memory Error: failed to offload data block of size 256.0 KiB \ - (18.9 MiB/19.0 MiB used).\nThis limit was set by the 'max_temp_directory_size' \ - setting.\nYou can adjust this setting, by using (for example) PRAGMA \ - max_temp_directory_size='10GiB'"; - - let configured = spill_cap_hint(capped, true).expect("the cap error must be recognised"); - assert!(configured.contains("DUCKDB_MAX_TEMP_DIRECTORY_SIZE")); - - let default = spill_cap_hint(capped, false).expect("the cap error must be recognised"); - assert!( - !default.contains("DUCKDB_MAX_TEMP_DIRECTORY_SIZE") && default.contains("low on disk"), - "with nothing configured the hint must point at the disk, not at an unset variable" - ); - - // Both must contradict the PRAGMA the engine suggests, which the lock always refuses. - assert!([configured, default] - .iter() - .all(|h| h.contains("PRAGMA suggested above is refused"))); - - assert!(spill_cap_hint("Binder Error: column \"x\" not found", true).is_none()); - } - fn test_ancestor(wid: &str) -> windmill_common::workspaces::DucklakeAncestorAttach { windmill_common::workspaces::DucklakeAncestorAttach { workspace_id: wid.to_string(), diff --git a/backend/windmill-worker/src/duckdb_runner.rs b/backend/windmill-worker/src/duckdb_runner.rs new file mode 100644 index 0000000000..46f5c5cbde --- /dev/null +++ b/backend/windmill-worker/src/duckdb_runner.rs @@ -0,0 +1,626 @@ +//! DuckDB jobs run in `windmill_duckdb_runner`, a process of their own, under the same job +//! isolation as every other language: inside nsjail when the worker jails jobs, under +//! `unshare` when it isolates that way, and as a plain child process otherwise. An engine +//! crash or a cancellation then ends the runner, never the worker. +//! +//! The runner hands back the string the engine call produced, so result collection and error +//! decoding stay in `duckdb_executor`. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + process::Stdio, + sync::{LazyLock, Mutex}, + time::Duration, +}; + +use serde::Serialize; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + process::Command, +}; +use windmill_common::error::{Error, Result}; + +use crate::{ + common::{build_command_with_isolation, resolve_nsjail_tmp_mount_block, DEV_CONF_NSJAIL}, + is_sandboxing_enabled, is_unshare_enabled, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, + NSJAIL_PATH, PATH_ENV, +}; + +const NSJAIL_CONFIG_RUN_DUCKDB_CONTENT: &str = include_str!("../nsjail/run.duckdb.config.proto"); + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum RunnerMode { + /// Inside nsjail: the jail confines the file system. + Jailed, + /// Under `unshare`, as other languages are on such a worker: its own PID namespace and a + /// cleared environment, with the worker user's view of the file system. + Unshare, + /// A plain child process, on a worker that does not isolate jobs. + Plain, +} + +/// Decided once per job, so every connection a job opens runs the same way. +pub(crate) fn runner_mode() -> RunnerMode { + if cfg!(unix) && is_sandboxing_enabled() && NSJAIL_AVAILABLE.is_some() { + RunnerMode::Jailed + } else if is_unshare_enabled() { + // `build_command_with_isolation` logs and runs unisolated when `unshare` is missing, + // as it does for every other language. + RunnerMode::Unshare + } else { + RunnerMode::Plain + } +} + +#[derive(Serialize, Clone, Copy)] +#[serde(rename_all = "snake_case")] +pub(crate) enum RequestMode { + Run, + Prepare, + Install, +} + +#[derive(Serialize, Clone, Copy)] +pub(crate) struct RunnerRequest<'a> { + pub mode: RequestMode, + pub blocks: &'a [String], + #[serde(skip_serializing_if = "Option::is_none")] + pub job_args: Option<&'a serde_json::value::RawValue>, + pub token: &'a str, + pub base_internal_url: &'a str, + pub w_id: &'a str, + pub memory_limit: &'a str, + pub temp_directory: &'a str, + pub max_result_size: usize, + pub collect_last_only: bool, + pub collect_first_row_only: bool, +} + +/// The runner's answer: the result string (`ERROR ` on failure) and the column order. +pub(crate) struct RunnerOutput { + pub result: String, + pub column_order: Option, +} + +fn runner_path() -> Result { + if let Ok(p) = std::env::var("DUCKDB_RUNNER_PATH") { + return Ok(p); + } + let exe = std::env::current_exe() + .map_err(|e| Error::InternalErr(format!("Could not locate the worker binary: {e}")))?; + let name = format!("windmill_duckdb_runner{}", std::env::consts::EXE_SUFFIX); + let path = exe.with_file_name(&name); + // Cargo runs test binaries from `target//deps`, one level below where the + // worker binary and `build_dev.sh`'s runner sit. + let in_target_dir = exe + .parent() + .filter(|d| d.ends_with("deps")) + .and_then(|d| d.parent()) + .map(|d| d.join(&name)); + let path = match in_target_dir { + Some(p) if !path.exists() && p.exists() => p, + _ => path, + }; + if !path.exists() { + return Err(Error::InternalErr(format!( + "DuckDB jobs need {}, alongside the worker binary. Build it with \ + windmill-duckdb-ffi-internal/build_dev.sh, or point DUCKDB_RUNNER_PATH at it.", + path.display() + ))); + } + Ok(path.to_string_lossy().into_owned()) +} + +/// The jail's own extension cache, separate from the worker user's `~/.duckdb` and mounted +/// read-only. Only [`install_missing_extensions`] writes to it, so a job can neither add to +/// nor replace what the next job loads from it. +fn extension_dir() -> String { + format!("{}/.duckdb/windmill_jail_extensions", HOME_ENV.as_str()) +} + +/// Extensions a job may load inside the jail, where the cache is read-only: the two +/// `setup_duckdb_connection` installs on every connection, the two the platform reaches for +/// without naming them (Parquet data files, the JSON type), and every extension a block +/// installs by name. A `FORCE INSTALL` stays forced: it is the one +/// statement here that must run every time. +fn needed_extensions(blocks: &[String]) -> Vec { + let mut out: Vec = ["httpfs", "azure", "parquet", "json"] + .iter() + .map(|s| format!("INSTALL {s};")) + .collect(); + for block in blocks { + let stmt = block.trim().trim_end_matches(';').trim(); + let mut toks = stmt.split_whitespace().peekable(); + let force = toks.peek().is_some_and(|t| t.eq_ignore_ascii_case("FORCE")); + if force { + toks.next(); + } + if !toks + .next() + .is_some_and(|t| t.eq_ignore_ascii_case("INSTALL")) + { + continue; + } + let Some(name) = toks.next() else { continue }; + if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + continue; + } + let name = name.to_ascii_lowercase(); + let force = if force { "FORCE " } else { "" }; + let stmt = match (toks.next(), toks.next(), toks.next()) { + (None, _, _) => format!("{force}INSTALL {name};"), + (Some(f), Some(r), None) + if f.eq_ignore_ascii_case("FROM") && r.eq_ignore_ascii_case("community") => + { + format!("{force}INSTALL {name} FROM community;") + } + _ => continue, + }; + if !out.contains(&stmt) { + out.push(stmt); + } + } + out +} + +static INSTALLED: LazyLock>> = LazyLock::new(Default::default); + +/// Installs, outside the jail, what the job will load inside it. The runner only runs +/// `INSTALL` here, and only of statements built above, so no script SQL and no extension +/// code runs unjailed. Once per extension per worker process. +async fn install_missing_extensions(blocks: &[String], job_dir: &str) -> Result<()> { + let missing: Vec = { + let installed = INSTALLED.lock().unwrap(); + needed_extensions(blocks) + .into_iter() + .filter(|s| s.starts_with("FORCE ") || !installed.contains(s)) + .collect() + }; + if missing.is_empty() { + return Ok(()); + } + let ext_dir = extension_dir(); + tokio::fs::create_dir_all(&ext_dir).await?; + let blocks: Vec = std::iter::once(format!( + "SET extension_directory='{}';", + ext_dir.replace('\'', "''") + )) + .chain(missing.iter().cloned()) + .collect(); + let out = spawn_runner( + RunnerMode::Plain, + &RunnerRequest { + mode: RequestMode::Install, + blocks: &blocks, + job_args: None, + token: "", + base_internal_url: "", + w_id: "", + memory_limit: "", + temp_directory: "", + max_result_size: 0, + collect_last_only: false, + collect_first_row_only: false, + }, + job_dir, + ) + .await?; + if let Some(raw) = out.result.strip_prefix("ERROR ") { + let msg = serde_json::from_str::(raw).unwrap_or_else(|_| raw.to_string()); + return Err(Error::ExecutionErr(format!( + "Installing DuckDB extensions for the jailed runner failed: {msg}" + ))); + } + INSTALLED.lock().unwrap().extend(missing.into_iter().filter(|s| !s.starts_with("FORCE "))); + Ok(()) +} + +/// `FORCE INSTALL` rewrites the cache, which is read-only inside the jail. The worker has +/// already honored it outside, so inside it becomes a plain `INSTALL`, which finds the +/// extension present and does nothing. Same block count, so result collection is unchanged. +fn unforce_installs(blocks: &[String]) -> Vec { + blocks + .iter() + .map(|b| { + let t = b.trim_start(); + match t.split_once(char::is_whitespace) { + Some((first, rest)) + if first.eq_ignore_ascii_case("FORCE") + && rest + .trim_start() + .get(..7) + .is_some_and(|k| k.eq_ignore_ascii_case("INSTALL")) => + { + rest.trim_start().to_string() + } + _ => b.clone(), + } + }) + .collect() +} + +/// The only place a jailed runner can write on the host: its spill directory and its result +/// file. The rest of the job dir is mounted read-only. +fn work_dir(job_dir: &str) -> String { + format!("{job_dir}/duckdb") +} + +/// DuckDB's spill cap, in DuckDB's spelling. The cloud runs one worker per pod on nodes with no +/// per-pod ephemeral-storage limit, so it gets one even when nothing configured it. +fn max_temp_directory_size() -> Option { + match std::env::var("DUCKDB_MAX_TEMP_DIRECTORY_SIZE") { + Ok(v) if !v.trim().is_empty() => Some(v.trim().to_string()), + _ if *windmill_common::worker::CLOUD_HOSTED => Some("6GiB".to_string()), + _ => None, + } +} + +/// Parses a DuckDB size (`6GiB`, `500 MB`, `1024`): 1024-based for `KiB`..`TiB`, 1000-based +/// for `KB`..`TB`. `None` for anything else, which DuckDB then refuses on its own. +fn parse_duckdb_size(s: &str) -> Option { + let s = s.trim(); + let split = s + .find(|c: char| !(c.is_ascii_digit() || c == '.')) + .unwrap_or(s.len()); + let (num, unit) = s.split_at(split); + let num: f64 = num.parse().ok()?; + let mult: f64 = match unit.trim().to_ascii_lowercase().as_str() { + "" | "b" | "byte" | "bytes" => 1.0, + "kb" => 1e3, + "mb" => 1e6, + "gb" => 1e9, + "tb" => 1e12, + "kib" => 1024.0, + "mib" => 1024.0 * 1024.0, + "gib" => 1024.0 * 1024.0 * 1024.0, + "tib" => 1024.0 * 1024.0 * 1024.0 * 1024.0, + _ => return None, + }; + Some((num * mult) as u64) +} + +pub(crate) async fn run( + mode: RunnerMode, + req: &RunnerRequest<'_>, + job_dir: &str, +) -> Result { + let spill_dir = format!("{}/spill", work_dir(job_dir)); + tokio::fs::create_dir_all(&spill_dir).await?; + let req = RunnerRequest { temp_directory: &spill_dir, ..*req }; + if mode != RunnerMode::Jailed { + return spawn_runner(mode, &req, job_dir).await; + } + install_missing_extensions(req.blocks, job_dir).await?; + let blocks = unforce_installs(req.blocks); + spawn_runner(mode, &RunnerRequest { blocks: &blocks, ..req }, job_dir).await +} + +/// Disk the directory actually uses (allocated blocks, so sparse files count for what they +/// hold), without following symlinks: the jailed script controls what is in there. +fn disk_usage(dir: &Path) -> u64 { + let mut total = 0; + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&d) else { + continue; + }; + for entry in entries.flatten() { + let Ok(meta) = entry.path().symlink_metadata() else { + continue; + }; + #[cfg(unix)] + { + total += std::os::unix::fs::MetadataExt::blocks(&meta) * 512; + } + #[cfg(not(unix))] + { + total += meta.len(); + } + if meta.is_dir() { + stack.push(entry.path()); + } + } + } + total +} + +/// Resolves once the runner's work dir holds more than `cap`. Inside the jail that dir is the +/// only place the script can write to on the host, and the script can raise DuckDB's own +/// `max_temp_directory_size`, so this is what makes the cap hold. +async fn exceeds(dir: PathBuf, cap: u64) -> u64 { + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + let d = dir.clone(); + let used = tokio::task::spawn_blocking(move || disk_usage(&d)) + .await + .unwrap_or(0); + if used > cap { + return used; + } + } +} + +/// Kills the runner's process group when the job future is dropped (cancellation, timeout) +/// before the runner exits. Killing the spawned process alone is not enough: `unshare --fork` +/// leaves its child running. +struct KillGroupOnDrop(Option); + +impl Drop for KillGroupOnDrop { + fn drop(&mut self) { + #[cfg(unix)] + if let Some(pgid) = self.0 { + let _ = nix::sys::signal::killpg( + nix::unistd::Pid::from_raw(pgid), + nix::sys::signal::Signal::SIGKILL, + ); + } + } +} + +async fn spawn_runner( + mode: RunnerMode, + req: &RunnerRequest<'_>, + job_dir: &str, +) -> Result { + let runner = runner_path()?; + let call_id = uuid::Uuid::new_v4(); + let work_dir = work_dir(job_dir); + tokio::fs::create_dir_all(&work_dir).await?; + let result_path = format!("{work_dir}/result_{call_id}.out"); + let spill_cap = max_temp_directory_size(); + + let mut cmd = match mode { + RunnerMode::Jailed => { + let ext_dir = extension_dir(); + tokio::fs::create_dir_all(&ext_dir).await?; + let bigquery_mount = match std::env::var("GOOGLE_APPLICATION_CREDENTIALS") { + Ok(p) if !p.is_empty() => format!( + "mount {{\n src: \"{p}\"\n dst: \"{p}\"\n is_bind: true\n mandatory: false\n}}" + ), + _ => String::new(), + }; + let config_name = format!("run.duckdb.{call_id}.config.proto"); + tokio::fs::write( + format!("{job_dir}/{config_name}"), + NSJAIL_CONFIG_RUN_DUCKDB_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{RUNNER_PATH}", &runner) + .replace("{EXTENSION_DIR}", &ext_dir) + .replace("{BIGQUERY_CREDENTIALS_MOUNT}", &bigquery_mount) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("#{DEV}", DEV_CONF_NSJAIL) + // Rooted at the work dir rather than the job dir: with a disk-backed /tmp + // the jail's /tmp is a host directory, and inside the work dir the disk cap + // covers it. A RAM-backed /tmp is bounded by its tmpfs size. + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(&work_dir).await, + ), + ) + .await?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.args([ + "--config", + &config_name, + "--", + "/tmp/windmill_duckdb_runner", + ]); + cmd + } + RunnerMode::Unshare => build_command_with_isolation(&runner, &[]), + RunnerMode::Plain => Command::new(&runner), + }; + + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + #[cfg(unix)] + cmd.process_group(0); + // The jail sets its own HOME, over the read-only extension cache. + if mode != RunnerMode::Jailed { + cmd.env("HOME", HOME_ENV.as_str()); + } + // Windows networking (Winsock) needs SystemRoot, so without these every extension install + // and S3 proxy request fails. Empty off Windows. + cmd.envs(crate::WIN_ENVS.iter().cloned()); + #[cfg(windows)] + cmd.env("windir", crate::SYSTEM_ROOT.as_str()); + // DuckDB's default TimeZone follows TZ, as it did in the worker's own environment. + if !crate::TZ_ENV.is_empty() { + cmd.env("TZ", crate::TZ_ENV.as_str()); + } + if let Some(cap) = &spill_cap { + cmd.env("DUCKDB_MAX_TEMP_DIRECTORY_SIZE", cap); + } + for var in [ + "GOOGLE_APPLICATION_CREDENTIALS", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + "all_proxy", + ] { + if let Ok(v) = std::env::var(var) { + cmd.env(var, v); + } + } + + let mut payload = serde_json::to_value(req).map_err(|e| Error::InternalErr(e.to_string()))?; + payload["result_path"] = serde_json::Value::String(result_path.clone()); + let payload = serde_json::to_vec(&payload).map_err(|e| Error::InternalErr(e.to_string()))?; + + // The engine's memory lives outside the worker, so a native OOM can take the job alone, as + // long as the kernel picks the runner first. Set before exec so nsjail or unshare, and the + // runner they start, inherit it: written after spawn it races that, and the jail's /proc is + // read-only. Only raw syscalls here, between fork and exec. + #[cfg(target_os = "linux")] + { + let adj = windmill_common::worker::JOB_OOM_SCORE_ADJ.to_string(); + unsafe { + cmd.pre_exec(move || { + let fd = nix::libc::open(c"/proc/self/oom_score_adj".as_ptr(), nix::libc::O_WRONLY); + if fd >= 0 { + nix::libc::write(fd, adj.as_ptr().cast(), adj.len()); + nix::libc::close(fd); + } + Ok(()) + }); + } + } + let mut child = cmd.spawn().map_err(|e| { + Error::InternalErr(format!("Could not start the DuckDB runner ({mode:?}): {e}")) + })?; + let mut guard = KillGroupOnDrop(child.id().map(|p| p as i32)); + let mut stdin = child.stdin.take().expect("stdin is piped"); + let write = async move { + stdin.write_all(&payload).await?; + stdin.shutdown().await + }; + let stdout = child.stdout.take().expect("stdout is piped"); + let stderr = child.stderr.take().expect("stderr is piped"); + let cap_bytes = spill_cap.as_deref().and_then(parse_duckdb_size); + let watchdog = async { + match cap_bytes { + Some(cap) if mode == RunnerMode::Jailed => exceeds(PathBuf::from(&work_dir), cap).await, + _ => std::future::pending().await, + } + }; + let (write, status, stdout, stderr) = tokio::select! { + r = async { tokio::join!(write, child.wait(), output_tail(stdout), output_tail(stderr)) } => r, + // Returning drops the child, and with it `kill_on_drop` and `guard` end the runner. + used = watchdog => { + return Err(Error::ExecutionErr(format!( + "DuckDB: this job wrote {} MB to disk, past this worker's limit of {} \ + (DUCKDB_MAX_TEMP_DIRECTORY_SIZE), and was stopped. Raising \ + max_temp_directory_size from the script does not lift it. Spill less (fewer \ + threads, a narrower scan, aggregating before the join), or write large outputs \ + to S3 rather than local files.", + used / 1_000_000, + spill_cap.as_deref().unwrap_or_default() + ))); + } + }; + let status = status?; + guard.0 = None; + let read = read_result(&result_path).await; + let _ = tokio::fs::remove_file(&result_path).await; + + if !status.success() || read.is_err() { + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal: Option = None; + let how = match signal { + Some(sig) => format!("was killed by signal {sig}"), + None => format!("exited with {status}"), + }; + return Err(Error::ExecutionErr(format!( + "The DuckDB engine process {how}. The worker is unaffected.{}{}{}", + write + .err() + .map(|e| format!("\nrequest: {e}")) + .unwrap_or_default(), + if stderr.trim().is_empty() { + String::new() + } else { + format!("\nstderr: {}", stderr.trim()) + }, + if stdout.trim().is_empty() { + String::new() + } else { + format!("\nstdout: {}", stdout.trim()) + }, + ))); + } + let mut column_order = read?; + let header_end = column_order.find('\n').unwrap_or(column_order.len()); + let result = column_order.split_off((header_end + 1).min(column_order.len())); + column_order.truncate(header_end); + Ok(RunnerOutput { column_order: (!column_order.is_empty()).then_some(column_order), result }) +} + +/// The result file sits in a directory the jailed script can write to, so a symlink planted there +/// must not make the worker read some other host file into the job result. +async fn read_result(path: &str) -> std::io::Result { + let mut opts = tokio::fs::OpenOptions::new(); + opts.read(true); + #[cfg(unix)] + opts.custom_flags(nix::libc::O_NOFOLLOW); + let mut s = String::new(); + opts.open(path).await?.read_to_string(&mut s).await?; + Ok(s) +} + +/// Drains a runner pipe keeping only its last few KB. A script controls how much the engine +/// prints (`CALL enable_logging(storage='stdout')`), so buffering it whole would let a jailed +/// job grow the worker's memory without bound. +async fn output_tail(mut r: impl AsyncRead + Unpin) -> String { + const KEEP: usize = 4096; + let mut buf = vec![0u8; 16 * 1024]; + let mut tail: Vec = Vec::with_capacity(2 * KEEP); + loop { + match r.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => { + tail.extend_from_slice(&buf[..n]); + if tail.len() > KEEP { + tail.drain(..tail.len() - KEEP); + } + } + } + } + String::from_utf8_lossy(&tail).into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn duckdb_sizes_parse_in_duckdb_units() { + assert_eq!(parse_duckdb_size("6GiB"), Some(6 * 1024 * 1024 * 1024)); + assert_eq!(parse_duckdb_size("500 MB"), Some(500_000_000)); + assert_eq!(parse_duckdb_size("1.5kib"), Some(1536)); + assert_eq!(parse_duckdb_size("1024"), Some(1024)); + // Kubernetes' spelling is not DuckDB's: the engine refuses it, so no cap is derived. + assert_eq!(parse_duckdb_size("6Gi"), None); + assert_eq!(parse_duckdb_size(""), None); + } + + fn strs(v: &[&str]) -> Vec { + v.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn force_installs_run_outside_and_become_plain_inside() { + let blocks = strs(&[ + "force install httpfs;", + "INSTALL bigquery FROM community;", + "INSTALL x FROM 'http://evil';", + "INSTALL '/tmp/x.so';", + "SELECT 'FORCE INSTALL';", + ]); + assert_eq!( + needed_extensions(&blocks)[4..], + strs(&["FORCE INSTALL httpfs;", "INSTALL bigquery FROM community;"]) + ); + assert_eq!( + unforce_installs(&blocks), + strs(&[ + "install httpfs;", + "INSTALL bigquery FROM community;", + "INSTALL x FROM 'http://evil';", + "INSTALL '/tmp/x.so';", + "SELECT 'FORCE INSTALL';", + ]) + ); + } +} diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 9fe6d4a616..fd26484f23 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -22,8 +22,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, script_envars, + start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, GoBuildLimits, DISABLE_NUSER, GOPRIVATE, GOPROXY, @@ -600,26 +600,10 @@ pub async fn handle_go_job( let child = if is_sandboxing_enabled() { let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_GO_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() - .envs(envs) .envs(reserved_variables) .envs( get_proxy_envs_for_lang( @@ -637,6 +621,22 @@ pub async fn handle_go_job( .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_GO_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{ENVARS}", &script_envars(&envs, &nsjail_cmd, &[])), + )?; start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { #[cfg(unix)] diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 2e47fced18..39c8804cd5 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -28,7 +28,7 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, resolve_job_timeout, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, - start_child_process, OccupancyMetrics, + script_envars, script_overrides, start_child_process, OccupancyMetrics, }, handle_child, is_sandboxing_enabled, read_ee_registry_bool_with_workspace_override, read_ee_registry_with_workspace_override, @@ -678,9 +678,13 @@ async fn compile<'a>( .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) .env("HOME", &*JAVA_HOME_DIR) - .env("BASE_INTERNAL_URL", base_internal_url) - .envs(envs) - .envs(reserved_variables) + .env("BASE_INTERNAL_URL", base_internal_url); + // javac runs outside the jail even when sandboxing is on: script env + // (`LD_PRELOAD`, `JAVA_TOOL_OPTIONS`) would load code into it on the host. + if !is_sandboxing_enabled() { + cmd.envs(envs); + } + cmd.envs(reserved_variables) .envs(PROXY_ENVS.clone()) .args(&[ "-classpath", @@ -777,29 +781,13 @@ async fn run<'a>( let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_JAVA_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", &*JAVA_CACHE_DIR) - .replace("{SHARED_MOUNT}", &shared_mount) - // .replace("{CACHED_TARGET}", &shared_mount) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); cmd.env_clear() - .current_dir(job_dir) + .current_dir(*job_dir) .env("PATH", PATH_ENV.as_str()) .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) - .envs(envs) - .envs(reserved_variables) + .envs(&reserved_variables) .envs(crate::get_otel_context_envs(&job.id)) .args(vec![ "--config", @@ -835,6 +823,32 @@ async fn run<'a>( cmd.args(vec!["-classpath", &classpath, "net.script.App"]); cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_JAVA_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", &*JAVA_CACHE_DIR) + .replace("{SHARED_MOUNT}", &shared_mount) + // .replace("{CACHED_TARGET}", &shared_mount) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace( + "{ENVARS}", + &script_envars( + &envs, + &cmd, + &script_overrides( + &["PATH", "HOME", "BASE_INTERNAL_URL"], + &reserved_variables, + ), + ), + ), + )?; start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? } else { append_logs( diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index aa1d9ab209..e56334df67 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -45,8 +45,8 @@ mod deno_executor; mod docker_v2; #[cfg(feature = "duckdb")] mod duckdb_executor; -#[cfg(all(feature = "duckdb", feature = "private"))] -mod duckdb_isolation_ee; +#[cfg(feature = "duckdb")] +mod duckdb_runner; mod global_cache; mod go_executor; mod graphql_executor; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 8aa0946666..b57225f606 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -14,8 +14,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, script_envars, script_overrides, + start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, @@ -248,30 +248,13 @@ async fn run<'a>( let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_NU_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{NU_PATH}", &NU_PATH) - .replace("{SHARED_MOUNT}", &shared_mount) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .env_clear() - .current_dir(job_dir) + .current_dir(*job_dir) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) - .envs(envs) - .envs(reserved_variables) + .envs(&reserved_variables) .envs( get_proxy_envs_for_lang( &ScriptLang::Nu, @@ -293,6 +276,26 @@ async fn run<'a>( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_NU_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{NU_PATH}", &NU_PATH) + .replace("{SHARED_MOUNT}", &shared_mount) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace( + "{ENVARS}", + &script_envars(&envs, &nsjail_cmd, &script_overrides(&["PATH", "BASE_INTERNAL_URL"], &reserved_variables)), + ), + )?; start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { append_logs( diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 8378455ef4..26559e8b37 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -22,7 +22,8 @@ use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, resolve_nsjail_timeout, - resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, + resolve_nsjail_tmp_mount_block, script_envars, start_child_process, MaybeLock, + OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, @@ -541,19 +542,6 @@ try {{ let child = if is_sandboxing_enabled() { let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_PHP_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); let args = vec![ @@ -566,7 +554,6 @@ try {{ nsjail_cmd .current_dir(job_dir) .env_clear() - .envs(envs) .envs(reserved_variables) .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) @@ -574,6 +561,20 @@ try {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_PHP_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{ENVARS}", &script_envars(&envs, &nsjail_cmd, &[])), + )?; start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { let script_path = format!("{job_dir}/wrapper.php"); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 4ab3d50df8..e7ddd91531 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -791,7 +791,8 @@ pub async fn handle_python_job( None }; - let (py_version, mut additional_python_paths) = handle_python_deps( + let module_sources = windmill_parser_py_imports::module_sources(&script_path, modules.as_ref()); + let (py_version, mut additional_python_paths) = handle_python_deps_with_modules( job_dir, requirements_o, inner_content, @@ -807,6 +808,7 @@ pub async fn handle_python_job( precomputed_agent_info, annotations.clone(), &temp_script_refs, + module_sources.as_ref(), ) .await?; @@ -1939,6 +1941,7 @@ async fn replace_pip_secret( } } +#[cfg(feature = "private")] pub(crate) async fn handle_python_deps( job_dir: &str, requirements_o: Option<&String>, @@ -1955,6 +1958,45 @@ pub(crate) async fn handle_python_deps( precomputed_agent_info: Option, annotations: PythonAnnotations, temp_script_refs: &Option>, +) -> error::Result<(PyV, Vec)> { + handle_python_deps_with_modules( + job_dir, + requirements_o, + inner_content, + w_id, + script_path, + job_id, + conn, + worker_name, + worker_dir, + mem_peak, + canceled_by, + occupancy_metrics, + precomputed_agent_info, + annotations, + temp_script_refs, + None, + ) + .await +} + +async fn handle_python_deps_with_modules( + job_dir: &str, + requirements_o: Option<&String>, + inner_content: &str, + w_id: &str, + script_path: &str, + job_id: &Uuid, + conn: &Connection, + worker_name: &str, + worker_dir: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + precomputed_agent_info: Option, + annotations: PythonAnnotations, + temp_script_refs: &Option>, + module_sources: Option<&HashMap>, ) -> error::Result<(PyV, Vec)> { create_dependencies_dir(job_dir).await; @@ -1984,6 +2026,7 @@ pub(crate) async fn handle_python_deps( &mut locked_v, &None, temp_script_refs, + module_sources, )) .await?; diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs index 9b29ae35b6..d273d08785 100644 --- a/backend/windmill-worker/src/r_executor.rs +++ b/backend/windmill-worker/src/r_executor.rs @@ -20,8 +20,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, - DEV_CONF_NSJAIL, + read_result, resolve_nsjail_tmp_mount_block, script_envars, script_overrides, start_child_process, + OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::{self}, @@ -608,28 +608,12 @@ async fn run<'a>( ) .await; - write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_R_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{SHARED_MOUNT}", &shared_mount) - .replace("{R_CACHE_DIR}", &*R_CACHE_DIR) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), - )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); cmd.env_clear() - .current_dir(job_dir) + .current_dir(*job_dir) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) - .envs(envs) - .envs(reserved_variables) + .envs(&reserved_variables) .envs(R_PROXY_ENVS.clone()) .envs( get_proxy_envs_for_lang( @@ -653,6 +637,25 @@ async fn run<'a>( ]); cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_R_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{SHARED_MOUNT}", &shared_mount) + .replace("{R_CACHE_DIR}", &*R_CACHE_DIR) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace( + "{ENVARS}", + &script_envars(&envs, &cmd, &script_overrides(&["PATH", "BASE_INTERNAL_URL"], &reserved_variables)), + ), + )?; start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? } else { append_logs( diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 7871c050c6..1cd477e1fd 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -23,8 +23,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, script_envars, + script_overrides, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::{self}, @@ -629,21 +629,11 @@ async fn install<'a>( // true, // crate::common::InstallStrategy::Single(Arc::new(move |dependency| { move |dependency| { - let mut cmd = if jailed { + let nsjail_proto = jailed.then(|| format!("{}.download.config.proto", Uuid::new_v4())); + let mut cmd = if let Some(nsjail_proto) = &nsjail_proto { std::fs::create_dir_all(&dependency.path)?; - let nsjail_proto = format!("{}.download.config.proto", Uuid::new_v4()); - let _ = write_file( - &job_dir, - &nsjail_proto, - &NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT - .replace("{TARGET}", &dependency.path) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("{TMP_MOUNT_BLOCK}", &nsjail_tmp_mount_block) - .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), - )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); - cmd.args(vec!["--config", &nsjail_proto, "--", GEM_PATH.as_str()]); + cmd.args(vec!["--config", nsjail_proto, "--", GEM_PATH.as_str()]); cmd } else { Command::new(if cfg!(windows) { @@ -699,10 +689,36 @@ async fn install<'a>( "--source", source.as_str(), ]) - .envs(envs.clone()) - .envs(reserved_variables.clone()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + if !jailed { + cmd.envs(&envs); + } + cmd.envs(&reserved_variables); + if let Some(nsjail_proto) = &nsjail_proto { + let set_before = ["PATH", "HOME"] + .into_iter() + .chain(RUBY_PROXY_ENVS.iter().map(|(k, _)| k.as_str())) + .collect::>(); + let _ = write_file( + &job_dir, + nsjail_proto, + &NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT + .replace("{TARGET}", &dependency.path) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("{TMP_MOUNT_BLOCK}", &nsjail_tmp_mount_block) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{ENVARS}", + &script_envars( + &envs, + &cmd, + &script_overrides(&set_before, &reserved_variables), + ), + ), + )?; + } if !verbose { // Make it output minimal info @@ -802,30 +818,13 @@ mount {{ let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_RUBY_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{SHARED_MOUNT}", &shared_mount) - .replace("{SHARED_DEPENDENCIES}", &shared_deps) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); cmd.env_clear() - .current_dir(job_dir) + .current_dir(*job_dir) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("RUBYLIB", rubylib.as_str()) - .envs(envs) - .envs(reserved_variables) + .envs(&reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) .envs( get_proxy_envs_for_lang( @@ -846,6 +845,33 @@ mount {{ ]); cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_RUBY_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{SHARED_MOUNT}", &shared_mount) + .replace("{SHARED_DEPENDENCIES}", &shared_deps) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace( + "{ENVARS}", + &script_envars( + &envs, + &cmd, + &script_overrides( + &["PATH", "BASE_INTERNAL_URL", "RUBYLIB"], + &reserved_variables, + ), + ), + ), + )?; start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? } else { append_logs( diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index dff7d58c2d..fabea34499 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -23,7 +23,8 @@ use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, resolve_nsjail_timeout, - resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + resolve_nsjail_tmp_mount_block, script_envars, start_child_process, OccupancyMetrics, + DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -757,28 +758,10 @@ pub async fn handle_rust_job( let child = if is_sandboxing_enabled() { let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_RUST_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) - .replace("{CACHE_HASH}", &hash) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("#{DEV}", DEV_CONF_NSJAIL) - .replace("{SHARED_MOUNT}", shared_mount) - .replace( - "{TMP_MOUNT_BLOCK}", - &resolve_nsjail_tmp_mount_block(job_dir).await, - ) - .replace("{TIMEOUT}", &nsjail_timeout), - )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() - .envs(envs) .envs(reserved_variables) .envs( get_proxy_envs_for_lang( @@ -796,6 +779,24 @@ pub async fn handle_rust_job( .args(vec!["--config", "run.config.proto", "--", "/tmp/main"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_RUST_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) + .replace("{CACHE_HASH}", &hash) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace("{SHARED_MOUNT}", shared_mount) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{ENVARS}", &script_envars(&envs, &nsjail_cmd, &[])), + )?; start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { let compiled_executable_name = &format!("{job_dir}/{RUST_BIN_NAME}"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6189d39b8d..4c50e63a8d 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1,5 +1,5 @@ use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fs::{create_dir_all, remove_dir_all}; #[cfg(feature = "python")] @@ -29,12 +29,14 @@ use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; use windmill_dep_map::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; -use windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies; +use windmill_dep_map::trigger_dependents::{ + trigger_dependents_to_recompute_dependencies, RELOCK_ALL_APP_COMPONENTS, +}; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; use windmill_common::{ - apps::AppScriptId, + apps::{raw_app_runnables_map, AppScriptId, RAW_APP_RUNNABLES}, cache::{self, RawData}, error, flows::{add_virtual_items_if_necessary, FlowValue}, @@ -42,6 +44,7 @@ use windmill_common::{ DB, }; use windmill_dep_map::lock_hash::record_lock_hashes; +use windmill_dep_map::raw_app_runnables::RawAppRunnables; pub use windmill_dep_map::{ extract_referenced_paths, extract_relative_imports, process_relative_imports, refresh_dependency_map, @@ -546,8 +549,14 @@ pub async fn handle_dependency_job( ); tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); append_logs(&job.id, w_id, log_msg, &db.into()).await; - requeue_relock(db, job, script_path, deployment_message, parent_path) - .await?; + requeue_relock( + db, + job, + script_dependent(script_path), + deployment_message, + parent_path, + ) + .await?; return Ok(to_raw_value_owned( json!({ "status": "Lock generation superseded by a newer version", "lock": content }), )); @@ -706,9 +715,14 @@ pub async fn handle_dependency_job( { Ok(RelockOutcome::Deployed(_)) => *deployment_tallied = false, Ok(RelockOutcome::Superseded(_)) => { - if let Err(e) = - requeue_relock(db, job, script_path, deployment_message, parent_path) - .await + if let Err(e) = requeue_relock( + db, + job, + script_dependent(script_path), + deployment_message, + parent_path, + ) + .await { tracing::error!(%e, "error queueing a relock of {script_path}") } @@ -830,14 +844,15 @@ async fn commit_relock( Ok(RelockOutcome::Deployed(ScriptHash(new_hash))) } -/// Queues another relative-import relock of `script_path`, through the same push the fan-out +/// Queues another relative-import relock of `dependent`, through the same push the fan-out /// uses. The version live now was deployed while a lock was generated for its predecessor; -/// when that deploy was a sibling relock it queued nothing for this path, and the result just -/// discarded may have been the one generated against the current imports. +/// when that deploy was a sibling relock it queued nothing for this path, and a deploy of an +/// edited app carries the locks of the version it was edited from, so the result just +/// discarded may have been the only one generated against the current imports. async fn requeue_relock( db: &DB, job: &MiniPulledJob, - script_path: &str, + dependent: DependencyDependent, deployment_message: Option, parent_path: Option, ) -> error::Result<()> { @@ -849,11 +864,7 @@ async fn requeue_relock( .unwrap_or_default(); trigger_dependents_to_recompute_dependencies( &job.workspace_id, - vec![DependencyDependent { - importer_path: script_path.to_string(), - importer_kind: "script".to_string(), - importer_node_ids: None, - }], + vec![dependent], deployment_message, parent_path, &job.permissioned_as_email, @@ -865,6 +876,55 @@ async fn requeue_relock( .await } +fn script_dependent(script_path: &str) -> DependencyDependent { + DependencyDependent { + importer_path: script_path.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + } +} + +/// The version an app is served from, which is the one a relative-import relock rewrites. +async fn published_app_version(db: &DB, w_id: &str, app_path: &str) -> error::Result> { + Ok(sqlx::query_scalar!( + "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2", + app_path, + w_id + ) + .fetch_optional(db) + .await? + .flatten()) +} + +/// The dependency-map node id of every inline script of an app version's `value`. +fn app_component_ids(value: &Value) -> error::Result> { + let mut ids = vec![]; + windmill_common::apps::traverse_app_inline_scripts(value, None, &mut |_, id| { + ids.push(id.unwrap_or_default()); + Ok(()) + })?; + ids.sort(); + ids.dedup(); + Ok(ids) +} + +/// A raw app's runnables are recorded in the dependency map under their name in `runnables`. +/// Versions deployed before that recorded every runnable under the empty node id, so a relock +/// asking for that id relocks each of them. +fn expand_legacy_raw_app_components( + components_to_relock: Option>, + value: &Value, +) -> Option> { + let mut components = components_to_relock?; + if let Some(i) = components.iter().position(String::is_empty) { + components.swap_remove(i); + if let Some(Value::Object(runnables)) = value.get(RAW_APP_RUNNABLES) { + components.extend(runnables.keys().cloned()); + } + } + Some(components) +} + fn remove_ansi_codes(s: &str) -> String { lazy_static::lazy_static! { static ref ANSI_REGEX: regex::Regex = regex::Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap(); @@ -2093,8 +2153,10 @@ async fn insert_app_script( code: String, lock: Option, language: Option, + modules: Option>, ) -> Result { let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code)); + let modules = modules.map(|m| serde_json::to_value(m)).transpose()?; let hash = { let mut hasher = sha2::Sha256::new(); hasher.update(app.to_le_bytes()); @@ -2106,14 +2168,18 @@ async fn insert_app_script( relative_imports_bytes(db, Some(&code), &format!("{path}/app"), language).await?, ); } + // Only when present, so a runnable without modules keeps the id it already had. + if let Some(modules) = modules.as_ref() { + hasher.update(modules.to_string()); + } format!("{:x}", hasher.finalize()) }; // Insert the app script if it doesn't exist. sqlx::query_scalar!( r#" - INSERT INTO app_script (app, hash, lock, code, code_sha256) - VALUES ($1, $2, $3, $4, $5) + INSERT INTO app_script (app, hash, lock, code, code_sha256, modules) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id RETURNING id "#, @@ -2121,7 +2187,8 @@ async fn insert_app_script( hash, lock, code, - code_sha256 + code_sha256, + modules ) .fetch_one(db) .await @@ -2299,6 +2366,7 @@ async fn reduce_app( path: &str, value: &mut Value, app: i64, + raw_app_runnables: &RawAppRunnables, ) -> Result<()> { match value { Value::Object(object) => { @@ -2319,26 +2387,20 @@ async fn reduce_app( Value::String(s) => Some(s), _ => None, }); - let id = insert_app_script( - db, - path, - app, - code, - lock, - language.map(|v| from_value(v).ok()).flatten(), - ) - .await?; + let language: Option = language.and_then(|v| from_value(v).ok()); + let modules = language.and_then(|l| raw_app_runnables.modules_of(path, &code, l)); + let id = insert_app_script(db, path, app, code, lock, language, modules).await?; // insert the `id` into the `script` object: script.insert("id".to_string(), json!(id.0)); } else { for (_, value) in object { - Box::pin(reduce_app(db, path, value, app)).await?; + Box::pin(reduce_app(db, path, value, app, raw_app_runnables)).await?; } } } Value::Array(array) => { for value in array { - Box::pin(reduce_app(db, path, value, app)).await?; + Box::pin(reduce_app(db, path, value, app, raw_app_runnables)).await?; } } _ => {} @@ -2383,6 +2445,7 @@ async fn lock_modules_app( raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, temp_script_refs: &Option>, + raw_app_runnables: &RawAppRunnables, ) -> Result { match value { Value::Object(mut m) => { @@ -2417,15 +2480,18 @@ async fn lock_modules_app( .to_string(); let mut logs = "".to_string(); - let referenced_paths = extract_referenced_paths( - &content, - &format!("{job_path}/app"), - Some(language), - ); + let referenced_paths = + raw_app_runnables.referenced_paths(job_path, &content, language); + // The lock covers what the imported runnables import, and nothing + // records whether one of them changed, so it is never reused. + let modules = + raw_app_runnables.modules_of(job_path, &content, language); + let modules = modules.map(|m| m.into_iter().collect::>()); if let Some((l, id)) = locks_to_reload .as_ref() .zip(container_id.as_ref()) + .filter(|_| modules.is_none()) // TODO: Remove fallback .and_then(|e| { if *WMDEBUG_NO_COMPONENTS_TO_RELOCK { @@ -2447,9 +2513,9 @@ async fn lock_modules_app( .await?; return Ok(Value::Object(m.clone())); } - } else if v - .get("lock") - .is_some_and(|x| !x.as_str().unwrap().trim().is_empty()) + } else if modules.is_none() + && v.get("lock") + .is_some_and(|x| !x.as_str().unwrap().trim().is_empty()) { if skip_creating_new_lock(&language, &content) && (MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await @@ -2486,7 +2552,7 @@ async fn lock_modules_app( base_internal_url, token, &format!("{}/app", job.runnable_path()), - None, + modules.as_ref(), occupancy_metrics, &None, existing_lock, @@ -2549,6 +2615,36 @@ async fn lock_modules_app( } } for (a, b) in m.clone().into_iter() { + if let Some(runnables) = raw_app_runnables_map(&a, &b, container_id.as_ref()) { + let mut locked = serde_json::Map::new(); + for (id, runnable) in runnables { + let runnable = lock_modules_app( + runnable.clone(), + job, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + worker_dir, + job_path, + base_internal_url, + token, + occupancy_metrics, + locks_to_reload, + Some(id.clone()), + dependency_map, + raw_workspace_dependencies_o, + triggered_by_relative_import, + temp_script_refs, + raw_app_runnables, + ) + .await?; + locked.insert(id.clone(), runnable); + } + m.insert(a, Value::Object(locked)); + continue; + } m.insert( a.clone(), lock_modules_app( @@ -2573,6 +2669,7 @@ async fn lock_modules_app( raw_workspace_dependencies_o, triggered_by_relative_import, temp_script_refs, + raw_app_runnables, ) .await?, ); @@ -2602,6 +2699,7 @@ async fn lock_modules_app( raw_workspace_dependencies_o, triggered_by_relative_import, temp_script_refs, + raw_app_runnables, ) .await?, ); @@ -2661,30 +2759,28 @@ pub async fn handle_app_dependency_job( .and_then(|x| x.get("temp_script_refs")) .and_then(|v| serde_json::from_str(v.get()).ok()); - // The version captured at job creation can be stale (the app may have been - // redeployed since). Relock the current latest instead, mirroring the flow - // dependency handler. - let id = if triggered_by_relative_import { - let latest_version = sqlx::query_scalar!( - "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", - job_path, - job.workspace_id - ) - .fetch_optional(db) - .await?; - match latest_version { - Some(latest_version) if latest_version != id => { + // A relative-import relock rewrites the published version in place and deploys no new one, + // and that version was tallied by the deploy that published it. + if triggered_by_relative_import { + *deployment_tallied = true; + } + + // The version captured at job creation can be stale (the app may have been redeployed + // since): relock the one published now. + let (id, version_switched) = if triggered_by_relative_import { + match published_app_version(db, &job.workspace_id, &job_path).await? { + Some(published) if published != id => { tracing::info!( - "App version changed since dependency job was queued ({} -> {}), using latest", + "App version changed since dependency job was queued ({} -> {}), using the published one", id, - latest_version + published ); - latest_version + (published, true) } - _ => id, + _ => (id, false), } } else { - id + (id, false) }; sqlx::query!( @@ -2716,6 +2812,27 @@ pub async fn handle_app_dependency_job( // TODO: Use transaction for entire segment? if let Some((app_id, value, is_raw_app)) = record { + // The components to relock were picked in the version the job was queued for, and the + // one published since may name them differently and carry its editor's stale locks. + // Every selection that can outlive its version is resolved here, against the value + // this job writes. + let relock_all = version_switched + || components_to_relock + .as_ref() + .is_some_and(|c| c.iter().any(|c| c == RELOCK_ALL_APP_COMPONENTS)); + let components_to_relock = if relock_all { + Some(app_component_ids(&value)?) + } else if is_raw_app { + expand_legacy_raw_app_components(components_to_relock, &value) + } else { + components_to_relock + }; + let read_value = value.clone(); + let raw_app_runnables = if is_raw_app { + RawAppRunnables::from_value(&job_path, &value) + } else { + RawAppRunnables::default() + }; let value = lock_modules_app( value, &job, @@ -2733,8 +2850,11 @@ pub async fn handle_app_dependency_job( None, &mut dependency_map, &raw_workspace_dependencies_o, - triggered_by_relative_import, + // A relock of every component regenerates every lock: the imported lock hashes a + // skip compares are stamped even by a deploy that kept its editor's stale lock. + triggered_by_relative_import && !relock_all, &temp_script_refs, + &raw_app_runnables, ) .await?; @@ -2747,20 +2867,11 @@ pub async fn handle_app_dependency_job( // Compute a lite version of the app value (w/ `inlineScript.{lock,code}`). let mut value_lite = value.clone(); - reduce_app(db, &job_path, &mut value_lite, app_id).await?; + reduce_app(db, &job_path, &mut value_lite, app_id, &raw_app_runnables).await?; if let Value::Object(object) = &mut value_lite { object.insert("version".to_string(), json!(id)); object.remove("files"); } - sqlx::query!( - "INSERT INTO app_version_lite (id, value) VALUES ($1, $2) - ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", - id, - sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, - ) - .execute(db) - .await?; - // Re-check cancelation to ensure we don't accidentially override an app. if sqlx::query_scalar!( "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", @@ -2777,27 +2888,63 @@ pub async fn handle_app_dependency_job( return Ok(()); } - sqlx::query!("UPDATE app_version SET value = $1 WHERE id = $2", value, id,) - .execute(db) - .await?; + let (deployment_message, parent_path) = + get_deployment_msg_and_parent_path_from_args(job.args.clone()); - // Re-publish the relocked version as latest for cache invalidation, but - // only if it is still the latest: the guard makes this a single atomic, - // never-demoting statement. Without it, a concurrent deploy that landed a - // newer version (e.g. same git-sync push) would be reverted, pointing a - // raw app's bundle_secret at a version with no bundle (404 / white screen). - if triggered_by_relative_import { - sqlx::query!( - "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE path = $2 AND workspace_id = $3 AND versions[array_upper(versions, 1)] = $1::bigint", - id, + let mut tx = db.begin().await?; + // Only the published version is relocked: a raw app's bundle is stored under the + // version id it was deployed with, so serving any other version 404s its bundle. + // The row lock orders this against a deploy appending a newer version. + let superseded = triggered_by_relative_import + && sqlx::query_scalar!( + "SELECT versions[array_upper(versions, 1)] FROM app WHERE path = $1 AND workspace_id = $2 FOR NO KEY UPDATE", &job_path, &job.workspace_id ) - .execute(db) - .await?; + .fetch_optional(&mut *tx) + .await? + .flatten() + != Some(id); + // Every relock of the app, and the dependency job of the deploy that published it, + // write this one row: overwriting a write made since it was read would drop its locks. + let current = sqlx::query_scalar!( + "SELECT value FROM app_version WHERE id = $1 FOR NO KEY UPDATE", + id + ) + .fetch_optional(&mut *tx) + .await?; + if superseded || current.as_ref() != Some(&read_value) { + drop(tx); + let log_msg = format!( + "\nApp version {id} was rewritten or superseded while its locks were generated; discarding them and queueing a relock of the published version" + ); + tracing::info!(workspace_id = %job.workspace_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, &job.workspace_id, log_msg, &db.into()).await; + tally_unfinished_dependency_deploy(db, &job, deployment_tallied).await; + if current.is_some() { + // The version published when the queued relock runs may name its components + // differently than this one, and carry its editor's stale locks. + let dependent = DependencyDependent { + importer_path: job_path, + importer_kind: "app".to_string(), + importer_node_ids: Some(vec![RELOCK_ALL_APP_COMPONENTS.to_string()]), + }; + requeue_relock(db, &job, dependent, deployment_message, parent_path).await?; + } + return Ok(()); } - let (deployment_message, parent_path) = - get_deployment_msg_and_parent_path_from_args(job.args.clone()); + sqlx::query!("UPDATE app_version SET value = $1 WHERE id = $2", value, id,) + .execute(&mut *tx) + .await?; + sqlx::query!( + "INSERT INTO app_version_lite (id, value) VALUES ($1, $2) + ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + id, + sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; let deployed_object = if is_raw_app { DeployedObject::RawApp { path: job_path, version: id, parent_path: parent_path.clone() } @@ -3311,6 +3458,7 @@ async fn capture_dependency_job( &mut locked_v, raw_workspace_dependencies_o, temp_script_refs, + windmill_parser_py_imports::module_sources(script_path, modules).as_ref(), ) .await? .0 @@ -3436,6 +3584,11 @@ async fn capture_dependency_job( // TODO: move inside gen_bun_lockfile if !wd_exist { write_file(job_dir, "main.ts", job_raw_code)?; + // The lock has to cover what the modules import, and the loader resolves a + // relative import against these files before asking for a deployed script. + if let Some(modules) = modules { + crate::worker::write_module_files(job_dir, modules, None).await?; + } } if let Some(lock) = gen_bun_lockfile( mem_peak, @@ -3457,10 +3610,9 @@ async fn capture_dependency_job( ) .await? { - // Nothing here writes the module files, so a bundle built now resolves a - // multi-file script's relative imports remotely rather than from its - // modules; caching that under a key naming them would serve the wrong - // code to every run. Leave it to the first run instead. + // A runnable with modules bundles on its first run instead: a multi-file + // script's per-module lock pass also lands here, once per module, and would + // prebundle each module as if it were an entrypoint no run ever asks for. if !wd_exist && modules.map_or(true, |m| m.is_empty()) { crate::bun_executor::prebundle_bun_script( job_raw_code, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index c47248c43a..1c810190d3 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.833.0"; +export const VERSION = "v1.835.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index 7b7da8884e..99b3647dc4 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/chat-sdk", - "version": "1.833.0", + "version": "1.835.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/chat-sdk", - "version": "1.833.0", + "version": "1.835.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 29488b7e47..e590d8656a 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "@windmill-labs/chat-sdk", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.833.0", + "version": "1.835.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0060c2c422..b4d8230bdb 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -34,6 +34,7 @@ export interface AppFile { public?: boolean; sandbox?: boolean; frontend_api?: string[]; + custom_path?: string; summary: string; policy: Policy; // Mirrors granular ACLs on the app path. Omitted from app.yaml when no @@ -123,6 +124,36 @@ export function stripEditorState(app: any) { delete app.value.parent_version; } } +/** The backend requires admin for any update that carries `custom_path`, even + * an unchanged one, so a value read from a checkout must only be sent when it + * differs from the deployed one or a non-admin can no longer deploy the app. */ +export function changedCustomPath( + local: string | undefined, + deployed: { custom_path?: string } | undefined, +): string | undefined { + return (local ?? "") === (deployed?.custom_path ?? "") ? undefined : local; +} + +/** Applies a workspace's `..custom_path.yaml`. An empty value + * clears the custom path; the app (low-code or raw) must already exist. */ +export async function pushAppCustomPath( + workspace: string, + remotePath: string, + localFile: { custom_path?: string } | undefined, +): Promise { + remotePath = remotePath.replaceAll(SEP, "/"); + const deployed = await wmill.getAppByPath({ workspace, path: remotePath }); + const customPath = changedCustomPath(localFile?.custom_path ?? "", deployed); + if (customPath === undefined) { + return; + } + log.info(colors.bold.yellow(`Setting custom path of app ${remotePath}...`)); + await wmill.updateApp({ + workspace, + path: remotePath, + requestBody: { custom_path: customPath, skip_draft_deletion: true }, + }); +} export function isExecutionModeAnonymous(app: any) { return app?.["policy"]?.["execution_mode"] == "anonymous"; } @@ -269,7 +300,8 @@ export async function pushApp( remotePath: string, localPath: string, message?: string, - permissionedAsContext?: PermissionedAsContext + permissionedAsContext?: PermissionedAsContext, + customPathFromFile = true ): Promise { if (alreadySynced.includes(localPath)) { return; @@ -326,14 +358,17 @@ export async function pushApp( extra_perms: localPerms, sandbox: _sandbox, frontend_api: _frontendApi, + custom_path: localCustomPath, ...localAppBody } = localApp as AppFile & { extra_perms?: Record; }; + const customPath = customPathFromFile ? localCustomPath : undefined; if (app) { if ( isSuperset(localAppBody, app) && + changedCustomPath(customPath, app) === undefined && isolationUpToDate(isolation, deployedPolicy, false) ) { log.info(colors.green(`App ${remotePath} is up to date`)); @@ -345,6 +380,7 @@ export async function pushApp( requestBody: { deployment_message: message, ...localAppBody, + custom_path: changedCustomPath(customPath, app), ...preserveFields, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, @@ -360,6 +396,7 @@ export async function pushApp( path: remotePath, deployment_message: message, ...localAppBody, + ...(customPath ? { custom_path: customPath } : {}), ...preserveFields, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index fdf26fa7da..fbeba2914f 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -67,7 +67,7 @@ const DEFAULT_PORT = 4000; const DEFAULT_HOST = "localhost"; // HTML template with live reload and SQL migration modal -const createHTML = (jsPath: string, cssPath: string) => ` +const createHTML = (jsPath: string, cssPath: string, appCtx: unknown) => ` @@ -228,6 +228,7 @@ const createHTML = (jsPath: string, cssPath: string) => ` + + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + + {/snippet} + diff --git a/frontend/src/lib/components/CronBuilder.svelte b/frontend/src/lib/components/CronBuilder.svelte index 3bceabb417..81f1f4490f 100644 --- a/frontend/src/lib/components/CronBuilder.svelte +++ b/frontend/src/lib/components/CronBuilder.svelte @@ -15,7 +15,7 @@ contentClasses="p-4" > {#snippet trigger()} - {/snippet} diff --git a/frontend/src/lib/components/CronInput.svelte b/frontend/src/lib/components/CronInput.svelte index d2b629d33e..ae06974c66 100644 --- a/frontend/src/lib/components/CronInput.svelte +++ b/frontend/src/lib/components/CronInput.svelte @@ -278,39 +278,46 @@
Execute schedule every
-
-
- -
+
+ - Valid range 0-59 - {:else if executeEvery == 'minute'} - - Valid range 0-59 - {:else if executeEvery == 'hour'} - - Valid range 0-23 - {:else if executeEvery == 'day-month'} - - {/if} -
+ {/if}
@@ -319,6 +326,7 @@ On day of the month {/if} -
-
- -
- - -
+ Schedule will only execute on valid calendar days @@ -374,7 +369,10 @@ {#if executeEvery == 'day-month' || executeEvery == 'month' || executeEvery == 'day-week'}
At time - +
{/if} @@ -390,7 +388,7 @@
diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index 8d2b51e8d0..e7a0769960 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -19,11 +19,15 @@ normalizeAgentRef, type LinkedAgentDraft } from './flows/linkedAgentDrafts' - import { AGENT_FLOW_LOCAL_KEYS } from './flows/agentResourceUtils' - import { agentTestInputTransforms } from './flows/agentFormFields' + import { AGENT_LINKED_STEP_KEYS } from './flows/agentResourceUtils' + import { AGENT_OVERRIDE_KEYS, agentTestInputTransforms } from './flows/agentFormFields' import { sendUserToast } from '$lib/toast' import { quickjsReady } from '$lib/utils/quickjsEval.svelte' - import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + import { previewOnBehalfOf } from '$lib/utils/previewOnBehalfOf' const operatingWorkspace = useOperatingWorkspace() @@ -57,6 +61,16 @@ } = getContext('FlowEditorContext') let previewBase = $derived($pathStore ?? '') + const operatingUser = useOperatingUser() + // A step runs as the flow does, so its test does too; a workspace script with its own + // `on_behalf_of` runs as that instead, in the flow as here. + let onBehalfOf = $derived( + previewOnBehalfOf( + operatingUser.current, + flowStore?.val?.on_behalf_of, + flowStore?.val?.on_behalf_of_email + ) + ) // Acting workspace when the flow editor runs in an AI session; else the nav workspace. let opWs = $derived(opWorkspace?.() ?? $operatingWorkspace) @@ -126,7 +140,8 @@ previewBase, undefined, devTempScriptRefs?.(), - timeout + timeout, + { onBehalfOf } ) } else if (val.type == 'script') { const script = val.hash @@ -144,7 +159,15 @@ previewBase, undefined, undefined, - timeout + timeout, + { + onBehalfOf: + previewOnBehalfOf( + operatingUser.current, + script.on_behalf_of, + script.on_behalf_of_email + ) ?? onBehalfOf + } ) } else if (val.type == 'flow') { await jobLoader?.runFlowByPath(val.path, args, callbacks) @@ -186,7 +209,13 @@ const inputTransforms = agentTestInputTransforms( agentVal.agent ? {} : ((agentVal.input_transforms ?? {}) as Record), args, - draft ? AGENT_FLOW_LOCAL_KEYS : Object.keys(args) + // An override the step does not hold is left out: fed from the form it would arrive as an + // authored empty value, which clears the agent's saved effort. + (draft ? AGENT_LINKED_STEP_KEYS : Object.keys(args)).filter( + (key) => + !(AGENT_OVERRIDE_KEYS as readonly string[]).includes(key) || + val.input_transforms?.[key] != undefined + ) ) await jobLoader?.runFlowPreview( @@ -213,7 +242,8 @@ schema }, callbacks, - previewBase + previewBase, + onBehalfOf ) } else if (val.type == 'aidecision') { const { schema } = await loadSchemaFromModule(mod, opWs) @@ -235,7 +265,8 @@ schema }, callbacks, - previewBase + previewBase, + onBehalfOf ) } else { throw Error('Not supported module type') diff --git a/frontend/src/lib/components/NavBreadcrumb.svelte b/frontend/src/lib/components/NavBreadcrumb.svelte index 4df66e0362..2169149426 100644 --- a/frontend/src/lib/components/NavBreadcrumb.svelte +++ b/frontend/src/lib/components/NavBreadcrumb.svelte @@ -17,7 +17,7 @@ not this component) — so the lead can run the whole line. import { Menu, Menubar } from '$lib/components/meltComponents' import MeltButton from '$lib/components/meltComponents/MeltButton.svelte' import WorkspacePickerBody from '$lib/components/sidebar/WorkspacePickerBody.svelte' - import BreadcrumbSegment from '$lib/components/BreadcrumbSegment.svelte' + import BreadcrumbNavPicker from './BreadcrumbNavPicker.svelte' import { userWorkspaces, workspaceStore, workspaceColor, superadmin } from '$lib/stores' import { findWorkspaceRoot } from '$lib/utils/workspaceHierarchy' import { useForkableWorkspaces } from '$lib/utils/useForkableWorkspaces.svelte' @@ -29,7 +29,7 @@ not this component) — so the lead can run the whole line. import { page } from '$app/state' import { navPageFor } from './sidebar/navPages' import { copyToClipboard, getContrastTextColor } from '$lib/utils' - import { KIND_LABEL_LOWER, kindKey } from '$lib/components/workspacePicker' + import { KIND_LABEL_LOWER } from '$lib/components/workspacePicker' import type { PageHeaderItem, PageHeaderSection } from './pageHeaderRegistry.svelte' import type { Snippet } from 'svelte' import { @@ -258,6 +258,20 @@ not this component) — so the lead can run the whole line. {/if} {/snippet} +{#snippet navPicker(measure: boolean)} + + {#if !nameOnly && !section?.noNavPicker} + {#if measure} + + + + {:else} + + {/if} + {/if} +{/snippet} + {#snippet slash(extra = '')} @@ -433,6 +447,7 @@ not this component) — so the lead can run the whole line. {@render pathLabel()} {/if} + {#if itemIsLast}{@render navPicker(measure)}{/if} {@render afterNameSlot(measure)} {#if pathCopied && !measure} {/if} {:else if section?.kind} - {#if measure} - - {@render sectionKindIcon()}{KIND_LABEL_LOWER[section.kind]} - - {:else} - {}} - /> - {/if} + + {@render sectionKindIcon()} + {KIND_LABEL_LOWER[section.kind]} + + {@render navPicker(measure)} {:else if section} {#if section.parent} @@ -507,6 +515,7 @@ not this component) — so the lead can run the whole line. >{/if} {/if} + {@render navPicker(measure)} {@render afterNameSlot(measure)} {:else if routePage} {@const RouteIcon = routePage.icon} @@ -519,6 +528,7 @@ not this component) — so the lead can run the whole line. {#if RouteIcon}{/if} {routePage.label} + {@render navPicker(measure)} {@render afterNameSlot(measure)} {/if} {/snippet} diff --git a/frontend/src/lib/components/OnBehalfOfSelector.svelte b/frontend/src/lib/components/OnBehalfOfSelector.svelte index 67648936ae..99ce21714e 100644 --- a/frontend/src/lib/components/OnBehalfOfSelector.svelte +++ b/frontend/src/lib/components/OnBehalfOfSelector.svelte @@ -26,7 +26,7 @@ - e.detail && loadUsers()} -> - {#snippet trigger()} - - - {#if selectedDisplayName} - {selectedDisplayName} - {/if} - {#if isFolderDefaultSelected} - + + {myDisplayName} + +{:else} + e.detail && loadUsers()} + > + {#snippet trigger()} + + + {#if selectedDisplayName} + {selectedDisplayName} + {/if} + {#if isFolderDefaultSelected} + + folder default + + {/if} + + {/snippet} + {#snippet content({ close: closePopover })} +
+
{label}
+ + {#if targetValue} + + {/if} + + + - {/if} - - - - -
- {/snippet} - + + {/snippet} + +{/if} -{#if !onLatest} - - {headUnknown - ? "Deploying may overwrite changes made by other users. Press 'Deploy' to see diff." - : "By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff."} - -
-{/if} - -
- - { - e.stopPropagation() - }, - onkeyup: () => { - if (appPath == '' && summary?.length > 0 && !dirtyPath) { - path?.setName( - summary - .toLowerCase() - .replace(/[^a-z0-9_]/g, '_') - .replace(/-+/g, '_') - .replace(/^-|-$/g, '') - ) - } - } - }} - bind:value={summary} - /> -
-
- -
- -
- - -
-
- - +
+ {#if !onLatest} + + {headUnknown + ? "Deploying may overwrite changes made by other users. Press 'Deploy' to see diff." + : "By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff."} + + {/if} -
- - A viewer of the app will execute the runnables of the app on behalf of the publisher (you) - - It ensures that all required resources/runnable visible for publisher but not for viewer at time - of creating the app would prevent the execution of the app. To guarantee tight security, a - policy is computed at time of deployment of the app which only allow the scripts/flows referred - to in the app to be called on behalf of. Furthermore, static parameters are not overridable. - Hence, users will only be able to use the app as intended by the publisher without risk for - leaking resources not used in the app. - - {#if canPreserve} -
- Because you are either an admin or part of the {WM_DEPLOYERS_GROUP} group, you can select another - user to run this app on behalf of. Once deployed the app will be run on behalf of - { - onBehalfOfChoice = choice - if (choice === 'me') { - policy.on_behalf_of_email = $userStore?.email - policy.on_behalf_of = `u/${$userStore?.username}` - customOnBehalfOfEmail = '' - preserveOnBehalfOf = false - } else if (choice === 'target') { - policy.on_behalf_of_email = savedOnBehalfOfEmail - policy.on_behalf_of = savedOnBehalfOf - customOnBehalfOfEmail = '' - preserveOnBehalfOf = true - } else if (choice === 'custom' && details) { - policy.on_behalf_of_email = details.email - policy.on_behalf_of = details.permissionedAs - customOnBehalfOfEmail = details.email - preserveOnBehalfOf = true +
+
+ + + { + e.stopPropagation() + }, + onkeyup: () => { + if (appPath == '' && summary?.length > 0 && !dirtyPath) { + path?.setName( + summary + .toLowerCase() + .replace(/[^a-z0-9_]/g, '_') + .replace(/-+/g, '_') + .replace(/^-|-$/g, '') + ) + } } }} - kind="app" - {canPreserve} - customValue={customOnBehalfOfEmail} - isDeployment={false} + bind:value={summary} + /> + +
+
+ +
- {/if} - - -
- -
-

Sandbox isolation

- Alpha -
-
- { - policy.sandbox = e.detail || undefined - // Counted where the toggle is flipped rather than where the policy is - // persisted: a not-yet-deployed app only mutates it locally, and skipping - // those would read as unused in the case where it is picked up front. - logFeatureUsage('app_sandbox', 'toggled', { - key: `${rawApp ? 'raw' : 'low_code'}:${e.detail ? 'on' : 'off'}` - }) - // Frontend API access exists only for a sandboxed app, so turning - // isolation off drops the declared scopes with it rather than leaving - // them set but inert. - if (!e.detail) { - policy.frontend_sdk_scopes = undefined - } - // A not-yet-deployed app has no row to PATCH — `setPublishState` (POST - // /apps/update) would 404. The flag rides along in the `policy` the first - // deploy sends (createApp), so here we only mutate it locally. Persist - // incrementally once the app exists. - if (savedApp && !newApp) { - setPublishState(e.detail ? 'Sandbox isolation enabled' : 'Sandbox isolation disabled') - } - }} - disabled={!savedApp || $operatorBuilderApps} - /> - {#if $operatorBuilderApps} -
- Required for apps built by operators, and cannot be turned off. +
+ +
- {/if} -
- Controls what the app's browser-side code can reach in each viewer's browser — distinct from the - on-behalf-of model above (which sets who its runnables run as). Off by default, the app's code - uses the viewer's own session; enable it to confine the app to a narrowly-scoped token instead, - on every surface (public URL and in-workspace). Leave it off if the app needs full browser - features (IndexedDB, third-party auth/SDKs, OAuth redirects).
- {#if newApp} -
Takes effect when you first deploy this app.
- {/if} - {#if policy.sandbox == true} -
- - Sandbox isolation is in alpha. After enabling, open the app from its public URL to confirm - it still works, and report any broken behavior. - -
- {/if} -
-{#if rawApp && policy.sandbox == true} -

Frontend API access

-
-
- Let the app's frontend code call the Windmill API through the windmill-client - SDK, authenticated as the viewer (unlike runnables, which run on behalf of the - publisher). Each viewer is asked to approve the scopes below before the app runs. Grant only - what the app needs: its code — or an XSS bug in it — can use them as that viewer. Add - windmill-client to the app's dependencies to import it; it configures itself from - the token handed to the bundle. -
- {#each FRONTEND_SDK_SCOPES as scope (scope.value)} -
- { - const current: string[] = policy.frontend_sdk_scopes ?? [] - const next = e.detail - ? [...current, scope.value] - : current.filter((s) => s !== scope.value) - // Keep the curated order so the consent banner and the stored - // consent compare stably across deploys. - const ordered = FRONTEND_SDK_SCOPES.map((s) => s.value).filter((s) => next.includes(s)) - policy.frontend_sdk_scopes = ordered.length > 0 ? ordered : undefined - // Same as sandbox: a not-yet-deployed app has no row to PATCH, so the - // scopes ride along in the first deploy's policy instead. - if (savedApp && !newApp) { - setPublishState('Frontend API access updated') +
+
+
+ { + onBehalfOfChoice = choice + if (choice === 'me') { + policy.on_behalf_of_email = $userStore?.email + policy.on_behalf_of = `u/${$userStore?.username}` + customOnBehalfOfEmail = '' + preserveOnBehalfOf = false + } else if (choice === 'target') { + policy.on_behalf_of_email = savedOnBehalfOfEmail + policy.on_behalf_of = savedOnBehalfOf + customOnBehalfOfEmail = '' + preserveOnBehalfOf = true + } else if (choice === 'custom' && details) { + policy.on_behalf_of_email = details.email + policy.on_behalf_of = details.permissionedAs + customOnBehalfOfEmail = details.email + preserveOnBehalfOf = true } }} - disabled={!savedApp} + kind="app" + {canPreserve} + customValue={customOnBehalfOfEmail} + isDeployment={false} + myPermissionedAs={myOnBehalfOf} + readOnlyWithoutPreserve /> -
{scope.description}
+ {#if onBehalfOfWillChange} + + will change from {savedOnBehalfOf} on deploy + {/if}
- {/each} - {#if newApp} -
Takes effect when you first deploy this app.
- {/if} - {#if policy.frontend_sdk_scopes?.length} -
- - A sandboxed app calls the API cross-origin, which older windmill-client versions - cannot do. An app bundled before this Windmill version fails with a CORS error until you deploy - it again, which re-bundles it against a current client. - -
- {/if} -
-{/if} - -{#if !hideSecretUrl} -

Access

- -
- {#if rulesetsLoaded && !canSetAnonymous && policy.execution_mode != 'anonymous'} - - Opening this app to anyone with the link is restricted to workspace admins and bypass users - by a workspace protection rule - -
- {/if} - {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest' && guestsAvailable} - - Opening this app to guests is restricted to workspace admins and bypass users by a workspace - protection rule - -
- {/if} -
- onAccessModeChange(e.detail)} - disabled={!savedApp} - > - {#snippet children({ item })} - - - - {/snippet} - -
-
- {#if policy.execution_mode == 'anonymous'} - Anyone holding the secret URL below can open this app without signing in. - {:else if policy.execution_mode == 'guest'} - {#if !guestsAvailable} - Guests are not available on Windmill Cloud, so this app still admits members only. They - require a self-hosted instance or a dedicated Windmill Cloud deployment. - {:else if guestUsage && !guestUsage.instance_enabled} - A superadmin has turned guests off for this instance, so this app still admits members - only. - {:else if guestAccessEnabled === undefined} - Checking whether this workspace allows guests… - {:else if guestAccessEnabled === false} - Guests are turned off for this workspace, so this app still admits members only. A - workspace admin can turn them on in the workspace settings. +
+ {#if canPreserve} + Viewers run the app's runnables as this user. {:else} - Anyone your identity provider authenticates can open this app without a Windmill account. - They join no workspace. Members of this workspace can open it too. - {#if guestUsage} - {guestUsage.guest_count} of {guestUsage.free_allowance} free guests used across this instance - in the last {guestUsage.window_days} days; beyond that, {guestUsage.metered - ? 'every four guests count as one seat' - : 'new guests are refused until the count drops'}. - {/if} + Only workspace admins and members of {WM_DEPLOYERS_GROUP} can pick another user. {/if} - {:else} - Only workspace members with read access on this app can open it. - {/if} -
- {#if !savedApp || newApp} - - {:else if secretUrlHref} -
- (embedMode = e.detail)} - options={{ left: 'URL', right: 'Embed' }} - />
- - {:else} - {/if} -
- {#if embedMode} - Paste this iframe snippet into another app. - {#if rawApp} - To embed it in a cross-origin isolated page, add wm_coep=on - Sets the cross-origin isolation headers (COEP) on the app, which such a page requires - of its iframes. The app then only loads cross-origin resources that send CORS or a - Cross-Origin-Resource-Policy header. to the URL. - {/if} - (if requiring login, top-level domain of embedding app must be the same as the one of Windmill) - {:else} - Share this url directly, or switch to Embed to get an iframe snippet. - {/if}
+
- {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase && guestsAvailable} -
-
- Embed for your own authenticated users (guest JWT) -
-
- To open this app for a user your own product already authenticates, mint a short-lived JWT - in your backend and append it to the app URL as guest.<jwt>. Each token - is its own seatless guest, confined to this app — no shared secret and no Windmill - account, unlike the plain secret URL above. -
-
- Windmill verifies the token against the workspace's guest JWT key (Workspace settings → - Guests) — a PEM public key or a JWKS URL{#if !isCloudHosted()}, or the instance's - configured issuer (JWT_EXT_JWKS_URL) when no workspace key is set{/if}. Set - the public half there; in your backend, sign each token with the matching - private key using RS256/384/512, PS256/384/512 or ES256/384 (symmetric HS* is - refused), carrying email, workspace_id = {opWs}, - app_path = {appPath} and exp (at most 24h ahead). -
- -
- Replace YOUR_GUEST_JWT with the token your backend signs per user. Past the instance's - free guest allowance a new guest email is refused (see the count above); guests already seen - in the window keep working. -
+
+ { + policy.sandbox = e.detail || undefined + // Counted where the toggle is flipped rather than where the policy is + // persisted: a not-yet-deployed app only mutates it locally, and skipping + // those would read as unused in the case where it is picked up front. + logFeatureUsage('app_sandbox', 'toggled', { + key: `${rawApp ? 'raw' : 'low_code'}:${e.detail ? 'on' : 'off'}` + }) + // Frontend API access exists only for a sandboxed app, so turning + // isolation off drops the declared scopes with it rather than leaving + // them set but inert. + if (!e.detail) { + policy.frontend_sdk_scopes = undefined + } + // A not-yet-deployed app has no row to PATCH — `setPublishState` (POST + // /apps/update) would 404. The flag rides along in the `policy` the first + // deploy sends (createApp), so here we only mutate it locally. Persist + // incrementally once the app exists. + if (savedApp && !newApp) { + setPublishState(e.detail ? 'Sandbox isolation enabled' : 'Sandbox isolation disabled') + } + }} + disabled={!savedApp || $operatorBuilderApps} + /> + {#if $operatorBuilderApps || newApp} +
+ {#if $operatorBuilderApps} + Required for apps built by operators, and cannot be turned off. + {/if} + {#if newApp} + Takes effect when you first deploy this app. + {/if}
{/if} - -
- {#if !(actingUser?.is_admin || actingUser?.is_super_admin)} - - Custom path can only be set by workspace admins + {#if !$operatorBuilderApps && policy.sandbox != true} +
+ + The app's frontend code runs with each viewer's full Windmill permissions. A future + version will ask viewers to approve this. Keep isolation off only if the app needs it. -
- {/if} - - {#if !$enterpriseLicense} - - {/if} - { - customPath = detail ? '' : undefined - if (customPath === undefined) { - customPathError = '' - } - }} - checked={customPath !== undefined} - options={{ - right: 'Use a custom URL' - }} - disabled={!$enterpriseLicense || !(actingUser?.is_admin || actingUser?.is_super_admin)} - /> +
+ {/if} - {#if customPath !== undefined} -
-
Custom path
-
- { - dirtyCustomPath = true - }} - /> -
-
Custom public URL
-
- + {#if rawApp && policy.sandbox == true} +
+
+ {#snippet badge()} + {#if sdkScopesToConfirm(policy.frontend_sdk_scopes ?? []).length} +
+ {#each FRONTEND_SDK_SCOPES.filter( (s) => policy.frontend_sdk_scopes.includes(s.value) ) as scope (scope.value)} + {scope.label} + {/each} +
+ {/if} + {/snippet} +
+ Let the app's frontend call the Windmill API as the viewer through the + windmill-client SDK, which it lists as a dependency. Grant only what the app + needs. +
+
+ {#each FRONTEND_SDK_SCOPES as scope (scope.value)} +
+ { + const current: string[] = policy.frontend_sdk_scopes ?? [] + const next = e.detail + ? [...current, scope.value] + : current.filter((s) => s !== scope.value) + // Keep the curated order so the consent banner and the stored + // consent compare stably across deploys. A leftover implicit scope is + // not in the list, so it drops here rather than keep minting a token + // for an app whose toggles all read off. + const ordered = FRONTEND_SDK_SCOPES.map((s) => s.value).filter((s) => + next.includes(s) + ) + policy.frontend_sdk_scopes = ordered.length > 0 ? ordered : undefined + // Same as sandbox: a not-yet-deployed app has no row to PATCH, so the + // scopes ride along in the first deploy's policy instead. + if (savedApp && !newApp) { + setPublishState('Frontend API access updated') + } + }} + disabled={!savedApp} + /> +
{scope.description}
+
+ {/each} +
+ {#if sdkScopesToConfirm(policy.frontend_sdk_scopes ?? []).length} +
+ + A sandboxed app calls the API cross-origin, which older windmill-client + versions cannot do. An app bundled before this Windmill version fails with a CORS error + until you deploy it again, which re-bundles it against a current client. + +
+ {/if} +
+
+ {/if} +
-
{dirtyCustomPath ? customPathError : ''} + {#if !hideSecretUrl} +
+
+ {#if rulesetsLoaded && !canSetAnonymous && policy.execution_mode != 'anonymous'} + + Opening this app to anyone with the link is restricted to workspace admins and bypass + users by a workspace protection rule + + {/if} + {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest' && guestsAvailable} + + Opening this app to guests is restricted to workspace admins and bypass users by a + workspace protection rule + + {/if} +
+ onAccessModeChange(e.detail)} + disabled={!savedApp} + > + {#snippet children({ item })} + + + + {/snippet} + +
+
+ {#if policy.execution_mode == 'anonymous'} + Anyone holding the secret URL below can open this app without signing in. + {:else if policy.execution_mode == 'guest'} + {#if !guestsAvailable} + Guests are not available on Windmill Cloud, so this app still admits members only. + They require a self-hosted instance or a dedicated Windmill Cloud deployment. + {:else if guestUsage && !guestUsage.instance_enabled} + A superadmin has turned guests off for this instance, so this app still admits members + only. + {:else if guestAccessEnabled === undefined} + Checking whether this workspace allows guests… + {:else if guestAccessEnabled === false} + Guests are turned off for this workspace, so this app still admits members only. A + workspace admin can turn them on in the workspace settings. + {:else} + Anyone your identity provider authenticates can open this app without a Windmill + account. They join no workspace. Members of this workspace can open it too. + {#if guestUsage} + {guestUsage.guest_count} of {guestUsage.free_allowance} free guests used across this + instance in the last {guestUsage.window_days} days; beyond that, {guestUsage.metered + ? 'every four guests count as one seat' + : 'new guests are refused until the count drops'}. + {/if} + {/if} + {:else} + Only workspace members with read access on this app can open it. + {/if} +
+
+ +
+ {#if !savedApp || newApp} + + {:else if secretUrlHref} +
+ (embedMode = e.detail)} + options={{ left: 'URL', right: 'Embed' }} + /> +
+ + {:else} + {/if} +
+ {#if embedMode} + Paste this iframe snippet into another app. + {#if rawApp} + To embed it in a cross-origin isolated page, add wm_coep=on + Sets the cross-origin isolation headers (COEP) on the app, which such a page + requires of its iframes. The app then only loads cross-origin resources that send + CORS or a Cross-Origin-Resource-Policy header. to the URL. + {/if} + (if requiring login, top-level domain of embedding app must be the same as the one of Windmill) + {:else} + Share this URL directly, or switch to Embed to get an iframe snippet. Only the latest + deployed version is served. + {/if} +
+
+ + {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase && guestsAvailable} +
+
+ Embed for your own authenticated users (guest JWT) +
+
+ To open this app for a user your own product already authenticates, mint a short-lived + JWT in your backend and append it to the app URL as guest.<jwt>. Each + token is its own seatless guest, confined to this app. Unlike the plain secret URL + above, there is no shared secret and no Windmill account. +
+
+ Windmill verifies the token against the workspace's guest JWT key (Workspace settings → + Guests): a PEM public key or a JWKS URL{#if !isCloudHosted()}, or the instance's + configured issuer (JWT_EXT_JWKS_URL) when no workspace key is set{/if}. + Set the public half there; in your backend, sign each token with the matching + private key using RS256/384/512, PS256/384/512 or ES256/384 (symmetric HS* is + refused), carrying email, workspace_id = {opWs}, + app_path = {appPath} and exp (at most 24h ahead). +
+ +
+ Replace YOUR_GUEST_JWT with the token your backend signs per user. Past the + instance's free guest allowance a new guest email is refused (see the count above); guests + already seen in the window keep working. +
{/if} -
-
- - You will still need to deploy the app to make visible the latest changes - - Embed this app in your own product to be used by your own users -{/if} +
+ {#if !(actingUser?.is_admin || actingUser?.is_super_admin)} + + Custom path can only be set by workspace admins + + {/if} +
+ { + customPath = detail ? '' : undefined + if (customPath === undefined) { + customPathError = '' + } + }} + checked={customPath !== undefined} + options={{ + right: 'Use a custom URL' + }} + disabled={!$enterpriseLicense || !(actingUser?.is_admin || actingUser?.is_super_admin)} + /> + {#if !$enterpriseLicense} + + {/if} +
+ + {#if customPath !== undefined} +
+ + { + dirtyCustomPath = true + } + }} + bind:value={customPath} + error={dirtyCustomPath ? customPathError : undefined} + /> + {#if dirtyCustomPath && customPathError} +
{customPathError}
+ {/if} +
+
+
Custom public URL
+ +
+ {/if} +
+ + Embed this app in your own product to be used by your own users + + {/if} +
diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index 3384ad3e87..2f94e086ad 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -83,6 +83,8 @@ * where the lite payload carries it; the flat field beside it is the apps *list* row's * spelling (ListableApp, which is what AppRow reads). */ let executionMode = $state(undefined) + /** From the lite read too: a sandboxed low-code app never fills `app`. */ + let summary = $state(undefined) let editHref = $derived(`/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined @@ -123,8 +125,8 @@ // Viewer side — used for the inline renderings (raw, and unsandboxed low-code); // the sandboxed low-code case loads inside the opaque /app_embed iframe instead. - // getAppByPath returns bundle_secret + runnables for raw apps, which - // PublicApp -> RawAppPreview needs. + // getAppLiteByPath returns the runnables (with their ids) that + // PublicApp -> RawAppPreview needs for raw apps. async function loadApp() { // Kept local and handed to PublicApp rather than written to `userStore`: this is the // membership in `workspace`, and a session preview tab shows a workspace the rest of @@ -134,8 +136,10 @@ // populated `userStore` for the workspace the page *is* on. appUser = await getUserExt(workspace) try { - const loaded: any = await AppService.getAppByPath({ workspace, path }) - // Raw apps need the bundle secret to load their bundle. getAppByPath + // The lite value: its inline scripts carry the ids of their deployed rows, the only + // code a deployed app runs (with its lock, and a raw app's imported runnables). + const loaded: any = await AppService.getAppLiteByPath({ workspace, path }) + // Raw apps need the bundle secret to load their bundle. getAppLiteByPath // doesn't compute it (unlike the public handlers), so fetch it here — the // same call the previous raw viewer used — and hand it to PublicApp -> // RawAppPreview via bundle_secret. @@ -174,8 +178,10 @@ appPerms = { path: lite?.path ?? path, extraPerms: lite?.extra_perms ?? {} } isRawApp = !!lite?.raw_app executionMode = lite?.policy?.execution_mode ?? lite?.execution_mode + summary = lite?.summary || undefined } catch (_) { appPerms = undefined + summary = undefined } } @@ -265,7 +271,7 @@ no item, and the breadcrumb would fall back to the section name "Apps". --> {#if ownsPageHeader} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index eacb3652e1..4c391c0336 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -194,21 +194,38 @@ // Top-level navigation relay (WIN-2006): in-app links to other apps (navbar // "app" items) must navigate the top page — navigating inside the opaque // iframe would load the SPA cookieless. Consumed by PublicApp via gotoFn. - setContext( - EMBED_NAV_CONTEXT_KEY, - isViewer - ? { - navigateTop: (href: string) => { - try { - window.parent.postMessage( - { type: 'wm_embed_navigate', href }, - expectedEmbedderOrigin ?? '*' - ) - } catch (_) {} - } - } - : undefined - ) + function navigateTop(href: string) { + try { + window.parent.postMessage({ type: 'wm_embed_navigate', href }, expectedEmbedderOrigin ?? '*') + } catch (_) {} + } + setContext(EMBED_NAV_CONTEXT_KEY, isViewer ? { navigateTop } : undefined) + + // Plain links (markdown, HTML components) would otherwise load inside the + // cookieless frame, or put an external site in it. Bound on so the link's own + // handlers run first and can cancel it, and before SvelteKit's router on , + // which skips a prevented click. A link to this same page only changes ?query/#hash, + // which the hash relay mirrors, so it stays in the frame. + function relayLinkClick(e: MouseEvent) { + if (e.defaultPrevented || e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) + return + // Attributes, not properties: on an SVG those are SVGAnimatedString. + const a = (e.target as Element | null)?.closest?.('a[href]') + const target = a?.getAttribute('target') + if ( + !a || + a.hasAttribute('download') || + (target && !['_self', '_top', '_parent'].includes(target)) + ) + return + const url = new URL(a.getAttribute('href')!, document.baseURI) + // The embedder only relays http(s); mailto:, tel: and the like work in the frame. + if (url.protocol !== 'http:' && url.protocol !== 'https:') return + const sameOrigin = url.origin === window.location.origin + if (sameOrigin && url.pathname === window.location.pathname) return + e.preventDefault() + navigateTop(sameOrigin ? url.pathname + url.search + url.hash : url.href) + } function installHashRelay() { origPushState = history.pushState origReplaceState = history.replaceState @@ -568,6 +585,7 @@ loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() + document.body.addEventListener('click', relayLinkClick) // Announce readiness so the embedder sends us the token. window.parent.postMessage({ type: 'wm_embed_ready' }, expectedEmbedderOrigin ?? '*') // This page is only ever loaded as the embedder's opaque iframe, which @@ -589,7 +607,10 @@ if (!BROWSER) return window.removeEventListener('message', handleViewerMessage) window.removeEventListener('message', handleEmbedderMessage) - if (isViewer) uninstallHashRelay() + if (isViewer) { + uninstallHashRelay() + document.body.removeEventListener('click', relayLinkClick) + } }) diff --git a/frontend/src/lib/components/common/table/AgentRow.svelte b/frontend/src/lib/components/common/table/AgentRow.svelte index aca6ae7f9c..05c15a582a 100644 --- a/frontend/src/lib/components/common/table/AgentRow.svelte +++ b/frontend/src/lib/components/common/table/AgentRow.svelte @@ -13,6 +13,7 @@ import ChatFlowBadge from '$lib/components/flows/ChatFlowBadge.svelte' import { keepsManagedMemory } from '$lib/components/flows/agentFormFields' import { agentMenuItems, deleteAgent } from '$lib/components/flows/agentActions' + import AgentHistory from '$lib/components/flows/AgentHistory.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' import Row from './Row.svelte' import { selectMenuItems, type RowSelection } from './rowSelection' @@ -54,8 +55,16 @@ let editHref = $derived(`/agents/edit/${agent.path}`) // A draft-only agent has nothing deployed for its page to run, so it opens in the editor. let rowHref = $derived(agent.draft_only ? editHref : `/agents/get/${agent.path}`) + let history: AgentHistory | undefined = $state(undefined) + dispatch('change')} +/> + {#snippet chatBadge()} {/snippet} @@ -69,7 +78,7 @@ path={agent.draft_path ?? agent.path} summary={agent.description} workspaceId={agent.workspace_id ?? $workspaceStore ?? ''} - canFavorite={false} + canFavorite={!agent.draft_only} {depth} {rowSelection} > @@ -110,8 +119,10 @@ canWrite: agent.canWrite, draftOnly: Boolean(agent.draft_only), wsSpecific: agent.ws_specific, + operator: $userStore?.operator, onPermissions: () => shareModal.openDrawer?.(agent.path, 'resource'), onDeploy: () => deploymentDrawer.openDrawer(agent.path, 'resource'), + onHistory: () => history?.open(), onDelete: (event) => { const { path } = agent if (event?.shiftKey) remove(path) diff --git a/frontend/src/lib/components/common/table/Row.svelte b/frontend/src/lib/components/common/table/Row.svelte index 76cd30640a..111618145f 100644 --- a/frontend/src/lib/components/common/table/Row.svelte +++ b/frontend/src/lib/components/common/table/Row.svelte @@ -260,7 +260,7 @@
{/if} - {#if canFavorite && (kind == 'app' || kind == 'raw_app' || kind == 'script' || kind == 'flow')} + {#if canFavorite && (kind == 'app' || kind == 'raw_app' || kind == 'script' || kind == 'flow' || kind == 'agent')}
diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index fb5806234c..5e54136f07 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -82,8 +82,7 @@ {#snippet trigger()} + {#if hint} + + {hint} + {/if}
{#if opened} diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 1ca2ed94fc..5694965663 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -33,7 +33,7 @@ import AiAgentStepInputs from './AiAgentStepInputs.svelte' import AgentToolBindings from './AgentToolBindings.svelte' import { getLinkedAgentTools, linkedToolsScope } from '../linkedAgentToolsStore.svelte' - import { flowLocalAgentSchema } from '../agentResourceUtils' + import { linkedAgentStepSchema } from '../agentResourceUtils' import { AI_AGENT_TOOL_AI_KEYS, AI_DECISION_TOOL_AI_KEYS } from '../agentToolUtils' import { setAiDecisionStep } from '../aiDecisionBranching' import DiffEditor from '$lib/components/DiffEditor.svelte' @@ -309,7 +309,7 @@ let inputTransformSchemaForm: { setArgs: (nargs: Record) => void } | undefined = $state(undefined) // The linked agent's memory, which decides which history inputs the step offers. - let linkedAgentMemory: { memory: unknown } | undefined = $state(undefined) + let linkedAgentConfig: { memory: unknown; provider?: unknown } | undefined = $state(undefined) let reloadError: string | undefined = $state(undefined) async function reload(flowModule: FlowModule) { @@ -1180,7 +1180,7 @@ opWorkspace={opWs} flowPath={$pathStore} fromAgentEditor={agentEditorHost?.() != undefined} - bind:linkedMemory={linkedAgentMemory} + bind:linkedAgent={linkedAgentConfig} chatInputEnabled={flowStore.val.value?.chat_input_enabled ?? false} bind:agent={ () => @@ -1233,7 +1233,7 @@ bind:this={inputTransformSchemaForm} pickableProperties={stepPropPicker.pickableProperties} schema={agentLinked - ? flowLocalAgentSchema( + ? linkedAgentStepSchema( flowStateStore.val[flowModule.id]?.schema ?? {} ) : (flowStateStore.val[flowModule.id]?.schema ?? {})} @@ -1263,7 +1263,7 @@ chatInputEnabled={flowStore.val.value?.chat_input_enabled ?? false} workspace={opWs} visibilityKey={agentFieldsKey} - linkedMemory={agentLinked ? linkedAgentMemory : undefined} + linkedAgent={agentLinked ? linkedAgentConfig : undefined} {agentLinked} tools={agentLinked ? getLinkedAgentTools( @@ -1367,7 +1367,7 @@ mod={flowModule} {noEditor} schema={agentLinked - ? flowLocalAgentSchema(flowStateStore.val[flowModule.id]?.schema ?? {}) + ? linkedAgentStepSchema(flowStateStore.val[flowModule.id]?.schema ?? {}) : (flowStateStore.val[flowModule.id]?.schema ?? {})} bind:testJob bind:testIsLoading diff --git a/frontend/src/lib/components/flows/content/FlowSettings.svelte b/frontend/src/lib/components/flows/content/FlowSettings.svelte index 0ae94b208e..dacb9efd4d 100644 --- a/frontend/src/lib/components/flows/content/FlowSettings.svelte +++ b/frontend/src/lib/components/flows/content/FlowSettings.svelte @@ -32,6 +32,8 @@ type OnBehalfOfChoice } from '$lib/components/OnBehalfOfSelector.svelte' import { modulesWithRetryOrSleep, SAME_WORKER_INCOMPATIBLE_MSG } from '../utils.svelte' + import { earlyReturnCandidates } from '../earlyReturnCandidates' + import Select from '$lib/components/select/Select.svelte' import { useOperatingUser, useOperatingWorkspace @@ -388,23 +390,26 @@ }} /> {#if flowStore.val.value.early_return} -
- ({ value: c.id, label: c.id }))} bind:value={flowStore.val.value.early_return} - class="text-xs" - > - - {#each flowStore.val.value?.modules?.map((x) => x.id) as name} - - {/each} - + placeholder="Node's id" + size="sm" + /> + {#if !selected} + + Step {flowStore.val.value.early_return} is not one of the steps above. Pick another + step. + + {:else if !selected.deterministic} + + This step only runs if its branch is taken. Otherwise, sync calls return the + flow's final result. + + {/if}
{/if}
diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index f6cc9af6f5..9d7f39ed2c 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -12,7 +12,8 @@ import { tick, untrack } from 'svelte' import type { Chat } from 'windmill-chat' import { saveFlowChatInputs } from './flowChatProps' - import type { FlowModule } from '$lib/gen' + import { ResourceService, type FlowModule } from '$lib/gen' + import { resource } from 'runed' import { useWorkspaceStorageConfigured } from '$lib/components/inputTransformEnv.svelte' import { attachmentsTargetFor, @@ -23,10 +24,12 @@ import FlowChatModelSettings from './FlowChatModelSettings.svelte' import { agentModelGap, + chatLinkedAgentPaths, composerOwnedInputs, resolveAgentModelWiring, showsModelButton, - withoutRejectedEffort + withoutRejectedEffort, + type SavedAgentProviders } from './agentChatInputs' interface Props { @@ -107,7 +110,26 @@ // The model gets its own button, shaped like the copilot's model settings, driven by // whichever provider fields the flow exposes. Attachments are the paperclip's; every other // flow input is asked for in the Configure-inputs modal. - const modelWiring = $derived(resolveAgentModelWiring(flowModules)) + // A step linked to a saved agent takes its provider from the agent, so that is read here. One + // the reader cannot load leaves the wiring unreadable, and the model button away. + const savedProviders = resource( + () => ({ workspace, paths: chatLinkedAgentPaths(flowModules) }), + async ({ workspace, paths }): Promise => { + if (!workspace || paths.length === 0) return {} + const entries = await Promise.all( + paths.map(async (path) => { + try { + const value: any = await ResourceService.getResourceValue({ workspace, path }) + return [path, value?.provider] as const + } catch { + return [path, undefined] as const + } + }) + ) + return Object.fromEntries(entries) + } + ) + const modelWiring = $derived(resolveAgentModelWiring(flowModules, savedProviders.current)) // An agent with nothing to call cannot answer, and the composer cannot fix it, so the // chat says what to go and do instead of offering controls that write nowhere. const modelGap = $derived(agentModelGap(modelWiring, subject)) diff --git a/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte b/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte index f27e14749e..f777a4cf49 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte @@ -25,6 +25,7 @@ import { resource } from 'runed' import { composerDrivenFields, + wiredFieldValue, type AgentModelWiring, type ProviderField } from './agentChatInputs' @@ -40,9 +41,7 @@ let { wiring, values, setValue, workspace }: Props = $props() function fieldValue(field: ProviderField): any { - if (wiring.whole) return values[wiring.whole]?.[field] - const name = wiring.fields[field] - return name ? values[name] : wiring.fixed[field] + return wiredFieldValue(wiring, values, field) } const driven = $derived(composerDrivenFields(wiring)) diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts index 00561330d0..6855f8bbf7 100644 --- a/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts @@ -6,6 +6,7 @@ import { parseProviderTransform, resolveAgentModelWiring, showsModelButton, + wiredFieldValue, withoutRejectedEffort } from './agentChatInputs' import type { FlowModule } from '$lib/gen' @@ -499,3 +500,84 @@ describe('parseProviderTransform with comments', () => { ).toBeUndefined() }) }) + +describe('a step linked to a saved agent', () => { + const saved = { + 'u/a/agent': { + kind: 'anthropic', + resource: '$res:u/a/claude', + model: 'claude-sonnet-4-5', + reasoning_effort: 'high' + } + } + function linked(input_transforms: Record): FlowModule { + return { + id: 'a', + value: { type: 'aiagent', agent: '$res:u/a/agent', tools: [], input_transforms } + } as unknown as FlowModule + } + const wired = linked({ + model: { type: 'javascript', expr: 'flow_input.model' }, + reasoning_effort: { type: 'javascript', expr: 'flow_input.effort' } + }) + + it('drives the overrides it wires, on the provider the agent saved', () => { + const wiring = resolveAgentModelWiring([wired], saved) + expect(wiring?.fields).toEqual({ model: 'model', reasoning_effort: 'effort' }) + expect(wiring?.fixed).toEqual({ kind: 'anthropic', resource: '$res:u/a/claude' }) + expect(agentModelWiringInputs(wiring).sort()).toEqual(['effort', 'model']) + }) + + it('shows the saved model for a model input left empty, and no effort for an empty effort', () => { + const wiring = resolveAgentModelWiring([wired], saved)! + expect(wiredFieldValue(wiring, {}, 'model')).toBe('claude-sonnet-4-5') + // Empty is how a model that reasons only on request is turned off. + expect(wiredFieldValue(wiring, { effort: '' }, 'reasoning_effort')).toBe('') + }) + + it('shows the saved effort where the step sets none, only on the saved model', () => { + const wiring = resolveAgentModelWiring( + [linked({ model: { type: 'javascript', expr: 'flow_input.model' } })], + saved + )! + expect(wiredFieldValue(wiring, {}, 'reasoning_effort')).toBe('high') + const none = resolveAgentModelWiring( + [linked({ reasoning_effort: { type: 'static', value: '' } })], + saved + )! + expect(wiredFieldValue(none, {}, 'reasoning_effort')).toBe('') + expect( + wiredFieldValue(wiring, { model: 'claude-haiku-4-5' }, 'reasoning_effort') + ).toBeUndefined() + }) + + it('does not show the saved effort for steps that fix different ones', () => { + const effort = (value: string) => + linked({ + model: { type: 'javascript', expr: 'flow_input.model' }, + reasoning_effort: { type: 'static', value } + }) + const wiring = resolveAgentModelWiring([effort('low'), effort('medium')], saved)! + expect(wiredFieldValue(wiring, {}, 'reasoning_effort')).toBeUndefined() + expect(wiredFieldValue(wiring, {}, 'model')).toBe('claude-sonnet-4-5') + + // Two agents that share a model and saved different efforts are in the same position. + const other = { ...wired, value: { ...wired.value, agent: '$res:u/a/other' } } as FlowModule + const two = resolveAgentModelWiring([wired, other], { + ...saved, + 'u/a/other': { ...saved['u/a/agent'], reasoning_effort: 'low' } + })! + expect(wiredFieldValue(two, {}, 'model')).toBe('claude-sonnet-4-5') + expect(two.saved?.reasoning_effort).toBeUndefined() + }) + + it('cannot be read until the agent has loaded, or with an expression it cannot place', () => { + expect(resolveAgentModelWiring([wired], {})).toBeUndefined() + expect( + resolveAgentModelWiring( + [linked({ model: { type: 'javascript', expr: 'pick(flow_input.model)' } })], + saved + ) + ).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts index 1c178bde26..6cfe730572 100644 --- a/frontend/src/lib/components/flows/conversations/agentChatInputs.ts +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts @@ -4,6 +4,12 @@ import { carriedReasoning } from '$lib/components/copilot/chatModelSettings' import { parseExpressionAt } from 'acorn' import { agentSteps, flowInputReads } from './agentAttachmentInput' +/** A step names its agent bare or as `$res:`/`res://`, as `linkedAgentDrafts` reads it; + * kept apart from that module, which loads the editor's draft stores. */ +function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + /** Block and line comments removed, so what is left is only what affects the value. */ function withoutComments(source: string): string { return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/[^\n]*/g, '') @@ -78,6 +84,82 @@ export type AgentModelWiring = { * which describe what the composer may offer. */ someAgentCannotRun?: boolean + /** + * What a step linked to a saved agent runs on where it overrides nothing: an empty model + * input keeps the saved model, and the saved effort applies only on that model, to a step + * that sets no effort of its own. + */ + saved?: { model?: string; reasoning_effort?: string } +} + +/** The provider saved on each linked agent, by resource path. */ +export type SavedAgentProviders = Record | undefined> + +/** The saved agents whose provider the wiring needs, as bare resource paths. */ +export function chatLinkedAgentPaths(modules: FlowModule[] | undefined): string[] { + const paths = chatFacingAgents(modules) + .map((module) => (module.value as any).agent as string | undefined) + .filter((agent): agent is string => !!agent) + .map(normalizeAgentRef) + return [...new Set(paths)].sort() +} + +/** + * Read a step linked to a saved agent: the provider is the agent's, except for the model and + * effort the step overrides, each a literal or one flow input. + */ +function parseLinkedStep( + provider: Record | undefined, + transforms: Record | undefined +): AgentModelWiring | undefined { + if (!provider || typeof provider !== 'object') return undefined + const fixed: AgentModelWiring['fixed'] = {} + for (const field of PROVIDER_FIELDS) { + if (provider[field] !== undefined) fixed[field] = provider[field] + } + const saved = { model: fixed.model, reasoning_effort: fixed.reasoning_effort } + // Read off `saved` by whoever shows it, since it holds only on the saved model. + delete fixed.reasoning_effort + const fields: AgentModelWiring['fields'] = {} + for (const field of ['model', 'reasoning_effort'] as const) { + const transform = transforms?.[field] as any + if (!transform) continue + if (transform.type === 'static') { + const value = typeof transform.value === 'string' ? transform.value.trim() : '' + // An empty model keeps the saved one; an empty effort is the step's choice of none. + if (value || field === 'reasoning_effort') fixed[field] = value + continue + } + if (transform.type !== 'javascript') return undefined + const name = flowInputName(parseExpression(transform.expr)) + if (!name) return undefined + fields[field] = name + delete fixed[field] + } + return { fields, fixed, saved } +} + +/** + * The value a provider field runs on, given what the composer holds, as the worker's + * `override_provider_model` resolves a linked agent's. + */ +export function wiredFieldValue( + wiring: AgentModelWiring, + values: Record, + field: ProviderField +): any { + if (wiring.whole) return values[wiring.whole]?.[field] + const name = wiring.fields[field] + const own = name ? values[name] : wiring.fixed[field] + if (!wiring.saved) return own + if (field === 'model') return own || wiring.saved.model + // An effort the step overrides is the run's whatever it holds: empty there is no effort. + if (field === 'reasoning_effort' && !name && own === undefined) { + return wiredFieldValue(wiring, values, 'model') === wiring.saved.model + ? wiring.saved.reasoning_effort + : own + } + return own } /** The flow input behind `flow_input.x`, `flow_input?.x` or `flow_input['x']`. */ @@ -121,23 +203,8 @@ export function parseProviderTransform( } if (transform?.type !== 'javascript') return undefined - // The author's own text may already be wrapped, so any balanced surround is fine — what - // the span check rejects is an expression with something else beside it. - const source = parenthesised(transform.expr) - let node: any - try { - node = parseExpressionAt(source, 0, { ecmaVersion: 'latest' }) - } catch { - return undefined - } - // A comment beside the expression is not another expression: the author annotating their - // own provider must not cost them the model control. Dropped before the check so what - // remains is only what would change the value — and so a paren inside a comment is not - // counted as one of the wrapping pair. - const before = withoutComments(source.slice(0, node.start)) - const after = withoutComments(source.slice(node.end)) - if (!/^[\s(]*$/.test(before) || !/^[\s)]*$/.test(after)) return undefined - if ((before.match(/\(/g)?.length ?? 0) !== (after.match(/\)/g)?.length ?? 0)) return undefined + const node = parseExpression(transform.expr) + if (!node) return undefined const whole = flowInputName(node) if (whole) return { whole, fields: {}, fixed: {} } @@ -162,6 +229,28 @@ export function parseProviderTransform( return { fields, fixed } } +/** The one expression a transform holds, or undefined where it is anything more than that. */ +function parseExpression(expr: string): any { + // The author's own text may already be wrapped, so any balanced surround is fine — what + // the span check rejects is an expression with something else beside it. + const source = parenthesised(expr) + let node: any + try { + node = parseExpressionAt(source, 0, { ecmaVersion: 'latest' }) + } catch { + return undefined + } + // A comment beside the expression is not another expression: the author annotating their + // own provider must not cost them the model control. Dropped before the check so what + // remains is only what would change the value — and so a paren inside a comment is not + // counted as one of the wrapping pair. + const before = withoutComments(source.slice(0, node.start)) + const after = withoutComments(source.slice(node.end)) + if (!/^[\s(]*$/.test(before) || !/^[\s)]*$/.test(after)) return undefined + if ((before.match(/\(/g)?.length ?? 0) !== (after.match(/\)/g)?.length ?? 0)) return undefined + return node +} + /** How one agent supplies a provider field: from an input, as a literal, or not at all. */ type FieldSupply = | { kind: 'wired'; name: string } @@ -192,12 +281,16 @@ function fieldSupply(wiring: AgentModelWiring, field: ProviderField): FieldSuppl * and field-by-field wiring is ambiguous throughout and yields nothing. */ export function resolveAgentModelWiring( - modules: FlowModule[] | undefined + modules: FlowModule[] | undefined, + savedProviders: SavedAgentProviders = {} ): AgentModelWiring | undefined { const agents = chatFacingAgents(modules) - const parsed = agents.map((agent) => - parseProviderTransform((agent.value as any).input_transforms?.['provider']) - ) + const parsed = agents.map((agent) => { + const value = agent.value as any + return value.agent + ? parseLinkedStep(savedProviders[normalizeAgentRef(value.agent)], value.input_transforms) + : parseProviderTransform(value.input_transforms?.['provider']) + }) if (parsed.length === 0) return undefined // An agent whose provider cannot be read is an agent the composer cannot speak for: // dropping it would let the rest declare a control that governs only some of them. @@ -219,19 +312,37 @@ export function resolveAgentModelWiring( const fields: AgentModelWiring['fields'] = {} const fixed: AgentModelWiring['fixed'] = {} + const disagreed = new Set() for (const field of PROVIDER_FIELDS) { // Every agent has to supply the field the same way for the composer to speak for // them all. One wired name among agents that otherwise fix it is not agreement: // the control would move that one agent and leave the others where they are. const supplies = new Set(wirings.map((w) => JSON.stringify(fieldSupply(w, field)))) + // Disagreement leaves the field neither editable nor known: a control offered here // would govern one agent while the rest ran on something else. - if (supplies.size > 1) continue + if (supplies.size > 1) { + disagreed.add(field) + continue + } const supply: FieldSupply = JSON.parse([...supplies][0]) if (supply.kind === 'wired') fields[field] = supply.name else if (supply.kind === 'fixed') fixed[field] = supply.value } - return { fields, fixed, someAgentCannotRun } + // An empty input means one thing only where every agent falls back to the same saved value, + // and a saved value must not stand in for a field the agents disagree about. The saved effort + // belongs to the saved model, so it is only kept alongside one. + const common = (field: 'model' | 'reasoning_effort') => { + const values = new Set(wirings.map((w) => w.saved?.[field])) + return values.size === 1 && !disagreed.has(field) ? [...values][0] : undefined + } + const model = common('model') + return { + fields, + fixed, + someAgentCannotRun, + saved: model === undefined ? undefined : { model, reasoning_effort: common('reasoning_effort') } + } } /** @@ -357,11 +468,9 @@ export function withoutRejectedEffort( const effortInput = wiring.fields.reasoning_effort if (!effortInput) return values - const kindInput = wiring.fields.kind - const modelInput = wiring.fields.model const rejected = rejectsEffort( - kindInput ? values[kindInput] : wiring.fixed.kind, - modelInput ? values[modelInput] : wiring.fixed.model, + wiredFieldValue(wiring, values, 'kind'), + wiredFieldValue(wiring, values, 'model'), values[effortInput] ) return rejected ? { ...values, [effortInput]: '' } : values diff --git a/frontend/src/lib/components/flows/earlyReturnCandidates.test.ts b/frontend/src/lib/components/flows/earlyReturnCandidates.test.ts new file mode 100644 index 0000000000..598d6fcd9b --- /dev/null +++ b/frontend/src/lib/components/flows/earlyReturnCandidates.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect } from 'vitest' +import type { FlowModule } from '$lib/gen' +import { earlyReturnCandidates } from './earlyReturnCandidates' + +function identity(id: string): FlowModule { + return { id, value: { type: 'identity' } } +} + +describe('earlyReturnCandidates', () => { + it('offers every step reached through branches, but not the steps inside loops', () => { + const modules: FlowModule[] = [ + identity('a'), + { + id: 'b', + value: { + type: 'branchone', + default: [identity('c')], + branches: [ + { + expr: 'true', + modules: [ + identity('d'), + { + id: 'e', + value: { + type: 'branchone', + default: [], + branches: [{ expr: 'true', modules: [identity('f')] }] + } + }, + { + id: 'g', + value: { type: 'branchall', branches: [{ modules: [identity('h')] }] } + }, + { + id: 'i', + value: { + type: 'forloopflow', + iterator: { type: 'javascript', expr: '[]' }, + skip_failures: false, + modules: [identity('j')] + } + } + ] + } + ] + } + }, + { id: 'k', value: { type: 'branchall', branches: [{ modules: [identity('l')] }] } }, + { + id: 'm', + value: { + type: 'forloopflow', + iterator: { type: 'javascript', expr: '[]' }, + skip_failures: false, + modules: [identity('n')] + } + } + ] + + expect(earlyReturnCandidates(modules)).toEqual([ + { id: 'a', deterministic: true }, + { id: 'b', deterministic: true }, + { id: 'c', deterministic: false }, + { id: 'd', deterministic: false }, + { id: 'e', deterministic: false }, + { id: 'f', deterministic: false }, + { id: 'g', deterministic: false }, + { id: 'h', deterministic: false }, + { id: 'i', deterministic: false }, + { id: 'k', deterministic: true }, + { id: 'l', deterministic: true }, + { id: 'm', deterministic: true } + ]) + }) +}) diff --git a/frontend/src/lib/components/flows/earlyReturnCandidates.ts b/frontend/src/lib/components/flows/earlyReturnCandidates.ts new file mode 100644 index 0000000000..0d838c9d2a --- /dev/null +++ b/frontend/src/lib/components/flows/earlyReturnCandidates.ts @@ -0,0 +1,28 @@ +import type { FlowModule } from '$lib/gen' + +export type EarlyReturnCandidate = { + id: string + // False for a step inside a branch to one: it only runs when its branch is taken. + deterministic: boolean +} + +// A loop's inner steps run once per iteration, so only the loop itself is offered. +export function earlyReturnCandidates(modules: FlowModule[]): EarlyReturnCandidate[] { + return withinBranches(modules, true) +} + +function withinBranches(modules: FlowModule[], deterministic: boolean): EarlyReturnCandidate[] { + return modules.flatMap((module) => { + const value = module.value + const nested = + value.type === 'branchone' + ? withinBranches([value.default, ...value.branches.map((b) => b.modules)].flat(), false) + : value.type === 'branchall' + ? withinBranches( + value.branches.flatMap((b) => b.modules), + deterministic + ) + : [] + return [{ id: module.id, deterministic }, ...nested] + }) +} diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index f2beefa0de..38a9a02c2d 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -3,8 +3,8 @@ import { loadSchemaFlow } from '$lib/scripts' import type { Schema } from '$lib/common' import { emptySchema } from '$lib/utils' import type { FlowModule, InputTransform } from '$lib/gen' -import { AGENT_FLOW_LOCAL_KEYS } from './agentResourceUtils' -import { AGENT_HISTORY_KEYS } from './agentFormFields' +import { AGENT_LINKED_STEP_KEYS } from './agentResourceUtils' +import { AGENT_UNSEEDED_KEYS } from './agentFormFields' /** Display names for the memory field's options, so anything else naming the setting an author * picked cannot drift from the button they see. */ @@ -390,20 +390,17 @@ export async function loadSchemaFromModule( } } else if (mod.type === 'aiagent') { let input_transforms = migrateAiAgentInputTransforms(mod.input_transforms ?? {}) - // A linked step's brain lives in the resource, so only the flow-local inputs get a placeholder - // transform: filling the brain keys back in would re-add the very fields linking strips, and - // they would be persisted on the next save. - const keys = mod.agent - ? (AGENT_FLOW_LOCAL_KEYS as readonly string[]) - : Object.keys(AI_AGENT_SCHEMA.properties ?? {}) + // A linked step's brain lives in the resource, so only the inputs it holds itself are kept: + // filling the brain keys back in would re-add the very fields linking strips, and they would + // be persisted on the next save. + const keys = mod.agent ? AGENT_LINKED_STEP_KEYS : Object.keys(AI_AGENT_SCHEMA.properties ?? {}) return { input_transforms: keys.reduce((accu, key) => { const transform = input_transforms[key] ?? - // A present history input is the step's choice at runtime, so it gets no placeholder. - ((AGENT_HISTORY_KEYS as readonly string[]).includes(key) - ? undefined - : { type: 'static', value: undefined }) + // A present history input or override is the step's choice at runtime, so it gets no + // placeholder. + (AGENT_UNSEEDED_KEYS.includes(key) ? undefined : { type: 'static', value: undefined }) if (transform) accu[key] = transform return accu }, {}), diff --git a/frontend/src/lib/components/flows/staleStepLocks.test.ts b/frontend/src/lib/components/flows/staleStepLocks.test.ts new file mode 100644 index 0000000000..ae12860391 --- /dev/null +++ b/frontend/src/lib/components/flows/staleStepLocks.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect } from 'vitest' +import { dropStaleStepLocks } from './staleStepLocks' + +const step = (id: string, content: string, lock?: string) => + ({ id, value: { type: 'rawscript', language: 'python3', content, lock } }) as any + +describe('dropStaleStepLocks', () => { + // A rename or duplicate gives the step a new id; its code and lock still match. + it('keeps the lock of a renamed or duplicated step whose code is unchanged', () => { + const base = { modules: [step('a', 'import requests', 'requests==2.0')] } + const value = { + modules: [ + step('a', 'import requests', 'requests==2.0'), + step('a_copy', 'import requests', 'requests==2.0'), + step('a_edited_copy', 'import yaml', 'requests==2.0') + ] + } + dropStaleStepLocks(value as any, base as any) + expect(value.modules.map((m) => m.value.lock)).toEqual([ + 'requests==2.0', + 'requests==2.0', + undefined + ]) + }) +}) diff --git a/frontend/src/lib/components/flows/staleStepLocks.ts b/frontend/src/lib/components/flows/staleStepLocks.ts new file mode 100644 index 0000000000..2b9cacfb72 --- /dev/null +++ b/frontend/src/lib/components/flows/staleStepLocks.ts @@ -0,0 +1,33 @@ +import type { FlowModule, FlowValue } from '$lib/gen' +import { forEachFlowModule } from './dfs' + +/** + * Drops, in place, the lock of every inline step whose code differs from the same step in + * `base`. The worker reuses any non-empty step lock, so a step whose code changed but kept its + * lock runs against the old dependencies. Unchanged steps keep theirs, so a deploy does not + * re-resolve (and possibly upgrade) what nobody touched. A step id `base` lacks (a renamed or + * duplicated step) keeps its lock only when some `base` step has the same code. Without a base + * nothing is dropped: a copied or imported flow carries locks that match its code. + */ +export function dropStaleStepLocks(value: FlowValue, base: FlowValue | undefined): void { + if (!base) return + const allModules = (v: FlowValue) => + [...(v.modules ?? []), v.preprocessor_module, v.failure_module].filter( + (m): m is FlowModule => m != null + ) + const contentOf = (m: FlowModule) => (m.value as { content?: unknown }).content + const baseContentById = new Map() + const baseContents = new Set() + forEachFlowModule(allModules(base), (m) => { + baseContentById.set(m.id, contentOf(m)) + baseContents.add(contentOf(m)) + }) + forEachFlowModule(allModules(value), (m) => { + const step = m.value as { content?: unknown; lock?: unknown } + if (step.lock == undefined) return + const unchanged = baseContentById.has(m.id) + ? baseContentById.get(m.id) === step.content + : baseContents.has(step.content) + if (!unchanged) delete step.lock + }) +} diff --git a/frontend/src/lib/components/flows/utils.svelte.ts b/frontend/src/lib/components/flows/utils.svelte.ts index 3efb4b30f0..3cac53cfe4 100644 --- a/frontend/src/lib/components/flows/utils.svelte.ts +++ b/frontend/src/lib/components/flows/utils.svelte.ts @@ -207,7 +207,8 @@ export async function runFlowPreview( tempScriptRefs?: Record, // The session's acting workspace when previewing inside an AI-session flow // editor; falls back to the navigation workspace for full-page previews. - workspace?: string + workspace?: string, + onBehalfOf?: string ) { const ws = workspace ?? get(workspaceStore) ?? '' const value = await withAgentDrafts(flow.value, ws) @@ -219,7 +220,8 @@ export async function runFlowPreview( path: path, tag: flow.tag, restarted_from: restartedFrom, - temp_script_refs: tempScriptRefs + temp_script_refs: tempScriptRefs, + on_behalf_of: onBehalfOf }, memoryId: conversationId }) diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte index 83e940602d..fdc76a1a00 100644 --- a/frontend/src/lib/components/home/HomeAIChat.svelte +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -50,6 +50,7 @@ import Button from '../common/button/Button.svelte' import { Badge } from '../common' import CloseButton from '../common/CloseButton.svelte' + import { logFeatureUsage } from '$lib/utils/featureUsage' import { startSessionWithPrompt } from '../sessions/sessionSwitch.svelte' import { copilotInfo, copilotWorkspace } from '$lib/aiStore' import { loadCopilot } from '$lib/components/copilot/loadCopilot' @@ -113,7 +114,9 @@ let pulsing = $state(false) let pulseEndTimer: ReturnType | undefined - function removeComposer() { + // `no_ai` is the overlay's button, shown only while AI is unusable here. + function removeComposer(source: 'close' | 'no_ai') { + logFeatureUsage('home', 'ai_hero_remove', { key: source }) setCollapsed(true) if (reducedMotion.val) return clearTimeout(pulseEndTimer) @@ -234,7 +237,12 @@
- + removeComposer('close')} + />
{/if}
@@ -325,7 +333,9 @@ {freeTierExhausted ? 'Add your own API key' : 'Configure AI'} {/if} - +
{/if} @@ -420,7 +430,10 @@ { displayName: 'Pin session hero to homepage', icon: WandSparkles, - action: () => setCollapsed(false) + action: () => { + logFeatureUsage('home', 'ai_hero_pin') + setCollapsed(false) + } } ] : []), diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index ee22e1a43a..7e71cd5a02 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -10,7 +10,6 @@ type ListableApp, type Script, ScriptService, - ResourceService, type Flow, type ListableRawApp, type ListableResource, @@ -175,12 +174,7 @@ let itemKind = $derived( (filterValues.val.kind ?? 'all') as 'script' | 'flow' | 'app' | 'agent' | 'all' ) - // Agents are `ai_agent` resources, listed through the resource endpoint: the runnables - // endpoint has no such kind, so it is asked for nothing on the agent view and for every - // kind on the combined one. - let showsRunnables = $derived(itemKind !== 'agent') - let showsAgents = $derived(itemKind === 'all' || itemKind === 'agent') - let runnableKinds = $derived(itemKind === 'all' || itemKind === 'agent' ? undefined : itemKind) + let runnableKinds = $derived(itemKind === 'all' ? undefined : itemKind) let archived = $derived(!!filterValues.val.archived) let includeWithoutMain = $derived((filterValues.val.include_library ?? true) as boolean) let filterUserFolders = $derived(!!filterValues.val.only_user_folders) @@ -318,11 +312,7 @@ let flows: TableFlow[] | undefined = $state() let apps: TableApp[] | undefined = $state() let raw_apps: TableRawApp[] | undefined = $state() - // Starts empty rather than undefined: the runnables page decides when the list renders, - // and agents join it when their own request lands. - let agents: TableAgent[] = $state([]) - let agentOwnerCounts: Record = $state({}) - let agentsGen = 0 + let agents: TableAgent[] | undefined = $state() // Monotonic fetch-order counter stamped onto each row as it arrives (see TableItem.ord). let fetchOrd = 0 @@ -353,7 +343,7 @@ } } - function mapRunnable(it: RunnableItem): TableScript | TableFlow | TableApp { + function mapRunnable(it: RunnableItem): TableScript | TableFlow | TableApp | TableAgent { const base = { ...it, canWrite: @@ -367,84 +357,14 @@ // combinedItems reads a script's time from `created_at`; the endpoint's // unified `edited_at` holds exactly that for scripts. if (it.type === 'script') return { ...base, created_at: it.edited_at } as unknown as TableScript + // An agent row reads its resource's description, which the endpoint reports as `summary`. + if (it.type === 'agent') return { ...base, description: it.summary } as unknown as TableAgent return base as unknown as TableFlow | TableApp } - /** - * The workspace's saved agents, as the kind the home page shows them as. Read from the - * resource listing, which the runnables' keyset order knows nothing of, so the order is applied - * here and the rows are stamped before every runnable ordinal: the agents lead as one block and - * a page of runnables keeps the server's order. Leading rather than trailing, or the list's - * first window would never reach them past a page of runnables. - */ - async function loadAgents(): Promise { - const ws = $workspaceStore - const gen = ++agentsGen - if (!ws || !$userStore || !showsAgents || archived) { - agents = [] - agentOwnerCounts = {} - return - } - const rows: ListableResource[] = [] - try { - // Paged even though search and scoping run over the whole set here: the listing caps a - // page, and only the first carries the draft-only rows, so a short page ends it. - const perPage = 1000 - for (let page = 1; ; page++) { - const batch = await ResourceService.listResource({ - workspace: ws, - resourceType: 'ai_agent', - includeDraftOnly: true, - page, - perPage - }) - rows.push(...batch) - if (batch.length < perPage || gen !== agentsGen) break - } - } catch (e: any) { - if (gen !== agentsGen) return - sendUserToast(`Failed to load agents: ${e?.body ?? e?.message ?? e}`, true) - agents = [] - agentOwnerCounts = {} - return - } - if (gen !== agentsGen) return - // Counted before the owner scope: the chips and tree nodes list every owner. - const counts: Record = {} - for (const r of rows) { - const owner = effectivePath(r).split('/').slice(0, 2).join('/') - counts[owner] = (counts[owner] ?? 0) + 1 - } - agentOwnerCounts = counts - const scoped = ownerFilter - ? rows.filter((r) => effectivePath(r).startsWith(ownerFilter + '/')) - : rows - const byTime = (r: ListableResource) => new Date(r.edited_at ?? 0).getTime() - const sorted = [...scoped].sort((a, b) => { - switch (sortOrder) { - case 'updated_asc': - return byTime(a) - byTime(b) - case 'name_asc': - return cmp(effectivePath(a), effectivePath(b)) - case 'name_desc': - return cmp(effectivePath(b), effectivePath(a)) - default: - return byTime(b) - byTime(a) - } - }) - agents = sorted.map((r, i) => ({ - ...r, - summary: r.description || undefined, - canWrite: editRights.canEditItem('agent', r.path, (r.extra_perms ?? {}) as any), - ord: AGENT_ORD_BASE + i - })) - } - /** Below any ordinal a runnables page stamps (they count up from 0), so agents lead as a block. */ - const AGENT_ORD_BASE = -1_000_000 - // The merged, server-ordered, keyset-paginated source. `reset` reloads from // the first page (order/filter change or workspace switch); otherwise it - // appends the next page. All three kinds arrive interleaved and are split into + // appends the next page. Every kind arrives interleaved and are split into // the existing per-kind arrays so the downstream pipeline is unchanged. async function loadRunnables(reset: boolean): Promise { const ws = $workspaceStore @@ -458,25 +378,6 @@ // arrays (mixing streams) or clobber the pending reset's generation. if (!reset && serverCursor === undefined) return if (scripts === undefined) loading = true - // One load per reset: agents are fetched whole, so a load-more has nothing to append. - const agentsLoad = reset ? loadAgents() : undefined - if (!showsRunnables) { - // Supersedes a page still in flight, which would otherwise land in the agent view. - const gen = ++loadGen - serverCursor = undefined - hasMoreServer = false - // The rows on screen stay until the agents replace them, as a reload keeps them, or the - // view reads as empty in between. - await agentsLoad - if (gen !== loadGen) return - scripts = [] - flows = [] - apps = [] - raw_apps = [] - pipelineMemberFolders = new Set() - loading = false - return - } if (reset) { serverCursor = undefined hasMoreServer = false @@ -519,18 +420,13 @@ // A newer request superseded this one (e.g. order changed mid-flight); drop // this response so a stale page/cursor can't be mixed with the new order. if (gen !== loadGen) return - // Landed together with the agents fetched beside it, or a workspace holding only agents - // reads as empty until they arrive. - if (agentsLoad) { - await agentsLoad - if (gen !== loadGen) return - } serverCursor = res.next_cursor ?? undefined hasMoreServer = !!res.next_cursor const s: TableScript[] = reset ? [] : [...(scripts ?? [])] const f: TableFlow[] = reset ? [] : [...(flows ?? [])] const a: TableApp[] = reset ? [] : [...(apps ?? [])] + const ag: TableAgent[] = reset ? [] : [...(agents ?? [])] const memberFolders = reset ? new Set() : new Set(pipelineMemberFolders) if (reset) fetchOrd = 0 for (const it of res.items ?? []) { @@ -546,11 +442,14 @@ f.push({ ...mapRunnable(it), ord: fetchOrd++ } as TableFlow) } else if (it.type === 'app') { a.push({ ...mapRunnable(it), ord: fetchOrd++ } as TableApp) + } else if (it.type === 'agent') { + ag.push({ ...mapRunnable(it), ord: fetchOrd++ } as TableAgent) } } scripts = s flows = f apps = a + agents = ag raw_apps = [] pipelineMemberFolders = memberFolders loading = false @@ -564,10 +463,13 @@ // skipping ones already present. Lets a search reach matches outside the loaded // browse window without re-fetching the whole list. function mergeRunnables(newItems: RunnableItem[]) { - const have = new Set([...(scripts ?? []), ...(flows ?? []), ...(apps ?? [])].map(itemKey)) + const have = new Set( + [...(scripts ?? []), ...(flows ?? []), ...(apps ?? []), ...(agents ?? [])].map(itemKey) + ) const s = [...(scripts ?? [])] const f = [...(flows ?? [])] const a = [...(apps ?? [])] + const ag = [...(agents ?? [])] const memberFolders = new Set(pipelineMemberFolders) let changed = false for (const it of newItems) { @@ -583,12 +485,14 @@ if (it.type === 'script') s.push(mapped as TableScript) else if (it.type === 'flow') f.push(mapped as TableFlow) else if (it.type === 'app') a.push(mapped as TableApp) + else if (it.type === 'agent') ag.push(mapped as TableAgent) changed = true } if (changed) { scripts = s flows = f apps = a + agents = ag pipelineMemberFolders = memberFolders } } @@ -656,13 +560,6 @@ // Track the prefix as open first — even a no-op call (re-expanding a cached node) // means it's on screen, so later reloads must refresh it. openOwners.add(owner) - // The agent view has no runnables to page in: its rows are the loaded agents, which - // `treeSource` adds itself. Rows an owner held from another kind go with the switch. - if (!showsRunnables) { - treeOwnerItems = treeOwnerItems.filter((x) => !effectivePath(x).startsWith(`${owner}/`)) - ownerLoad[owner] = { hasMore: false, loading: false, loaded: true, gen: treeGen } - return - } const st = ownerLoad[owner] // Only a load for the CURRENT generation blocks a new one. A load left in flight // by a superseded generation (treeGen bumped on a sort/filter reload) has already @@ -1105,7 +1002,6 @@ [() => $workspaceStore, () => archived, () => itemKind, () => includeWithoutMain], async ([ws, showArchived, kind, withoutMain]) => { if (!ws || showArchived) return undefined - if (kind === 'agent') return {} try { const res = await ScriptService.countRunnablesByOwner({ workspace: ws, @@ -1121,16 +1017,7 @@ } } ) - // The runnables endpoint never counts an agent, so the loaded agents are added per owner. - let ownerCounts = $derived.by(() => { - const runnables = ownerCountsRes.current - if (runnables == undefined) return undefined - const counts = { ...runnables } - for (const [owner, n] of Object.entries(agentOwnerCounts)) { - counts[owner] = (counts[owner] ?? 0) + n - } - return counts - }) + let ownerCounts = $derived(ownerCountsRes.current) // The counts decide which owners the tree renders, so drawing it before they land // would show every workspace folder and then prune it away. Hold the skeleton // until the first response instead — it is fetched in parallel with the listing, @@ -1314,8 +1201,7 @@ const kind = itemKind // Any term/scope change restarts search paging. searchCursor = undefined - // Agents are all loaded, so the client-side search over them is already complete. - if (term === '' || !ws || !$userStore || kind === 'agent') return + if (term === '' || !ws || !$userStore) return const handle = setTimeout(async () => { let res: { items: RunnableItem[]; next_cursor?: string } try { @@ -1371,7 +1257,7 @@ search: term, showArchived: showArchived ? true : undefined, includeWithoutMain: withoutMain ? true : undefined, - kinds: kind !== 'all' && kind !== 'agent' ? kind : undefined, + kinds: kind !== 'all' ? kind : undefined, pathStart: owner ? owner + '/' : undefined, includeDraftOnly: true, perPage: 1000, @@ -1423,7 +1309,7 @@ type: 'raw_app' as 'raw_app', time: new Date(x.edited_at).getTime() })), - ...agents.map((x) => ({ + ...(agents ?? []).map((x) => ({ ...x, type: 'agent' as 'agent', time: new Date(x.edited_at ?? 0).getTime() @@ -1487,12 +1373,10 @@ // injected as a node and its rows come from the on-demand `treeOwnerItems` store, // so the tree never depends on which owners happen to be in the loaded window. // Otherwise (scoped/search/label) the tree just groups the global `items`. - // Agents ride along in lazy mode too: the owner store is fed by the runnables endpoint, - // which never returns one, while the resource listing already holds them all. let treeSource = $derived( treeLazyMode - ? [...treeOwnerItems, ...agents.map((x) => ({ ...x, type: 'agent' as 'agent' }))].filter( - (x) => filterItemsPathsBaseOnUserFilters(x, filterUserFolders, filterUserFoldersType) + ? treeOwnerItems.filter((x) => + filterItemsPathsBaseOnUserFilters(x, filterUserFolders, filterUserFoldersType) ) : items ) diff --git a/frontend/src/lib/components/icons/ExcelIcon.svelte b/frontend/src/lib/components/icons/ExcelIcon.svelte new file mode 100644 index 0000000000..250214c56f --- /dev/null +++ b/frontend/src/lib/components/icons/ExcelIcon.svelte @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/icons/OnedriveIcon.svelte b/frontend/src/lib/components/icons/OnedriveIcon.svelte new file mode 100644 index 0000000000..71b5f60cf4 --- /dev/null +++ b/frontend/src/lib/components/icons/OnedriveIcon.svelte @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/icons/SharepointIcon.svelte b/frontend/src/lib/components/icons/SharepointIcon.svelte new file mode 100644 index 0000000000..26e1a30e72 --- /dev/null +++ b/frontend/src/lib/components/icons/SharepointIcon.svelte @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/icons/WordIcon.svelte b/frontend/src/lib/components/icons/WordIcon.svelte new file mode 100644 index 0000000000..e72371b1e6 --- /dev/null +++ b/frontend/src/lib/components/icons/WordIcon.svelte @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index 70d2450f29..8cb8ce7bb3 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -3,6 +3,10 @@ import DbIcon from './DbIcon.svelte' import DiscordIcon from './DiscordIcon.svelte' import GcalIcon from './GcalIcon.svelte' import GdriveIcon from './GdriveIcon.svelte' +import OnedriveIcon from './OnedriveIcon.svelte' +import SharepointIcon from './SharepointIcon.svelte' +import ExcelIcon from './ExcelIcon.svelte' +import WordIcon from './WordIcon.svelte' import GithubIcon from './GithubIcon.svelte' import GitlabIcon from './GitlabIcon.svelte' import GmailIcon from './GmailIcon.svelte' @@ -361,6 +365,10 @@ export const APP_TO_ICON_COMPONENT = { gcloud: GoogleCloudIcon, gcal: GcalIcon, gdrive: GdriveIcon, + ms_onedrive: OnedriveIcon, + ms_sharepoint: SharepointIcon, + ms_excel: ExcelIcon, + ms_word: WordIcon, airtable: AirtableIcon, toggl: TogglIcon, s3: S3Icon, @@ -694,6 +702,10 @@ export { DiscordIcon, GcalIcon, GdriveIcon, + OnedriveIcon, + SharepointIcon, + ExcelIcon, + WordIcon, GithubIcon, GitlabIcon, GmailIcon, diff --git a/frontend/src/lib/components/pageHeaderRegistry.svelte.ts b/frontend/src/lib/components/pageHeaderRegistry.svelte.ts index 6098095a30..91f092e88d 100644 --- a/frontend/src/lib/components/pageHeaderRegistry.svelte.ts +++ b/frontend/src/lib/components/pageHeaderRegistry.svelte.ts @@ -31,6 +31,9 @@ export type PageHeaderSection = { content?: Snippet /** Scopes the segment's picker, when the section maps to a kind of item. */ kind?: WorkspaceItemKind + /** No chevron after the name: the page's siblings are not the workspace's pages and items, + * which is all that picker offers. A session is one. */ + noNavPicker?: boolean } export type PageHeaderContent = { diff --git a/frontend/src/lib/components/raw_apps/RawAppBackgroundRunner.svelte b/frontend/src/lib/components/raw_apps/RawAppBackgroundRunner.svelte index 9f675664b1..487da15f1f 100644 --- a/frontend/src/lib/components/raw_apps/RawAppBackgroundRunner.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppBackgroundRunner.svelte @@ -5,7 +5,7 @@ import type { JobById } from '../apps/types' import { JobService } from '$lib/gen' import type { Runnable } from './rawAppPolicy' - import { RAW_APP_PREVIEW_RELAY } from './utils' + import { RAW_APP_PREVIEW_RELAY, rawAppPreviewModules } from './utils' import { undefinedIfEmpty } from '$lib/utils' interface Props { @@ -16,6 +16,8 @@ jobsById?: Record editor: boolean workspace: string + /** Editor only: identity the previewed runnables run as (see `appPreviewOnBehalfOf`). */ + previewOnBehalfOf?: string /** * Restrict waitJob/getJob/streamJob to job ids launched by this app * instance (WIN-2006): a SANDBOXED bundle must not read arbitrary @@ -45,6 +47,7 @@ jobsById = $bindable({}), editor, workspace, + previewOnBehalfOf, gateJobIds = true, extraSourceWindow }: Props = $props() @@ -155,9 +158,13 @@ ) ) : undefined, - force_viewer_delete_after_secs: editor ? runnable?.delete_after_secs : undefined + force_viewer_delete_after_secs: editor ? runnable?.delete_after_secs : undefined, + on_behalf_of: editor ? previewOnBehalfOf : undefined, + viewer_data_access: editor ? runnable?.viewer_data_access || undefined : undefined }, - undefined + undefined, + undefined, + editor ? rawAppPreviewModules(runnables, runnable_id) : undefined ) launchedJobs.add(uuid) let job: JobById = { component: runnable_id, created_at: Date.now(), job: uuid } diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 454030009b..2006aef442 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -52,6 +52,8 @@ import { captureScale, MAX_IMAGE_EDGE } from '../copilot/chat/imageUtils' import { rawAppLintStore } from './lintStore' import { dbSchemas, userStore } from '$lib/stores' + import { useActingUser } from '$lib/actingUser.svelte' + import { appPreviewOnBehalfOf } from '$lib/utils/previewOnBehalfOf' import RawAppSdkConsent from './RawAppSdkConsent.svelte' import { hasStoredSdkConsent, @@ -258,6 +260,8 @@ const opWorkspace = $derived(autosaveWorkspace ?? $operatingWorkspace) // Everything under the editor acts on it too (see operatingWorkspace.svelte.ts). setOperatingWorkspace(() => opWorkspace) + const operatingUser = useActingUser(() => opWorkspace) + let previewOnBehalfOf = $derived(appPreviewOnBehalfOf(operatingUser.current, policy)) // The path autosaves land on, which is what the session preview loads the app by. const draftStoragePath = $derived(autosavePath ?? liveEditorDraftStoragePath) @@ -460,7 +464,7 @@ addEventListener('message', function (e) { n.hidden = !e.data.text } else if (e.source === opener) frame.postMessage(e.data, '*') }) -<\/script>` +<\/script>` ], { type: 'text/html' } ) @@ -508,6 +512,8 @@ addEventListener('message', function (e) { 'The preview is waiting for the user to approve the permissions the app declares in policy.frontend_sdk_scopes, so the app code is not running and the page is empty. This is not a bug in the app: do not change the code. Ask the user to approve the permissions in the preview, then retry.' const SDK_STARTING_TEXT = 'The preview is still loading the permissions the app declares in policy.frontend_sdk_scopes, so the app code has not started yet. Retry in a moment.' + const VIEWER_PENDING_TEXT = + 'The preview is still looking up the viewer the app receives in window.ctx, so the app code has not started yet. This is not a bug in the app: do not change the code. Retry in a moment.' const PREVIEW_SETTLE_MS = 1000 let editorDestroyed = false // Latest uncaught runtime error thrown by the rendered app; cleared on next build. @@ -1813,12 +1819,13 @@ addEventListener('message', function (e) { } function syncExternalPreview() { - if (previewSdkPending || !externalPreviewReady) return + if (previewSdkPending || previewUserPending || !externalPreviewReady) return if (lastBuild) { postToExternalPreview({ type: 'preview', css: lastBuild.css, js: lastBuild.js, + ctx: previewCtx(), sdk: previewSdk, storage: previewStorage() }) @@ -1832,18 +1839,41 @@ addEventListener('message', function (e) { // Between dropping a credential and settling its replacement the shell stays // blank; whichever settles last — the mint or the shell's own `load` — starts // the app. Same for a shell still reloading: it replays once ready. - if (previewSdkPending || !previewIframeLoaded) return + if (previewSdkPending || previewUserPending || !previewIframeLoaded) return runtimeError = undefined emptyRender = false postToPreview({ type: 'preview', css: build.css, js: build.js, + ctx: previewCtx(), sdk: previewSdk, storage: previewStorage() }) } + // The deployed app's `window.ctx`: the viewer's membership in the app's workspace. + function previewCtx() { + return $state.snapshot({ ctx: operatingUser.current, workspace: opWorkspace }) + } + + // A build fed before the viewer is known keeps that viewer until the next one. Outside + // the navigation workspace the viewer is looked up; in it, `$userStore` still holds the + // previous workspace's membership right after a switch. A failed lookup also settles + // this: the app then runs with no viewer, as we don't know who it is. + const previewUserPending = $derived( + !!opWorkspace && + (!operatingUser.resolved(opWorkspace) || + (!!operatingUser.current && operatingUser.current.workspace_id !== opWorkspace)) + ) + $effect(() => { + if (previewUserPending) return + untrack(() => { + if (lastBuild) feedPreviewIframe(lastBuild) + syncExternalPreview() + }) + }) + // The shell seeds a fresh document's storage shim from this; unsandboxed // previews use real storage and get none. function previewStorage() { @@ -2005,6 +2035,7 @@ addEventListener('message', function (e) { if (buildTracker.pending) return { text: BUILD_PENDING_TEXT } if (sdkPrompt) return { text: SDK_CONSENT_PENDING_TEXT } if (previewSdkPending) return { text: SDK_STARTING_TEXT } + if (previewUserPending) return { text: VIEWER_PENDING_TEXT } const res = await readPreviewDom(query) return res && buildError !== undefined ? { text: `${buildFailedText(buildError)}\n\n${res.text}` } @@ -2183,6 +2214,7 @@ addEventListener('message', function (e) { if (buildTracker.pending) throw new Error(BUILD_PENDING_TEXT) if (sdkPrompt) throw new Error(SDK_CONSENT_PENDING_TEXT) if (previewSdkPending) throw new Error(SDK_STARTING_TEXT) + if (previewUserPending) throw new Error(VIEWER_PENDING_TEXT) if (buildError !== undefined) throw new Error(buildFailedText(buildError)) const body = await requestFromPreview('elementRect', { selector: 'body' }) if (!body?.rect) throw new Error('App preview is not ready') @@ -2569,6 +2601,7 @@ addEventListener('message', function (e) { bind:jobsById {runnables} {path} + {previewOnBehalfOf} gateJobIds={sandboxed} extraSourceWindow={() => externalPreviewWindow} /> @@ -2776,6 +2809,7 @@ addEventListener('message', function (e) { {#if selectedRunnable !== undefined} { @@ -2860,6 +2894,8 @@ addEventListener('message', function (e) { title="Replay the last build into the preview" aria-label="Rebuild" onClick={() => { + // Retries a viewer lookup that failed; the hold replays the build once it answers. + operatingUser.forgetFailures() if (lastBuild) { feedPreviewIframe(lastBuild) } @@ -2901,6 +2937,7 @@ addEventListener('message', function (e) { title="App preview" src={previewShellUrl} sandbox={sandboxed ? RAW_APP_SANDBOX_FLAGS : undefined} + allow="clipboard-read; clipboard-write; fullscreen" referrerpolicy={sandboxed ? 'no-referrer' : undefined} class="w-full flex-1 block" > diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte index 73fe0852f4..d60c2262ec 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte @@ -21,7 +21,8 @@ import SchemaForm from '../SchemaForm.svelte' import RunnableJobPanelInner from '../apps/editor/RunnableJobPanelInner.svelte' import JobLoader from '../JobLoader.svelte' - import type { Job, OpenFlow, ScriptLang } from '$lib/gen' + import { FlowService, type Job, type OpenFlow, type ScriptLang } from '$lib/gen' + import { getScriptByPath } from '$lib/scripts' import { slide } from 'svelte/transition' import { DebugToolbar, DebugPanel, debugState } from '$lib/components/debug' import LogViewer from '$lib/components/LogViewer.svelte' @@ -30,6 +31,7 @@ import { userStore } from '$lib/stores' import { isHubFlowPath } from '$lib/utils' import { sendUserToast } from '$lib/toast' + import { rawAppPreviewModules } from './utils' import { useOperatingUser, useOperatingWorkspace @@ -50,6 +52,10 @@ runnable: Runnable id: string appPath: string + /** All of the app's runnables, which a test run ships as modules for `./` imports */ + runnables?: Record + /** Identity the test runs as instead of the user (see `appPreviewOnBehalfOf`). */ + previewOnBehalfOf?: string lastDeployedCode?: string | undefined /** Called when code is selected in the editor */ onSelectionChange?: ( @@ -63,7 +69,14 @@ ) => void } - let { runnable = $bindable(), id, appPath, onSelectionChange }: Props = $props() + let { + runnable = $bindable(), + id, + appPath, + runnables, + previewOnBehalfOf, + onSelectionChange + }: Props = $props() const dispatch = createEventDispatcher() @@ -179,7 +192,15 @@ runnable.inlineScript?.content ?? '', runnable.inlineScript?.language, args, - undefined + undefined, + undefined, + undefined, + undefined, + undefined, + runnables ? rawAppPreviewModules(runnables, id) : undefined, + undefined, + undefined, + { onBehalfOf: previewOnBehalfOf, viewerDataAccess: runnable.viewer_data_access } ) } else if (isRunnableByPath(runnable)) { if (jobLoader && isRunnableByPath(runnable)) { @@ -189,12 +210,42 @@ sendUserToast('Hub flow preview is still loading', true) return } - await jobLoader.runFlowPreview(args, hubFlowPreview, undefined, runnable.path) + await jobLoader.runFlowPreview( + args, + hubFlowPreview, + undefined, + runnable.path, + previewOnBehalfOf + ) + } else if (previewOnBehalfOf) { + // The deployed app runs a path runnable as the app's identity, which a run + // by path cannot ask for: preview the deployed definition instead. + const flow = await FlowService.getFlowByPath({ workspace: opWs!, path: runnable.path }) + await jobLoader.runFlowPreview(args, flow, undefined, runnable.path, previewOnBehalfOf) } else { await jobLoader.runFlowByPath(runnable.path, args) } } else if (runnable.runType == 'script' || runnable.runType == 'hubscript') { - await jobLoader.runScriptByPath(runnable.path, args) + if (previewOnBehalfOf) { + const script = await getScriptByPath(runnable.path, opWs) + await jobLoader.runPreview( + runnable.path, + script.content, + script.language, + args, + script.tag, + script.lock, + script.hash, + undefined, + undefined, + undefined, + undefined, + undefined, + { onBehalfOf: previewOnBehalfOf } + ) + } else { + await jobLoader.runScriptByPath(runnable.path, args) + } } } } diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptsPanel.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptsPanel.svelte index 891fcfa7c8..5a2399457f 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptsPanel.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptsPanel.svelte @@ -10,6 +10,7 @@ runnables: Record selectedRunnable: string | undefined appPath: string + previewOnBehalfOf?: string /** Called when code is selected in the editor */ onSelectionChange?: ( selection: { @@ -26,6 +27,7 @@ runnables = $bindable(), selectedRunnable = $bindable(), appPath, + previewOnBehalfOf, onSelectionChange }: Props = $props() @@ -38,6 +40,8 @@ {#key selectedRunnable} { createScriptFromInlineScript(selectedRunnable ?? '', e.detail, opWs ?? '', appPath) }} diff --git a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte index fa3e7e6211..36c3fb360c 100644 --- a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte @@ -351,6 +351,7 @@ title="raw-app" src={iframeSrc} sandbox={sandboxAttr} + allow="clipboard-read; clipboard-write; fullscreen" referrerpolicy={unsandboxed ? undefined : 'no-referrer'} onload={() => (loadedSrc = iframeSrc)} style="display: block; width: 100%; height: 100%; border: none" diff --git a/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte b/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte index 28940eb02a..babe0b8c85 100644 --- a/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte @@ -2,7 +2,7 @@ import { Button } from '$lib/components/common' import Toggle from '$lib/components/Toggle.svelte' import { ShieldCheck } from 'lucide-svelte' - import { sdkScopeDescription, sdkScopeLabel } from './sdkScopes' + import { sdkScopeDescription, sdkScopeLabel, sdkScopesToConfirm } from './sdkScopes' let { scopes, @@ -35,7 +35,7 @@ The app's code will be able to call the Windmill API on your behalf, restricted to:

    - {#each scopes as scope (scope)} + {#each sdkScopesToConfirm(scopes) as scope (scope)}
  • {sdkScopeLabel(scope)}{#if sdkScopeDescription(scope)} { beforeEach(() => localStorage.clear()) it('only covers scopes the viewer actually approved', () => { - storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read']) - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(true) + storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read']) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read'])).toBe(true) // The app added a scope after the viewer consented: it must ask again // rather than silently minting a broader token. - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read', 'jobs:run'])).toBe(false) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read', 'jobs:run'])).toBe(false) }) it('does not leak one viewer or app to another', () => { - storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read']) - expect(hasStoredSdkConsent('b@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(false) - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/other', ['users:read'])).toBe(false) - expect(hasStoredSdkConsent('a@w.dev', 'other', 'u/a/app', ['users:read'])).toBe(false) + storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read']) + expect(hasStoredSdkConsent('b@w.dev', 'ws', 'u/a/app', ['jobs:read'])).toBe(false) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/other', ['jobs:read'])).toBe(false) + expect(hasStoredSdkConsent('a@w.dev', 'other', 'u/a/app', ['jobs:read'])).toBe(false) }) it('keeps the editor preview and the deployed app apart, and moves preview consent', () => { - storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read']) - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'], true)).toBe(false) - storeSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'], true) - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'])).toBe(false) + storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read']) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read'], true)).toBe(false) + storeSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['jobs:read'], true) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['jobs:read'])).toBe(false) movePreviewSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', 'u/a/real') - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/real', ['users:read'], true)).toBe(true) - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'], true)).toBe(false) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/real', ['jobs:read'], true)).toBe(true) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['jobs:read'], true)).toBe(false) + }) + + it('never asks for the identity scope every token carries', () => { + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(true) + storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read']) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read', 'users:read'])).toBe(true) }) it('treats unreadable storage as no consent', () => { localStorage.setItem('wm_sdk_consent:a@w.dev:ws:u/a/app', 'not json') - expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(false) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['jobs:read'])).toBe(false) + // Nothing to ask for: storage must not turn that into an empty prompt. + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(true) }) }) diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.ts b/frontend/src/lib/components/raw_apps/sdkScopes.ts index 238f1b7278..a4600979ba 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.ts @@ -1,7 +1,12 @@ // Frontend-SDK permissions for raw apps: the curated scopes an app author may // declare in `policy.frontend_sdk_scopes` (mirrors FRONTEND_SDK_ALLOWED_SCOPES -// in the backend `apps.rs` — both lists must stay in sync), plus the viewer-side -// consent persistence for the permission banner. +// in the backend `apps.rs` — both lists must stay in sync, except for the +// implicit scope below), plus the viewer-side consent persistence for the +// permission banner. + +/** Never toggled nor asked for: every SDK token reaches whoami without it, and whoami + * returns what the bundle already gets in `window.ctx`. */ +export const IMPLICIT_SDK_SCOPE = 'users:read' export const FRONTEND_SDK_SCOPES: { value: string; label: string; description: string }[] = [ { @@ -14,11 +19,6 @@ export const FRONTEND_SDK_SCOPES: { value: string; label: string; description: s label: 'Read jobs and results', description: 'Read jobs and their results, and list the runs the viewer can see' }, - { - value: 'users:read', - label: 'Read your identity', - description: 'Call whoami (Read the viewer username, email, groups, is_super_admin...)' - }, { value: 'resources:read', label: 'Read resources', @@ -59,10 +59,18 @@ function sdkConsentKey(viewer: string, workspace: string, path: string, preview: return `wm_sdk_consent:${preview ? 'p:' : ''}${viewer}:${workspace}:${path}` } +/** The declared scopes the viewer is asked to approve. */ +export function sdkScopesToConfirm(scopes: string[]): string[] { + return scopes.filter((s) => s !== IMPLICIT_SDK_SCOPE) +} + /** True when `approved` (a consent the viewer gave) covers every scope the app now * declares. Fewer scopes need no new consent; one more does. */ export function sdkConsentCovers(approved: unknown, scopes: string[]): boolean { - return Array.isArray(approved) && scopes.every((s) => approved.includes(s)) + const needed = sdkScopesToConfirm(scopes) + return ( + needed.length === 0 || (Array.isArray(approved) && needed.every((s) => approved.includes(s))) + ) } /** True when a previously stored "do not ask again" consent covers every @@ -80,7 +88,7 @@ export function hasStoredSdkConsent( ) return sdkConsentCovers(stored, scopes) } catch (_) { - return false + return sdkConsentCovers(null, scopes) } } diff --git a/frontend/src/lib/components/raw_apps/templates.ts b/frontend/src/lib/components/raw_apps/templates.ts index 00d7594386..e54f89fdfa 100644 --- a/frontend/src/lib/components/raw_apps/templates.ts +++ b/frontend/src/lib/components/raw_apps/templates.ts @@ -1074,20 +1074,14 @@ const App = () => { description="Windmill passes the workspace and the viewer to the app." code={${JSON.stringify(BUILDER_CONTEXT_CODE)}} > - {ctx.workspace ? ( - <> -
    - Workspace - {ctx.workspace} -
    -
    - Viewer - {viewer ? viewer.name || viewer.username : 'Anonymous visitor'} -
    - - ) : ( - The editor preview has no viewer. Deploy the app to see yours here. - )} +
    + Workspace + {ctx.workspace} +
    +
    + Viewer + {viewer ? viewer.name || viewer.username : 'Anonymous visitor'} +
@@ -1166,15 +1160,11 @@ const builderAppSvelte = ` -
- +
{ + el.addEventListener('keydown', onSearchKeydown) + return () => el.removeEventListener('keydown', onSearchKeydown) + }} + > +
+ + +
+
+ {/if} + -
- {#each familyWorkspaces as { workspace, depth, isForked, parentName }} - {@const isActive = $workspaceStore === workspace.id} - {@const forkAccent = isForked ? forkAccentStyle(workspace.color) : undefined} - + {#if separatorBefore && !searching} + + {/if} + - {@const isSelected = - isActive || - (!strictWorkspaceSelect && - depth === 0 && - lineageRoot?.id === workspace.id && - !expandedFamilies.has(workspace.id))} - {@const expandable = - !strictWorkspaceSelect && depth === 0 && familiesWithForks.has(workspace.id)} - - -
- onWorkspaceItemClick(e, workspace)} - {item} > -
-
- -
-
-
- {workspace.name}{workspace.disabled ? ' (user disabled)' : ''} + onWorkspaceItemClick(e, workspace)} + {item} + > +
+
+ +
+
+
+ {#if markedNames.get(workspace.id)} + + {@html markedNames.get(workspace.id)} + {:else} + {workspace.name} + {/if}{workspace.disabled ? ' (user disabled)' : ''} +
+ {#if workspace.is_dev_workspace} + + {/if}
- {#if workspace.is_dev_workspace} - + {#if ambiguousNames.has(workspace.name) && !markedNames.has(workspace.id)} +
+ {workspace.id} +
{/if}
- {#if ambiguousNames.has(workspace.name)} -
- {workspace.id} -
- {/if}
+ {#if isSelected} + + {/if}
- {#if isSelected} - - {/if} -
- - -
- -
- {#if expandable} - {@const expanded = expandedFamilies.has(workspace.id)} - - {:else} - + + {:else} + - + + {/if} +
+ {/each} +
+ {#if (isCloudHosted() || $superadmin || canForkHere || forkBlockedHere) && !strictWorkspaceSelect} +
+ {#if isCloudHosted() || $superadmin} +
+ + + Workspace + +
+ {/if} + {#if canForkHere} +
+ (globalForkModal.val = { opened: true })} + {item} + > + + Workspace fork + +
+ {:else if forkBlockedHere} + +
+ + Workspace fork +
+ {#snippet text()} + {OPERATOR_FORK_BLOCKED_REASON} + {/snippet} + {/if}
- {/each} + {/if} + {#if canManageWorkspace && !strictWorkspaceSelect} +
+ + + {(activeWorkspace && ambiguousNames.has(activeWorkspace.name) + ? activeWorkspace.id + : activeWorkspace?.name) ?? $workspaceStore} settings + +
+ {/if} + {#if !strictWorkspaceSelect} +
+ clearWorkspaceFromStorage()} + class={actionClass('all')} + {item} + > + All workspaces + +
+ {/if}
- {#if (isCloudHosted() || $superadmin || canForkHere || forkBlockedHere) && !strictWorkspaceSelect} + {#if isCloudHosted() && $isPremiumStore === false && !strictWorkspaceSelect}
- {#if isCloudHosted() || $superadmin} - - - Workspace - - {/if} - {#if canForkHere} - (globalForkModal.val = { opened: true })} {item}> - - Workspace fork - - {:else if forkBlockedHere} - + {#if $workspaceStore != 'demo'} + + {$workspaceUsageStore ?? '—'}/1000 free workspace execs + + {#snippet text()} + {EXECUTIONS_HINT} + {/snippet} + + +
+
+ {/if} + {#if $userStore?.is_admin} +
+ { + goto('/workspace_settings?tab=premium') + }} + {item} > - - Workspace fork -
- {#snippet text()} - {OPERATOR_FORK_BLOCKED_REASON} - {/snippet} - + Upgrade + +
{/if} -
- {/if} - {#if canManageWorkspace && !strictWorkspaceSelect} -
- - - {(activeWorkspace && ambiguousNames.has(activeWorkspace.name) - ? activeWorkspace.id - : activeWorkspace?.name) ?? $workspaceStore} settings - -
- {/if} - {#if !strictWorkspaceSelect} -
- clearWorkspaceFromStorage()} - class={itemClass} - {item} - > - All workspaces -
{/if}
-{#if isCloudHosted() && $isPremiumStore === false && !strictWorkspaceSelect} -
- {#if $workspaceStore != 'demo'} - - {$workspaceUsageStore ?? '—'}/1000 free workspace execs - - {#snippet text()} - {EXECUTIONS_HINT} - {/snippet} - - -
-
-
- {/if} - {#if $userStore?.is_admin} - { - goto('/workspace_settings?tab=premium') - }} - {item} - > - Upgrade - - {/if} -
-{/if} {#if $enterpriseLicense && !strictWorkspaceSelect} {/if} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte index f9dd3dda02..eb7e22c858 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte @@ -1,5 +1,5 @@ -
- - - Keep in mind that rulesets can take up to one minute to take effect - -
- - {#if isNameLocked} -
- Managed by the dev workspace pairing, which locates this rule by name. Its restrictions and - bypassers below can still be changed. -
- {:else if nameError} -
{nameError}
- {/if} -
+ +{#snippet description(text: string)} +

{text}

+{/snippet} - -
- -
- - !selectedUsers.includes(u)))} - placeholder="Select users..." - /> - {#if selectedUsers.length > 0} -
- {#each selectedUsers as user (user)} - - {user} - - - {/each} -
- {/if} -
-
+
+ + +
- -
-
- -
+
+
-
- Users must use a fork or git branch to make changes. Direct edits are not allowed. -
+ {@render description( + 'Users must use a fork or git branch to make changes. Direct edits are not allowed.' + )}
- - -
- -
Users cannot create forks of this workspace.
+
+ + {@render description('Users cannot create forks of this workspace.')}
- -
+
-
- Operators cannot fork a workspace unless a ruleset turns this on. It applies to every - operator, whatever the bypass list says. An operator who forks is a developer in the fork - and counts as a developer seat. There they can run any code, including against this - workspace's data tables and shared ducklakes, which the fork uses directly. They still - cannot deploy scripts, flows or apps back to this workspace. -
+ {@render description( + "Operators cannot fork a workspace unless a ruleset turns this on. It applies to every operator, whatever the bypass list says. An operator who forks is a developer in the fork and counts as a developer seat. There they can run any code, including against this workspace's data tables and shared ducklakes, which the fork uses directly. They still cannot deploy scripts, flows or apps back to this workspace." + )}
- - -
+
-
- Only workspace admins and members of wm_deployers can deploy to this workspace. - Non-deployers can still fork, browse, and request a review. -
+ {@render description( + 'Only workspace admins and members of wm_deployers can deploy to this workspace. Non-deployers can still fork, browse, and request a review.' + )}
- - -
+
-
- Only workspace admins and bypass users can make an app publicly accessible without login - (anonymous execution mode). Apps that are already public can still be redeployed. -
+ {@render description( + 'Only workspace admins and bypassers can make an app publicly accessible without login (anonymous execution mode). Apps that are already public can still be redeployed.' + )}
- - -
+
-
- Only workspace admins and bypass users can open an app to guests (anyone who signs in, - with no workspace membership and no seat). Apps that already admit guests can still be - redeployed. -
+ {@render description( + 'Only workspace admins and bypassers can open an app to guests (anyone who signs in, with no workspace membership and no seat). Apps that already admit guests can still be redeployed.' + )}
- - -
+
-
- Only workspace admins and bypass users can mint a public link to a run (readable without - login). The read-only link for workspace members stays available to everyone. -
+ {@render description( + 'Only workspace admins and bypassers can mint a public link to a run (readable without login). The read-only link for workspace members stays available to everyone.' + )}
-
-
+
+
- -
+ {#snippet actions()} - - {#if hasUnsavedChanges && !isCreateMode} - You have unsaved changes - {/if} -
-
+ {/snippet} + diff --git a/frontend/src/lib/components/workspaceSettings/WorkspaceRulesets.svelte b/frontend/src/lib/components/workspaceSettings/WorkspaceRulesets.svelte index 142c2ad29f..805dc85764 100644 --- a/frontend/src/lib/components/workspaceSettings/WorkspaceRulesets.svelte +++ b/frontend/src/lib/components/workspaceSettings/WorkspaceRulesets.svelte @@ -1,5 +1,5 @@ - - { - loadRules() - ruleDrawer?.closeDrawer() - }} - /> - + ruleDrawer?.closeDrawer()} + onUpdate={() => { + loadRules() + ruleDrawer?.closeDrawer() + }} + /> {#if !$enterpriseLicense} diff --git a/frontend/src/lib/components/workspaceSettings/WorkspaceSettingsYaml.svelte b/frontend/src/lib/components/workspaceSettings/WorkspaceSettingsYaml.svelte new file mode 100644 index 0000000000..891ed55008 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/WorkspaceSettingsYaml.svelte @@ -0,0 +1,278 @@ + + +
+

+ The workspace settings in the layout of the settings.yaml file that + wmill sync reads and writes. Deleting a key clears that setting. Secrets show as + __SENSITIVE_AND_UNCHANGED__: leave the placeholder to keep the stored value. +

+ + {#each READ_ONLY_KEYS as key, i (key)}{key}{i < READ_ONLY_KEYS.length - 1 + ? ', ' + : ''}{/each} are shown for reference and never applied: change the name from the General pane + and connect Slack from the Slack pane. + {#each NOT_CLEARABLE_KEYS as key, i (key)}{key}{i < NOT_CLEARABLE_KEYS.length - 1 + ? ' and ' + : ''}{/each} are applied when set, but deleting them does not clear them. + + +
+ {#if loading && !initialCode} + + {:else} + {#await import('$lib/components/SimpleEditor.svelte')} + + {:then Module} + + {/await} + {/if} +
+ + {#if loadError} + {loadError} + {/if} + {#if parseError} + {parseError} + {/if} + {#if applyError} + {applyError} + {/if} + {#each warnings as warning, i (i)} + {warning} + {/each} + + +
+ + +
+ {#each pendingChanges as change (change.key)} +
+ {change.key} + + {formatValue(change.before)} → {formatValue(change.after)} + +
+ {/each} +
+ diff --git a/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts index ed046418f2..29e5c73b9a 100644 --- a/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts +++ b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts @@ -1,30 +1,34 @@ import { WorkspaceService, type DatatableMigrationWithStatus } from '$lib/gen' +import { migrationScope, scopesOverlap } from '../datatableMigrationSchemas' /** - * Migrations that are defined but not yet applied. A newly-created migration - * always gets the highest timestamp, so every pending migration is "earlier": - * running the new one on its own would apply it ahead of them (out of order). + * The pending migrations a new migration declaring `sql`'s scope would run ahead of. A + * newly-created migration always gets the highest timestamp, so every pending migration is + * "earlier"; only those whose scope overlaps its own can be ones it depends on. */ export function pendingMigrations( - migrations: DatatableMigrationWithStatus[] + migrations: DatatableMigrationWithStatus[], + sql: string ): DatatableMigrationWithStatus[] { - return migrations.filter((m) => m.status !== 'ran') + const scope = migrationScope(sql) + return migrations.filter((m) => m.status !== 'ran' && scopesOverlap(m.schemas, scope)) } -/** Fetch the data table's migration status and return the pending ones. */ +/** Fetch the data table's migration status and return the pending ones `sql` would run ahead of. */ export async function fetchPendingMigrations( workspace: string, - datatableName: string + datatableName: string, + sql: string ): Promise { const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({ workspace, datatableName }) - return pendingMigrations(migrations) + return pendingMigrations(migrations, sql) } /** Confirmation copy shown before running a just-created migration ahead of * `count` still-pending earlier ones (mirrors the row-level Run warning). */ export function outOfOrderRunMessage(count: number): string { - return `${count} earlier migration(s) have not been run yet. This migration might depend on them. Run it anyway?` + return `${count} earlier migration(s) touching the same schemas have not been run yet. This migration might depend on them. Run it anyway?` } diff --git a/frontend/src/lib/components/workspaceTree.test.ts b/frontend/src/lib/components/workspaceTree.test.ts index f9c00b4d7b..96d54bb8be 100644 --- a/frontend/src/lib/components/workspaceTree.test.ts +++ b/frontend/src/lib/components/workspaceTree.test.ts @@ -7,7 +7,7 @@ import { type WorkspaceItem, type WorkspaceItemKind } from './workspacePicker' -import type { DrillBranch, DrillLeaf, DrillNode } from './drillPicker' +import { resolveScope, type DrillBranch, type DrillLeaf, type DrillNode } from './drillPicker' const item = ( kind: WorkspaceItemKind, @@ -522,6 +522,17 @@ describe('legacyScopeToPath', () => { ]) }) + it('walks every ancestor folder, so a nested dir resolves in the built tree', () => { + const loaded = { flow: [item('flow', 'f/a/b/c')], script: [] } + for (const layout of ['flat', 'by-kind'] as const) { + const tree = buildWorkspaceTree({ loaded, kinds: ['flow', 'script'], loadingKind: {}, layout }) + const path = legacyScopeToPath({ kind: 'flow', dir: 'f/a/b' }, ['flow', 'script'], layout) + expect(resolveScope(tree, path)?.children.map((c) => c.key)).toEqual([ + leafKeyFor('flow', 'f/a/b/c') + ]) + } + }) + it('flat: returns [] without a dir', () => { expect(legacyScopeToPath({ kind: 'all' }, ['flow', 'script'], 'flat')).toEqual([]) }) diff --git a/frontend/src/lib/components/workspaceTree.ts b/frontend/src/lib/components/workspaceTree.ts index 9475b58af5..6c719dafb1 100644 --- a/frontend/src/lib/components/workspaceTree.ts +++ b/frontend/src/lib/components/workspaceTree.ts @@ -271,19 +271,20 @@ export function legacyScopeToPath( layout: 'by-kind' | 'flat' = 'by-kind' ): string[] { if (!scope) return [] + // A subfolder is a child of its parent's branch, not a root entry, so a scope into + // `f/a/b` must walk `f/a` first or it resolves to nothing. + const dirChain = (kind: WorkspaceItemKind | 'all') => { + if (!scope.dir) return [] + const parts = scope.dir.split('/') + return parts.slice(1).map((_, i) => dirKey(kind, parts.slice(0, i + 2).join('/'))) + } // Flat layout: no kind branches at all — scope dirs live at root and are // always keyed under the cross-kind 'all' namespace. - if (layout === 'flat') { - return scope.dir ? [dirKey('all', scope.dir)] : [] - } + if (layout === 'flat') return dirChain('all') // Single-kind mode: there's no kind branch at root; scope's `kind` is // implicit. Only the dir (if any) makes it to the path. - if (kinds.length === 1) { - return scope.dir ? [dirKey(scope.kind, scope.dir)] : [] - } - const path: string[] = [kindKey(scope.kind)] - if (scope.dir) path.push(dirKey(scope.kind, scope.dir)) - return path + if (kinds.length === 1) return dirChain(scope.kind) + return [kindKey(scope.kind), ...dirChain(scope.kind)] } /** Return `absolutePath` shortened to its segment relative to the deepest diff --git a/frontend/src/lib/editRights.test.ts b/frontend/src/lib/editRights.test.ts index faa853e708..a6d36f693c 100644 --- a/frontend/src/lib/editRights.test.ts +++ b/frontend/src/lib/editRights.test.ts @@ -37,11 +37,18 @@ describe('edit rights', () => { expect(roleCanAuthor('script', adminOperator, undefined)).toBe(false) }) - // drafts.rs refuses an operator every draft but a builder flow, rights to write directly or not. - it('refuses an operator drafts of what they may write directly', () => { - expect(roleCanAuthor('trigger', user({}), undefined)).toBe(true) - expect(roleCanDraft('trigger', user({}), undefined)).toBe(false) - expect(roleCanDraft('resource', user({}), undefined)).toBe(false) + // drafts.rs admits an operator's draft of a kind they could deploy, and no other. + it('lets an operator draft schedules and triggers until the right is withdrawn', () => { + expect(roleCanDraft('schedule', user({}), undefined)).toBe(true) + expect(roleCanDraft('trigger', user({}), undefined)).toBe(true) + expect(roleCanDraft('schedule', user({}), { manage_schedules: false })).toBe(false) + expect(roleCanDraft('trigger', user({}), { manage_triggers: false })).toBe(false) + }) + + it('lets an operator draft resources and variables, but not scripts', () => { + expect(roleCanDraft('resource', user({}), undefined)).toBe(true) + expect(roleCanDraft('variable', user({}), undefined)).toBe(true) + expect(roleCanDraft('script', user({}), undefined)).toBe(false) }) // The role alone is not enough: a builder still cannot edit a flow in a folder they only read. diff --git a/frontend/src/lib/editRights.ts b/frontend/src/lib/editRights.ts index 9073a03d28..888b61e818 100644 --- a/frontend/src/lib/editRights.ts +++ b/frontend/src/lib/editRights.ts @@ -69,11 +69,11 @@ export function roleCanAuthor( } /** - * Whether the user may save drafts of `kind` — what every session editor writes. Mirrors drafts.rs - * `require_can_write_path`, which differs from the item handlers both ways: it admits an admin - * before its operator branch, and it refuses every operator draft but a flow or full-code app - * under its builder right — schedules, triggers, resources and variables included, though an operator may - * write those directly. + * Whether the user may save drafts of `kind` — what every session editor and the chat's draft + * tools write. Mirrors drafts.rs `require_can_write_path`, which differs from the item handlers + * both ways: it admits an admin before its operator branch, and an operator drafts only the kinds + * they could deploy — resources and variables always, flows and full-code apps under their builder + * right, schedules and triggers under the manage rights. */ export function roleCanDraft( kind: EditKind, @@ -81,7 +81,19 @@ export function roleCanDraft( operatorSettings: OperatorRights ): boolean { if (user?.is_admin || user?.is_super_admin || !user?.operator) return true - return builderGrants(kind, operatorSettings) + switch (kind) { + case 'flow': + case 'raw_app': + return builderGrants(kind, operatorSettings) + case 'schedule': + case 'trigger': + return roleCanAuthor(kind, user, operatorSettings) + case 'resource': + case 'variable': + return true + default: + return false + } } function builderGrants(kind: EditKind, operatorSettings: OperatorRights): boolean { diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index e0c29f4d7b..b3b70cfaa2 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -65,6 +65,46 @@ export const mcpEndpointTools: EndpointTool[] = [ queryFieldRenames: undefined, bodyFieldRenames: undefined }, + { + name: "getWorkspaceSettingsDocument", + description: "get the workspace settings document: The workspace's settings in the layout of the settings.yaml that `wmill sync` writes. Admin only", + instructions: "", + path: "/w/{workspace}/workspaces/settings_document", + method: "GET", + pathParamsSchema: undefined, + queryParamsSchema: undefined, + bodySchema: undefined, + queryFieldRenames: undefined, + bodyFieldRenames: undefined + }, + { + name: "applyWorkspaceSettingsDocument", + description: "apply a workspace settings document: Applies a settings document, each setting through the endpoint that owns it. Admin only", + instructions: "Send only the top-level keys to change, each whole as getWorkspaceSettingsDocument returns it: a key the document omits is left as it is, and a key set to null clears it. Keep `__SENSITIVE_AND_UNCHANGED__` for a secret you were not given. Call with dry_run true first, show the user the changes and apply only once they agree.", + path: "/w/{workspace}/workspaces/settings_document", + method: "POST", + pathParamsSchema: undefined, + queryParamsSchema: undefined, + bodySchema: { + "type": "object", + "properties": { + "document": { + "type": "object", + "description": "The settings to apply, in the layout of the document. A secret given as `__SENSITIVE_AND_UNCHANGED__` keeps its stored value; one prefixed with `$encrypted:` is decrypted with the workspace key." + }, + "dry_run": { + "description": "Only compute the changes, without applying them.", + "type": "boolean" + } + }, + "required": [ + "document" + ], + "minProperties": 1 +}, + queryFieldRenames: undefined, + bodyFieldRenames: undefined + }, { name: "listDataMetrics", description: "list declared measures and dimensions on DuckLake tables: Call this before writing any aggregate query over a DuckLake table. A declared measure is the canonical definition of that number, and reproducing it yourself will silently disagree with it (a `revenue` measure typically excludes refunds or test rows). Filter by `table` for one table's declarations, or by `path_prefix` (e.g. `f/analytics`) for everything declared under a folder; omit both to browse the whole catalog. Results are keyset-paged: a full page may mean more remain, so continue with the `cursor_*` params rather than assuming a measure does not exist. Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates can share one GROUP BY. If a number you need has no declared measure, write your own aggregate as usual. Results are limited to declarations whose producing script the caller can read", @@ -568,6 +608,10 @@ export const mcpEndpointTools: EndpointTool[] = [ "include_draft_only": { "type": "boolean", "description": "When true, append per-user draft resources whose path has\nno deployed resource. Synthesized rows carry\n`draft_only: true`.\n" + }, + "starred_only": { + "type": "boolean", + "description": "Only the `ai_agent` resources the caller starred" } }, "required": [] @@ -1303,7 +1347,7 @@ export const mcpEndpointTools: EndpointTool[] = [ "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write). Every such token can call whoami without consent, whether `users:read` is declared or not.\n" } } } @@ -1420,7 +1464,7 @@ export const mcpEndpointTools: EndpointTool[] = [ "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write). Every such token can call whoami without consent, whether `users:read` is declared or not.\n" } } }, diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index f9f1fa775e..eb0b3c48a5 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -248,6 +248,8 @@ export async function getScriptByPath( lock?: string created_at?: string hash?: string + on_behalf_of?: string + on_behalf_of_email?: string }> { if (path.startsWith('hub/')) { const { content, language, schema, lockfile } = await ScriptService.getHubScriptByPath({ path }) @@ -277,7 +279,9 @@ export async function getScriptByPath( concurrency_time_window_s: script.concurrency_time_window_s, lock: script.lock, hash: script.hash, - created_at: script.created_at + created_at: script.created_at, + on_behalf_of: script.on_behalf_of, + on_behalf_of_email: script.on_behalf_of_email } } } diff --git a/frontend/src/lib/utils/previewOnBehalfOf.ts b/frontend/src/lib/utils/previewOnBehalfOf.ts new file mode 100644 index 0000000000..5ca4d9133c --- /dev/null +++ b/frontend/src/lib/utils/previewOnBehalfOf.ts @@ -0,0 +1,33 @@ +type ActingUser = + | { is_admin?: boolean; is_super_admin?: boolean; operator?: boolean; groups?: string[] } + | undefined + +/** + * The identity an editor preview should run as: the runnable's configured run-as identity, so a + * test behaves like the current deployed run. Only admins and `wm_deployers` may preview as + * someone else, and never operators (the server refuses everyone else); anyone else previews as + * themselves, which is also what their own save would record. + */ +export function previewOnBehalfOf( + user: ActingUser, + onBehalfOf: string | null | undefined, + onBehalfOfEmail?: string | null +): string | undefined { + const canPreserve = + !user?.operator && + (!!user?.is_admin || !!user?.is_super_admin || !!user?.groups?.includes('wm_deployers')) + if (!canPreserve) return undefined + return onBehalfOf || onBehalfOfEmail || undefined +} + +/** An app's runnables run as its policy identity unless it is in viewer mode, which runs them as + * whoever views it — the editor's user, in a preview. */ +export function appPreviewOnBehalfOf( + user: ActingUser, + policy: + | { execution_mode?: string; on_behalf_of?: string; on_behalf_of_email?: string } + | undefined +): string | undefined { + if (!policy || policy.execution_mode === 'viewer') return undefined + return previewOnBehalfOf(user, policy.on_behalf_of, policy.on_behalf_of_email) +} diff --git a/frontend/src/lib/utils/workspaceHierarchy.test.ts b/frontend/src/lib/utils/workspaceHierarchy.test.ts index 14e063cba8..6855effedc 100644 --- a/frontend/src/lib/utils/workspaceHierarchy.test.ts +++ b/frontend/src/lib/utils/workspaceHierarchy.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from 'vitest' import { + buildWorkspaceHierarchy, devWorkspacesInChainAbove, findDefaultForkBase, findWorkspaceAncestors, - findWorkspaceRoot + findWorkspaceRoot, + workspaceSubtreeRows } from './workspaceHierarchy' import type { UserWorkspace } from '../stores' @@ -75,3 +77,39 @@ describe('devWorkspacesInChainAbove', () => { expect(devWorkspacesInChainAbove('prod', nestedFamily)).toEqual([]) }) }) + +describe('buildWorkspaceHierarchy', () => { + const mine = { created_by: 'me@x.com' } + const theirs = { created_by: 'other@x.com' } + const forks = [ + ws('wm-fork-a', 'prod', theirs), + ws('wm-fork-b', 'prod', theirs), + ws('wm-fork-c', 'prod', theirs), + ws('wm-fork-y', 'prod', mine), + ws('wm-fork-z', 'prod', mine) + ] + const ids = (rows: ReturnType) => + rows.map((h) => (h.separatorBefore ? '| ' : '') + h.workspace.id) + + it("puts the user's own forks after the dev workspace, divided from the rest", () => { + const all = [root, ws('devws', 'prod', { is_dev_workspace: true, ...mine }), ...forks] + expect(ids(workspaceSubtreeRows('prod', buildWorkspaceHierarchy(all, 'me@x.com')))).toEqual([ + 'devws', + 'wm-fork-y', + 'wm-fork-z', + '| wm-fork-a', + 'wm-fork-b', + 'wm-fork-c' + ]) + }) + + it('draws no divider in a short list', () => { + const all = [root, ...forks.slice(2)] + expect(ids(buildWorkspaceHierarchy(all, 'me@x.com'))).toEqual([ + 'prod', + 'wm-fork-y', + 'wm-fork-z', + 'wm-fork-c' + ]) + }) +}) diff --git a/frontend/src/lib/utils/workspaceHierarchy.ts b/frontend/src/lib/utils/workspaceHierarchy.ts index 55d54e1b2c..c19eae3c74 100644 --- a/frontend/src/lib/utils/workspaceHierarchy.ts +++ b/frontend/src/lib/utils/workspaceHierarchy.ts @@ -6,14 +6,26 @@ export interface WorkspaceHierarchyItem { isForked: boolean parentName?: string hasChildren: boolean + /** First fork after the caller's own forks among its siblings; pickers draw a divider above it. */ + separatorBefore?: boolean } +/** Sibling forks below this count stay undivided: a divider in a short list is noise. */ +const OWNED_FORKS_SEPARATOR_MIN_SIBLINGS = 5 + /** * Builds a hierarchical structure from a flat array of workspaces. * Supports unlimited nesting levels (fork of fork of fork...). * Returns a flattened array with hierarchy metadata for easy rendering. + * + * With `currentUserEmail`, the forks that user created lead each sibling group (after the dev + * workspace). When every admin and developer is added to new forks, a family lists everyone's forks, + * and without this the user's own are scattered alphabetically among them. */ -export function buildWorkspaceHierarchy(workspaces: UserWorkspace[]): WorkspaceHierarchyItem[] { +export function buildWorkspaceHierarchy( + workspaces: UserWorkspace[], + currentUserEmail?: string | null +): WorkspaceHierarchyItem[] { if (!workspaces || workspaces.length === 0) { return [] } @@ -45,12 +57,15 @@ export function buildWorkspaceHierarchy(workspaces: UserWorkspace[]): WorkspaceH const result: WorkspaceHierarchyItem[] = [] + const isOwnedFork = (w: UserWorkspace) => !w.is_dev_workspace && isForkOwner(w, currentUserEmail) + // Recursively build the hierarchy function addWorkspaceAndChildren( workspace: UserWorkspace, depth: number, isForked: boolean, - parentName?: string + parentName?: string, + separatorBefore?: boolean ) { // Add the current workspace result.push({ @@ -58,19 +73,31 @@ export function buildWorkspaceHierarchy(workspaces: UserWorkspace[]): WorkspaceH depth, isForked, parentName, - hasChildren: hasChildrenSet.has(workspace.id) + hasChildren: hasChildrenSet.has(workspace.id), + separatorBefore }) - // Add its children: the canonical dev workspace first, then throwaway forks by name. - const children = childrenMap.get(workspace.id) || [] - children - .sort((a, b) => { - if (!!a.is_dev_workspace !== !!b.is_dev_workspace) return a.is_dev_workspace ? -1 : 1 - return a.name.localeCompare(b.name) - }) - .forEach((child) => { - addWorkspaceAndChildren(child, depth + 1, true, workspace.name) - }) + // Add its children: the canonical dev workspace first, then the user's own forks, then the + // other forks, each by name. + const children = (childrenMap.get(workspace.id) || []).sort((a, b) => { + if (!!a.is_dev_workspace !== !!b.is_dev_workspace) return a.is_dev_workspace ? -1 : 1 + if (isOwnedFork(a) !== isOwnedFork(b)) return isOwnedFork(a) ? -1 : 1 + return a.name.localeCompare(b.name) + }) + const forkCount = children.filter((c) => !c.is_dev_workspace).length + const firstOtherFork = + forkCount >= OWNED_FORKS_SEPARATOR_MIN_SIBLINGS && children.some(isOwnedFork) + ? children.find((c) => !c.is_dev_workspace && !isOwnedFork(c)) + : undefined + children.forEach((child) => { + addWorkspaceAndChildren( + child, + depth + 1, + true, + workspace.name, + child === firstOtherFork || undefined + ) + }) } // Process root workspaces (sorted by name for consistency) @@ -89,6 +116,21 @@ export function buildWorkspaceHierarchy(workspaces: UserWorkspace[]): WorkspaceH return result } +/** + * The hierarchy rows below `rootId` (the root itself excluded), in the order and with the depths + * `buildWorkspaceHierarchy` gives them. + */ +export function workspaceSubtreeRows( + rootId: string, + hierarchy: WorkspaceHierarchyItem[] +): WorkspaceHierarchyItem[] { + const at = hierarchy.findIndex((h) => h.workspace.id === rootId) + if (at < 0) return [] + const rootDepth = hierarchy[at].depth + const end = hierarchy.findIndex((h, i) => i > at && h.depth <= rootDepth) + return hierarchy.slice(at + 1, end < 0 ? undefined : end) +} + /** * Helper function to get the indentation padding based on depth. * Each level adds 24px of left padding. diff --git a/frontend/src/lib/utils_draft_deploy.test.ts b/frontend/src/lib/utils_draft_deploy.test.ts index 5f503989c8..3164a05bfc 100644 --- a/frontend/src/lib/utils_draft_deploy.test.ts +++ b/frontend/src/lib/utils_draft_deploy.test.ts @@ -124,6 +124,64 @@ describe('deployDraft preserves on_behalf_of', () => { }) }) +// The backend skips the dependency job when `lock` is set, so a draft that inherited the +// deployed lock would deploy new content against the old lockfile. +describe('deployDraft: script lock', () => { + beforeEach(() => vi.clearAllMocks()) + + it('never sends the draft lock, so the backend relocks', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + hash: 'v1', + draft: { + path: 'f/admin/fetch', + content: 'import requests', + lock: 'certifi==2024.1.1', + lock_error_logs: 'old error' + } + } as any) + + await deployDraft('script', 'f/admin/fetch', 'ws') + const body = vi.mocked(ScriptService.createScript).mock.calls[0][0].requestBody + expect(body).not.toHaveProperty('lock') + expect(body).not.toHaveProperty('lock_error_logs') + }) + + it('flow: drops the lock of a step whose code differs from the deployed one only', async () => { + const step = (id: string, content: string) => ({ + id, + value: { type: 'rawscript', language: 'python3', content, lock: 'requests==2.0' } + }) + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + value: { modules: [step('edited', 'import requests'), step('kept', 'import requests')] }, + draft: { + path: 'f/admin/flow', + value: { modules: [step('edited', 'import yaml'), step('kept', 'import requests')] } + } + } as any) + + await deployDraft('flow', 'f/admin/flow', 'ws') + const [edited, kept] = vi.mocked(FlowService.updateFlow).mock.calls[0][0].requestBody.value + .modules as any[] + expect(edited.value).not.toHaveProperty('lock') + expect(kept.value.lock).toBe('requests==2.0') + }) + + // A never-deployed flow is often a copy or an import, whose locks match its code. + it('flow: keeps every lock when nothing is deployed to compare against', async () => { + const step = { id: 'copied', value: { type: 'rawscript', content: 'x', lock: 'requests==2.0' } } + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + no_deployed: true, + value: { modules: [step] }, + draft: { path: 'f/admin/copy', value: { modules: [step] } } + } as any) + + await deployDraft('flow', 'f/admin/copy', 'ws', { draftOnly: true }) + const [copied] = vi.mocked(FlowService.createFlow).mock.calls[0][0].requestBody.value + .modules as any[] + expect(copied.value.lock).toBe('requests==2.0') + }) +}) + // The resource branch reads the item again when the deploy lands, and falls back to the deployed // row when the draft has gone. That row keeps its value under `value` and carries no `args` at // all, so reading it as a draft (`value: d.args ?? {}`) would replace a live resource with `{}`. diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index 611d5b96e8..b17b17cbc7 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -46,6 +46,7 @@ import { setLocalDraftHint } from '$lib/localDraftHints.svelte' import { userStore } from '$lib/stores' import { deployTriggers, type Trigger } from '$lib/components/triggers/utils' import { saveScheduleFromCfg } from '$lib/components/flows/scheduleUtils' +import { dropStaleStepLocks } from '$lib/components/flows/staleStepLocks' import { saveHttpRouteFromCfg } from '$lib/components/triggers/http/utils' import { saveWebsocketTriggerFromCfg } from '$lib/components/triggers/websocket/utils' import { savePostgresTriggerFromCfg } from '$lib/components/triggers/postgres/utils' @@ -469,7 +470,15 @@ export async function deployDraft( const r = (await ScriptService.getScriptByPath({ workspace, path, getDraft: true })) as any const d = r.draft ?? r // Drop editor-only / server-managed keys; deploy as a real (non-draft) version. - const { draft_triggers: draftTriggers, draft_only: _o, ...rest } = d + // A draft seeded from the deployed script carries its `lock`, and the backend skips + // the dependency job whenever `lock` is set, so the new content would keep the old lock. + const { + draft_triggers: draftTriggers, + draft_only: _o, + lock: _lock, + lock_error_logs: _lockErrorLogs, + ...rest + } = d const scriptPath = d.path ?? path // Deploy at the draft's path so a rename in the draft is honored (same as // the editor: createScript at the new path with parent_hash links lineage). @@ -490,6 +499,9 @@ export async function deployDraft( } else if (kind === 'flow') { const r = (await FlowService.getFlowByPath({ workspace, path, getDraft: true })) as any const d = r.draft ?? r + const value = d.value ? structuredClone(d.value) : d.value + // With nothing deployed the server fills `r.value` from the draft itself. + if (value) dropStaleStepLocks(value, draftOnly || r.no_deployed ? undefined : r.value) const requestBody = { // Deploy at the draft's intended path: flow/app/raw-app drafts keep the // user-typed path in `draft_path` (a never-deployed item is parked at a @@ -498,7 +510,7 @@ export async function deployDraft( path: d.draft_path ?? d.path ?? path, summary: d.summary ?? '', description: d.description ?? '', - value: d.value, + value, schema: d.schema, tag: d.tag, dedicated_worker: d.dedicated_worker, @@ -551,7 +563,7 @@ export async function deployDraft( // undefined so the backend preserves the existing route. The draft has no // custom_path, so admins fall back to the deployed route (`''` when none). const isAdmin = !!(get(userStore)?.is_admin || get(userStore)?.is_super_admin) - const policy = r.policy ?? { execution_mode: 'publisher' } + const policy = r.policy ?? { execution_mode: 'publisher', sandbox: true } const requestBody = { value: appValue, summary: draftSummary ?? r.summary ?? '', diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 2dcdeab5dd..e5c4b45788 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -7,6 +7,7 @@ AssetService, FlowService, OpenAPI, + ResourceService, ScriptService, SettingService, UserService, @@ -670,6 +671,11 @@ workspace: $workspaceStore ?? '', starredOnly: true }) + const agents = await ResourceService.listResource({ + workspace: $workspaceStore ?? '', + resourceType: 'ai_agent', + starredOnly: true + }) const assets = await AssetService.listFavoriteAssets({ workspace: $workspaceStore ?? '' }) favoriteManager.current = [ ...scripts.map((s) => ({ @@ -690,6 +696,12 @@ href: getFavoriteHref(f.path, 'app'), kind: 'app' as const })), + ...agents.map((a) => ({ + label: a.description || getFavoriteLabel(a.path, 'agent'), + path: a.path, + href: getFavoriteHref(a.path, 'agent'), + kind: 'agent' as const + })), ...assets.map((a) => ({ label: getFavoriteLabel(a.path, 'asset'), path: a.path, diff --git a/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte index ceb37c0a10..af96078f5a 100644 --- a/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte @@ -17,6 +17,7 @@ import { Alert } from '$lib/components/common' import { keepsManagedMemory } from '$lib/components/flows/agentFormFields' import { agentMenuItems, deleteAgent } from '$lib/components/flows/agentActions' + import AgentHistory from '$lib/components/flows/AgentHistory.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' @@ -40,6 +41,7 @@ let configModal: AgentConfigModal | undefined = $state(undefined) let shareModal: ShareModal | undefined = $state(undefined) let deploymentDrawer: DeployWorkspaceDrawer | undefined = $state(undefined) + let history: AgentHistory | undefined = $state(undefined) let deleteOpen = $state(false) let evalsModal: AgentEvalsModal | undefined = $state(undefined) let runForm: RunForm | undefined = $state(undefined) @@ -77,6 +79,7 @@ deployUiSettings: deployUiSettings.current, onPermissions: () => shareModal?.openDrawer?.(path, 'resource'), onDeploy: () => deploymentDrawer?.openDrawer(path, 'resource'), + onHistory: () => history?.open(), onDelete: () => (deleteOpen = true) }).map((item) => ({ label: item.displayName, @@ -91,6 +94,7 @@ + reloaded++} /> usersWorkspaceStore.set(w)), + loadCopilot($workspaceStore!) + ]) + } + // The creator of a fork gets the fork members screen even when they are not an admin of it: // their `usr` row is copied from the parent, so forking as an ordinary developer leaves them // unable to bring anyone in to collaborate. Nothing else on this page opens up — the backend @@ -676,12 +722,8 @@ slack_team_name = settings.slack_name teams_team_id = settings.teams_team_id teams_team_name = settings.teams_team_name - if (settings.slack_command_script) { - itemKind = settings.slack_command_script.split('/')[0] as 'flow' | 'script' - } - if (settings.teams_command_script) { - itemKind = settings.teams_command_script.split('/')[0] as 'flow' | 'script' - } + const commandScript = settings.teams_command_script || settings.slack_command_script + itemKind = commandScript ? (commandScript.split('/')[0] as 'flow' | 'script') : 'flow' slackScriptPath = (settings.slack_command_script ?? '').split('/').slice(1).join('/') teamsScriptPath = (settings.teams_command_script ?? '').split('/').slice(1).join('/') slackInitialPath = slackScriptPath @@ -764,6 +806,8 @@ triggers: (settings.deploy_ui.include_type?.indexOf('trigger') ?? -1) >= 0 } } + } else { + deployUiSettings = defaultDeployUiSettings() } // Store initial deploy settings state for unsaved changes detection @@ -867,7 +911,7 @@ } } - let deployUiSettings: { + type DeployUiSettings = { include_path: string[] include_type: { scripts: boolean @@ -878,18 +922,22 @@ secrets: boolean triggers: boolean } - } = $state({ - include_path: [], - include_type: { - scripts: true, - flows: true, - apps: true, - resources: true, - variables: true, - secrets: true, - triggers: true + } + function defaultDeployUiSettings(): DeployUiSettings { + return { + include_path: [], + include_type: { + scripts: true, + flows: true, + apps: true, + resources: true, + variables: true, + secrets: true, + triggers: true + } } - }) + } + let deployUiSettings: DeployUiSettings = $state(defaultDeployUiSettings()) $effect(() => { // `canAdmin` is read as a dependency, not inside untrack: $userStore is repopulated @@ -1187,6 +1235,9 @@ // Combined function to check for unsaved changes across all tabs function getAllUnsavedChanges() { + if (showYaml) { + return yamlEditor?.unsavedChanges() ?? { savedValue: {}, modifiedValue: {} } + } switch (tab) { case 'windmill_data_tables': return dataTableSettingsComponent?.unsavedChanges() ?? { savedValue: {}, modifiedValue: {} } @@ -1235,6 +1286,10 @@ // Combined function to discard changes based on current tab function discardAllChanges() { + if (showYaml) { + yamlEditor?.discard() + return + } switch (tab) { case 'ai': aiSettingsComponent?.discard() @@ -1482,7 +1537,11 @@ two controls that sat beside the title are the page's actions. The panes below scroll inside themselves, so the bar keeps its edge at rest rather than only under a scroll. --> @@ -1492,17 +1551,19 @@ -
- { - const params = new URLSearchParams($page.url.searchParams) - params.set('tab', id) - goto(`?${params.toString()}`) - }} - /> -
+ {#if !showYaml} +
+ { + const params = new URLSearchParams($page.url.searchParams) + params.set('tab', id) + goto(`?${params.toString()}`) + }} + /> +
+ {/if}
@@ -1510,6 +1571,8 @@
{#if !loadedSettings} + {:else if showYaml} + {:else if tab == 'users'} {#if canAdmin} @@ -2439,7 +2502,28 @@ export async function main( tabMode={true} /> + { + yamlConfirm = undefined + }} +> + + {yamlConfirm === 'enter' + ? 'Switching to YAML mode discards the changes you made in this pane.' + : 'Leaving YAML mode discards the changes you made in the editor.'} + + + {#snippet settingsActions()} + {#if canAdmin} + {#key yamlToggleKey} + yamlMode, setYamlMode} options={{ right: 'YAML' }} size="sm" /> + {/key} + {/if} {#if $superadmin}