diff --git a/.claude/settings.json b/.claude/settings.json index a464ca3719..37aaeeb0f5 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -55,22 +55,18 @@ "mcp__claude_ai_Gmail__list_drafts" ], "deny": [ - "Read(.env)", - "Read(.env.*)", - "Read(**/.env)", - "Read(**/.env.*)", - "Read(**/secrets/**)", - "Read(**/*.pem)", - "Read(**/*.key)", - "Read(**/credentials.json)", - "Read(**/.secret*)", - "Read(**/.secrets*)", - "Read(**/*.secret)", - "Read(**/*.secrets)", "Edit(.env)", "Edit(.env.*)", "Edit(**/.env)", - "Edit(**/.env.*)" + "Edit(**/.env.*)", + "Edit(**/secrets/**)", + "Edit(**/*.pem)", + "Edit(**/*.key)", + "Edit(**/credentials.json)", + "Edit(**/.secret*)", + "Edit(**/.secrets*)", + "Edit(**/*.secret)", + "Edit(**/*.secrets)" ], "ask": [ "Bash(rmdir:*)", diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index e2e7318ab6..f9153c0ce0 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -101,7 +101,6 @@ jobs: - name: Install OpenSSL via vcpkg run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cdd0e2a212..9f65c2ae47 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -61,7 +61,7 @@ jobs: bun-version: 1.4.0 - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "24" - uses: astral-sh/setup-uv@v6.2.1 with: version: "0.11.24" diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 84f4366235..46befe42b6 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -62,7 +62,6 @@ jobs: - name: Cargo build binary windows timeout-minutes: 180 run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index ee732569dd..bd15ec8876 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -9,6 +9,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -22,6 +23,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -59,7 +61,7 @@ jobs: echo "$CHANGED_FILES" # Direct git sync file changes — always relevant. - if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|frontend/src/lib/hubPaths\.json|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Relevant: direct git sync file changes" exit 0 diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index d159619fba..b94d5d57c7 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -51,7 +51,6 @@ jobs: - name: Cargo build windows timeout-minutes: 180 run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index e908485536..249f8350ae 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.800.0" + ".": "1.803.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index e27f2166a3..33e24c0c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,68 @@ # Changelog +## [1.803.0](https://github.com/windmill-labs/windmill/compare/v1.802.0...v1.803.0) (2026-09-03) + + +### Features + +* expose request headers to scripts invoked via MCP ([#10903](https://github.com/windmill-labs/windmill/issues/10903)) ([e474e88](https://github.com/windmill-labs/windmill/commit/e474e8803ce2ff5c2df09a58dab51d45f5c922ca)) +* reuse an existing workspace resource in the project import wizard ([#10935](https://github.com/windmill-labs/windmill/issues/10935)) ([582761e](https://github.com/windmill-labs/windmill/commit/582761e37c776e92dc1c6ebfee8c4efe7c35d822)) + + +### Bug Fixes + +* bump git sync hub scripts to cli 1.802.1, test the fork ui pull ([#10955](https://github.com/windmill-labs/windmill/issues/10955)) ([ca88009](https://github.com/windmill-labs/windmill/commit/ca8800959aa6a0017cc29bad187c9f49e0d13cc4)) +* **cli:** make a sync push into a fork converge on schedules and inline names ([#10951](https://github.com/windmill-labs/windmill/issues/10951)) ([0f5a1db](https://github.com/windmill-labs/windmill/commit/0f5a1db2abba8df30a2f975f4498e269f13cf93d)) +* fade the home Build with AI placeholder every 10s instead of typing it ([#10953](https://github.com/windmill-labs/windmill/issues/10953)) ([3d089b5](https://github.com/windmill-labs/windmill/commit/3d089b57344f5814086e6176301c5031dc519674)) +* let operators use wmill.datatable() from within running jobs ([#10931](https://github.com/windmill-labs/windmill/issues/10931)) ([9b64a89](https://github.com/windmill-labs/windmill/commit/9b64a89cd46ae718d6c58fa12f925fa041fb1032)) + +## [1.802.0](https://github.com/windmill-labs/windmill/compare/v1.801.0...v1.802.0) (2026-09-02) + + +### Features + +* add retention cleanup for the otel_traces table ([#10949](https://github.com/windmill-labs/windmill/issues/10949)) ([d472193](https://github.com/windmill-labs/windmill/commit/d472193e5bf5f6428e0096a402eb2c9299634fb2)) +* open path links from chat messages in the session preview panel ([#10881](https://github.com/windmill-labs/windmill/issues/10881)) ([f10ac6c](https://github.com/windmill-labs/windmill/commit/f10ac6c2b3644fb16697e650efbc4f7cd3c6944c)) +* restore owner and label filter chips on the homepage ([#10942](https://github.com/windmill-labs/windmill/issues/10942)) ([ccf8476](https://github.com/windmill-labs/windmill/commit/ccf84761dd664b9228dfe2f65867e8c32cd20c21)) +* **sessions:** offer the item you came from when starting a new session ([#10940](https://github.com/windmill-labs/windmill/issues/10940)) ([d3747d6](https://github.com/windmill-labs/windmill/commit/d3747d62555ebcb09c78cfabcaa3b6177758d6ea)) +* workspace setting to hide the AI assistant, agent steps unaffected ([#10941](https://github.com/windmill-labs/windmill/issues/10941)) ([fdd3b36](https://github.com/windmill-labs/windmill/commit/fdd3b36423344a2e1a464674179406581074e926)) + + +### Bug Fixes + +* apply object-storage test SSRF validation to all non-super-admins ([#10933](https://github.com/windmill-labs/windmill/issues/10933)) ([4fef119](https://github.com/windmill-labs/windmill/commit/4fef1195adaa9fa036a219884bd6c996460ca37f)) +* connect to dev server instead of localhost ([#10912](https://github.com/windmill-labs/windmill/issues/10912)) ([337154b](https://github.com/windmill-labs/windmill/commit/337154b8304a5969f35216add627b5c1153c0f6c)) +* preselect first row of AI agent and AI sandbox insert panes ([#10937](https://github.com/windmill-labs/windmill/issues/10937)) ([95b6bbd](https://github.com/windmill-labs/windmill/commit/95b6bbd46ada11d96a914ae5b0e92aba4dd02530)) +* record supplied script lock hashes so importers can skip relocking ([#10915](https://github.com/windmill-labs/windmill/issues/10915)) ([17ba521](https://github.com/windmill-labs/windmill/commit/17ba521c352aec65a8270893752bbadd7f3d6eaa)) +* sandbox script-controlled content types in result_to_response ([#10932](https://github.com/windmill-labs/windmill/issues/10932)) ([419741e](https://github.com/windmill-labs/windmill/commit/419741e5d226c67c51429094fb6ded9474afed99)) + +## [1.801.0](https://github.com/windmill-labs/windmill/compare/v1.800.1...v1.801.0) (2026-09-01) + + +### Features + +* **ai-chat:** make reusable skills ai_skill resources you select per workspace ([#10914](https://github.com/windmill-labs/windmill/issues/10914)) ([cfcfe29](https://github.com/windmill-labs/windmill/commit/cfcfe298dd9ab50196bd64926ef78c4563f58c2c)) +* **ai-sessions:** show a running session across tabs and reload finished turns ([#10916](https://github.com/windmill-labs/windmill/issues/10916)) ([816dc9d](https://github.com/windmill-labs/windmill/commit/816dc9dcd2c310e499d2d210a0abcd403469f29c)) +* edit folders and groups in a drawer that saves once ([#10873](https://github.com/windmill-labs/windmill/issues/10873)) ([5d5ad4e](https://github.com/windmill-labs/windmill/commit/5d5ad4e8974e076ef53a26a5584e4209255a2248)) +* make the home Build with AI composer dismissible, quiet the rest of the home page ([#10930](https://github.com/windmill-labs/windmill/issues/10930)) ([772fafe](https://github.com/windmill-labs/windmill/commit/772fafec8316a1e0c0e76b9a0737cc41d40a9a8c)) + + +### Bug Fixes + +* let a principal without a login account own a draft ([#10925](https://github.com/windmill-labs/windmill/issues/10925)) ([94af8d0](https://github.com/windmill-labs/windmill/commit/94af8d0fb5aceebe83936fd6761c6c1c02c75323)) +* resolve chat path links against the session's operating workspace ([#10924](https://github.com/windmill-labs/windmill/issues/10924)) ([9074de2](https://github.com/windmill-labs/windmill/commit/9074de25ea730ca02653c9a2e2b8b99eda6f3137)) +* tolerate string app_id in GHES app config deserialization ([#10923](https://github.com/windmill-labs/windmill/issues/10923)) ([af8ff38](https://github.com/windmill-labs/windmill/commit/af8ff3868748412cb658c803ebc8a71edc3cd8fb)) + +## [1.800.1](https://github.com/windmill-labs/windmill/compare/v1.800.0...v1.800.1) (2026-09-01) + + +### Bug Fixes + +* add top margin to the home Build with AI section ([#10909](https://github.com/windmill-labs/windmill/issues/10909)) ([bedf5ae](https://github.com/windmill-labs/windmill/commit/bedf5ae57445025729e94e16f1b5f13f6ff38ffa)) +* **ai-chat:** consume an @ mention with the message that carried it ([#10907](https://github.com/windmill-labs/windmill/issues/10907)) ([c512110](https://github.com/windmill-labs/windmill/commit/c512110a1f8d0d3437c20048f6446ef62b10222c)) +* keep a local dbt descriptor under sync pull --keep-deleted ([#10911](https://github.com/windmill-labs/windmill/issues/10911)) ([4b5be38](https://github.com/windmill-labs/windmill/commit/4b5be386ce0f851a087f43c0b0ac6e4b1b055a47)) +* keep windmill-indexer out of builds without tantivy ([#10908](https://github.com/windmill-labs/windmill/issues/10908)) ([db0f004](https://github.com/windmill-labs/windmill/commit/db0f004613e3f90428fea4c824cc53f1b2fc03b0)) + ## [1.800.0](https://github.com/windmill-labs/windmill/compare/v1.799.0...v1.800.0) (2026-08-31) diff --git a/CONTEXT.md b/CONTEXT.md index 6efb92b669..64aa1b93d8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -36,3 +36,22 @@ _Avoid_: argument field, param **Expression input**: Any other place a property can be picked into: the loop iterator, skip and early-stop predicates, the retry condition, a branch predicate, timeout. Its prop picker opens in a popover from the connect button rather than taking a pane. _Avoid_: JS field, code input + +### Permissions + +**Member**: +A user or group granted a role on a folder, a group, or an item's extra ACL. The list of them is +"Members (n)" everywhere it is shown, and one is added with "Add member". +_Avoid_: participant, collaborator, owner, ACL entry, permission (that names the concept, not the people) + +**Role**: +The access level a member holds: viewer, writer or admin on a folder; member or admin on a group. +Viewers read, writers also edit, admins also manage the members. A group role of **manager** — +manages the group without belonging to it — is a legacy state the UI shows and can leave, but +offers no way to enter. +_Avoid_: permission level, access level, rank + +**Owner**: +Reserved for the path prefix that says where an item lives — `u/alice` or `f/team`. A folder's +`owners` column in the database is its admin members; call those admins, never owners, in the UI. +_Avoid_: using "owner" for a folder admin diff --git a/Dockerfile b/Dockerfile index 8ecee96725..6a290c3553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,9 +141,9 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 ARG KUBECTL_VERSION=1.36.2 -ARG HELM_VERSION=3.21.2 +ARG HELM_VERSION=3.21.4 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte -ARG GO_VERSION=1.26.0 +ARG GO_VERSION=1.26.8 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true @@ -250,8 +250,8 @@ RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_r RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ +RUN curl -sL https://deb.nodesource.com/setup_24.x | bash - +RUN apt-get -y update && apt-get install -y --no-install-recommends curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* # go build is slower the first time it is ran, so we prewarm it in the build @@ -299,7 +299,7 @@ RUN bun install -g windmill-cli \ RUN curl -fsSL https://claude.ai/install.sh | bash \ && cp /root/.local/share/claude/versions/* /usr/bin/claude -COPY --from=php:8.3.30-cli-trixie /usr/local/bin/php /usr/bin/php +COPY --from=php:8.3.33-cli-trixie /usr/local/bin/php /usr/bin/php COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 5c682be5eb..6d80f19197 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -7,6 +7,7 @@ import type { ListableApp, ListableResource, ListableVariable, + Resource, Script } from '../../../frontend/src/lib/gen' import type { @@ -81,6 +82,15 @@ export interface BenchmarkWorkspaceAiProvider { isDefault?: boolean } +/** A plain (non-AI) resource of the benchmark workspace, for cases about referencing a + * credential — passing one as a run argument, say. `value` is what `get_resource` returns. */ +export interface BenchmarkWorkspaceResource { + path: string + resource_type: string + value?: Record + description?: string +} + export interface BenchmarkWorkspaceJob { /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ id?: string @@ -98,6 +108,7 @@ export interface BenchmarkWorkspaceRunnables { apps?: BenchmarkWorkspaceApp[] variables?: BenchmarkWorkspaceVariable[] aiProviders?: BenchmarkWorkspaceAiProvider[] + resources?: BenchmarkWorkspaceResource[] datatables?: BenchmarkDatatableSeed[] jobs?: BenchmarkWorkspaceJob[] } @@ -284,15 +295,71 @@ export function listBenchmarkAiProviderResources(workspace: string): ListableRes })) } -/** The value of a seeded AI provider resource. Only the endpoint fields are modelled — a key is - * never needed, because no eval run calls the provider through this resource. */ +/** Plain seeded resources of a benchmark workspace, shaped like `ResourceService.listResource` + * rows. Null when the workspace is not a benchmark one. */ +export function listBenchmarkPlainResources(workspace: string): ListableResource[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + return null + } + return (runnables.resources ?? []).map((seed) => ({ + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: null, + is_oauth: false, + is_linked: false, + is_refreshed: false, + extra_perms: {}, + edited_at: BENCHMARK_TIMESTAMP + })) +} + +/** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both + * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI + * providers too, and a case that lists resources and then reads one by path would otherwise get + * a row it cannot fetch. */ +export function getBenchmarkResource(workspace: string, path: string): Resource | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const seed = runnables?.resources?.find((entry) => entry.path === path) + if (seed) { + return { + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: seed.value ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource + } + const provider = runnables?.aiProviders?.find((entry) => entry.path === path) + if (!provider) { + return null + } + return { + workspace_id: workspace, + path: provider.path, + resource_type: provider.kind, + value: getBenchmarkResourceValue(workspace, path) ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource +} + +/** The value of a seeded resource. For an AI provider only the endpoint fields are modelled — a + * key is never needed, because no eval run calls the provider through this resource. */ export function getBenchmarkResourceValue( workspace: string, path: string ): Record | null { - const seed = benchmarkWorkspaceRunnables - .get(workspace) - ?.aiProviders?.find((entry) => entry.path === path) + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const plain = runnables?.resources?.find((entry) => entry.path === path) + if (plain) { + return plain.value ?? {} + } + const seed = runnables?.aiProviders?.find((entry) => entry.path === path) if (!seed) { return null } diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index e59ec11ad9..338ed8504c 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -70,7 +70,9 @@ vi.mock('$lib/gen', async () => { getBenchmarkResourceValue, getBenchmarkVariableByPath, hasBenchmarkWorkspace, + getBenchmarkResource, listBenchmarkAiProviderResources, + listBenchmarkPlainResources, listBenchmarkApps, listBenchmarkDatatables, listBenchmarkDrafts, @@ -359,18 +361,24 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkResourceValue(data.workspace, data.path)) : actual.ResourceService.existsResource(data), - // Only AI provider resources are modelled: they are what an AI agent step references. listResource: async (data: { workspace: string; resourceType?: string }) => { if (!hasBenchmarkWorkspace(data.workspace)) { return actual.ResourceService.listResource(data) } - const seeded = listBenchmarkAiProviderResources(data.workspace) ?? [] + const seeded = [ + ...(listBenchmarkAiProviderResources(data.workspace) ?? []), + ...(listBenchmarkPlainResources(data.workspace) ?? []) + ] const wanted = data.resourceType?.split(',') return wanted ? seeded.filter((r) => wanted.includes(r.resource_type)) : seeded }, getResource: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + const resource = getBenchmarkResource(data.workspace, data.path) + if (!resource) { + throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + } + return resource } return actual.ResourceService.getResource(data) }, diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 21f80bef7e..94ecb5c029 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -2365,3 +2365,37 @@ - the step uses the workspace's anthropic resource f/evals/global/anthropic_main - the model is the Opus one the user asked for, taken from the models that resource serves - the diff flow input reaches the agent + +# The failure this pins: passing a resource as `{"$res": ""}` (or as a bare path), which +# reaches the script unresolved because the backend only substitutes a string value that itself +# starts with `$res:`. The mock preview echoes args back and reports success, so nothing in the +# loop corrects a wrong shape — the arg form is the whole test. +- id: global-run-arg-resource-reference + prompt: |- + Run `f/evals/global/github_repo_stats` against the `windmill-labs/windmill` repo, passing our + GitHub credentials at `f/evals/global/github_main` as its `gh_auth` input, and tell me whether + it went through. + initial: ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - test_run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + toolCallArgs: + # Exact: the mock never resolves the reference, so a near-miss path like + # `$res:f/evals/global/github_main_backup` would otherwise pass. + - tool: test_run_script + field: args.gh_auth + stringEqualsAnyOf: + - "$res:f/evals/global/github_main" + # The judge only sees drafts, and this case makes none — the deliverable is the shape of the + # run argument, checked deterministically above. + skipJudge: true + judgeChecklist: + - runs the existing script rather than rewriting it + - passes the GitHub resource as the bare string $res:f/evals/global/github_main diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index c9a4f7830e..4107dc53da 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -160,6 +160,13 @@ export interface ToolCallArgumentRule { field: string; stringStartsWithAnyOf?: string[]; stringMustNotStartWithAnyOf?: string[]; + /** + * Universal over calls: every recorded call to `tool` must carry `field` as + * exactly one of these strings. Use when a near-miss would still satisfy a + * prefix — a resource reference like `$res:f/a/b` shares its prefix with the + * wrong `$res:f/a/b_backup`, and the mock never resolves it to catch that. + */ + stringEqualsAnyOf?: string[]; /** * Case-insensitive "contains", existential over calls: at least one recorded * call to `tool` must have `field` containing one of these substrings. Other diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 9b606c1c85..4b7891f67e 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -228,6 +228,43 @@ describe("validateToolExpectations", () => { }); }); + // A resource reference shares its prefix with a wrong sibling path, and the mock + // never resolves it, so only exact matching separates the two. + it("rejects a resource reference whose path merely shares the prefix", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["test_run_script"], + toolCallDetails: [ + { + name: "test_run_script", + arguments: { args: { gh_auth: "$res:f/evals/global/github_main_backup" } }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [ + { + tool: "test_run_script", + field: "args.gh_auth", + stringEqualsAnyOf: ["$res:f/evals/global/github_main"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "test_run_script.args.gh_auth matches an accepted value", + passed: false, + details: + 'accepted values: $res:f/evals/global/github_main; values: "$res:f/evals/global/github_main_backup"', + }); + }); + // The whole point of the same-call rule: the per-field rules are existential over // calls, so two single-filter pages would satisfy them while never opening the // combined view the case asks for. diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 7150dab0c3..132a9ff3d0 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -278,6 +278,20 @@ export function validateToolExpectations(input: { ); } + if (rule.stringEqualsAnyOf && rule.stringEqualsAnyOf.length > 0) { + const invalidValues = values.filter( + (value) => + typeof value !== "string" || !rule.stringEqualsAnyOf!.includes(value) + ); + checks.push( + check( + `${rule.tool}.${rule.field} matches an accepted value`, + invalidValues.length === 0, + `accepted values: ${rule.stringEqualsAnyOf.join(", ")}; values: ${summarizeToolValues(values)}` + ) + ); + } + if (rule.stringMustNotStartWithAnyOf && rule.stringMustNotStartWithAnyOf.length > 0) { const invalidValues = values.filter( (value) => diff --git a/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json new file mode 100644 index 0000000000..80e1b960c5 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "resources": [ + { + "path": "f/evals/global/github_main", + "resource_type": "github", + "description": "GitHub credentials", + "value": { "token": "$var:f/evals/global/github_token" } + } + ], + "scripts": [ + { + "path": "f/evals/global/github_repo_stats", + "summary": "Count open issues on a GitHub repository", + "description": "Reads the open issue count for a repository using GitHub credentials.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "gh_auth": { + "type": "object", + "format": "resource-github", + "description": "GitHub credentials" + }, + "repo": { + "type": "string", + "description": "Repository in owner/name form" + } + }, + "required": ["gh_auth", "repo"] + }, + "content": "type Github = { token: string }\n\nexport async function main(gh_auth: Github, repo: string) {\n const res = await fetch(`https://api.github.com/repos/${repo}/issues?state=open`, {\n headers: { Authorization: `Bearer ${gh_auth.token}` }\n })\n const issues = await res.json()\n return { repo, open_issues: issues.length }\n}\n" + } + ] + } +} diff --git a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json b/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json deleted file mode 100644 index abae0ba456..0000000000 --- a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COUNT(*)::bigint AS \"total!\",\n COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS \"replacing!\"\n FROM ai_skill\n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "total!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "replacing!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7" -} diff --git a/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json new file mode 100644 index 0000000000..304efc22d2 --- /dev/null +++ b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n LEFT JOIN password p\n ON p.email = d.email\n AND p.super_admin = true\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)\n ORDER BY d.email NULLS LAST", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username?", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "draft_saved_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f" +} diff --git a/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json new file mode 100644 index 0000000000..8275796e90 --- /dev/null +++ b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88" +} diff --git a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json similarity index 50% rename from backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json rename to backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json index 29ce0c6f2c..03f11ef137 100644 --- a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json +++ b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", + "query": "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "name", - "type_info": "Varchar" + "name": "lockfile_hash", + "type_info": "Int8" } ], "parameters": { @@ -19,5 +19,5 @@ false ] }, - "hash": "e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61" + "hash": "1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a" } diff --git a/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json new file mode 100644 index 0000000000..693d84173a --- /dev/null +++ b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trace_id FROM otel_traces ORDER BY trace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trace_id", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3" +} diff --git a/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json new file mode 100644 index 0000000000..73d5000f2d --- /dev/null +++ b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.path = dest.path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef" +} diff --git a/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json new file mode 100644 index 0000000000..f35dbc23e4 --- /dev/null +++ b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM otel_traces WHERE ctid IN (\n SELECT ctid FROM otel_traces\n WHERE start_time_unix_nano < EXTRACT(\n EPOCH FROM now() - ($1::bigint::text || ' s')::interval\n )::bigint * 1000000000\n LIMIT $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e" +} diff --git a/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json new file mode 100644 index 0000000000..f7c39c21fc --- /dev/null +++ b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET email = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8" +} diff --git a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json b/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json deleted file mode 100644 index 8d86ff3db6..0000000000 --- a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Text", - "Varchar", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57" -} diff --git a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json b/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json deleted file mode 100644 index 63283a830a..0000000000 --- a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at)\n VALUES ('admins', $1, $2, $3, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb" -} diff --git a/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json new file mode 100644 index 0000000000..681206a381 --- /dev/null +++ b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668" +} diff --git a/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json new file mode 100644 index 0000000000..67b672228f --- /dev/null +++ b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano)\n VALUES ($1, $2, 'GET /', 3, $3, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc" +} diff --git a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json b/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json deleted file mode 100644 index d55722ed24..0000000000 --- a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by)\n VALUES ($1, $2, $3, $4, now(), $5)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET description = EXCLUDED.description,\n instructions = EXCLUDED.instructions,\n edited_at = now(),\n edited_by = EXCLUDED.edited_by", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8" -} diff --git a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json b/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json similarity index 63% rename from backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json rename to backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json index d453eb430d..3b9a2e2f34 100644 --- a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json +++ b/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", "describe": { "columns": [ { @@ -13,12 +13,14 @@ "Left": [ "Text", "Jsonb", - "Text" + "Text", + "Text", + "Bool" ] }, "nullable": [ null ] }, - "hash": "1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae" + "hash": "8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a" } diff --git a/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json new file mode 100644 index 0000000000..610030cffc --- /dev/null +++ b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, lock FROM script\n WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "lock", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455" +} diff --git a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json b/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json deleted file mode 100644 index 733478e0e9..0000000000 --- a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7" -} diff --git a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json new file mode 100644 index 0000000000..3624a32893 --- /dev/null +++ b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at)\n VALUES ('admins', $1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9" +} diff --git a/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json new file mode 100644 index 0000000000..c7db67f4d6 --- /dev/null +++ b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e" +} diff --git a/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json new file mode 100644 index 0000000000..0a0f8b4732 --- /dev/null +++ b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + null + ] + }, + "hash": "9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032" +} diff --git a/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json new file mode 100644 index 0000000000..f4c4fd31de --- /dev/null +++ b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text", + "Varchar", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4" +} diff --git a/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json new file mode 100644 index 0000000000..c3ef22f973 --- /dev/null +++ b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db" +} diff --git a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json b/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json deleted file mode 100644 index ca39442f59..0000000000 --- a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "description", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b" -} diff --git a/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json new file mode 100644 index 0000000000..19dc4781a6 --- /dev/null +++ b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n SELECT $1, * FROM UNNEST($2::text[], $3::bigint[])\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash\n WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a" +} diff --git a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json similarity index 72% rename from backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json rename to backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json index b31e532fc2..2947dd8413 100644 --- a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json +++ b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4" + "hash": "cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f" } diff --git a/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json new file mode 100644 index 0000000000..eba1b0da99 --- /dev/null +++ b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a" +} diff --git a/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json new file mode 100644 index 0000000000..6ffd197e90 --- /dev/null +++ b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_fileset, format_extension FROM resource_type\n WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fileset", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "format_extension", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a" +} diff --git a/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json new file mode 100644 index 0000000000..fc42c451f8 --- /dev/null +++ b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a" +} diff --git a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json b/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json deleted file mode 100644 index 121d7fe04a..0000000000 --- a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "instructions", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d" -} diff --git a/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json new file mode 100644 index 0000000000..bea6be2a7f --- /dev/null +++ b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, lock_error_logs, created_at)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, $6::text, clock_timestamp()\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637" +} diff --git a/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json new file mode 100644 index 0000000000..24f957a4e2 --- /dev/null +++ b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM draft ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2067bfd789..d314168b35 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -981,9 +981,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -5177,9 +5177,9 @@ dependencies = [ [[package]] name = "frostem" -version = "1.20260821.3" +version = "1.20260821.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2e81eab151ba68484704bb3d21b4b4d2747314d3081fa86fa6c7300a4c42e2" +checksum = "481ace7f781f5ae54a5c0a6d6d8edb30adba737cfa1230fbd5632d63ba8dfd80" [[package]] name = "fs3" @@ -7262,9 +7262,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ "bitflags 2.13.1", "libc", @@ -7387,9 +7387,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.3" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] @@ -7776,9 +7776,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -7870,10 +7870,11 @@ dependencies = [ [[package]] name = "mysql_async" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2" +checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4" dependencies = [ + "arc-swap", "bytes", "crossbeam-queue", "crossbeam-utils", @@ -7882,7 +7883,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.3", + "lru 0.18.4", "mysql_common", "native-tls", "pem 3.0.6", @@ -10523,9 +10524,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.1.4" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d45508e9b4ba024fe996c2638799635d75b6dd0ba8f32ccf08f8026f0c780" +checksum = "cdf1c49bd4d52014b94db0877410db273c2008f01628b0252a2e9460ad9b7fda" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -11753,9 +11754,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" dependencies = [ "serde", ] @@ -12812,7 +12813,7 @@ dependencies = [ "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.18.3", + "lru 0.18.4", "lz4_flex 0.14.0", "measure_time", "memmap2", @@ -14746,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -14831,7 +14832,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.800.0" +version = "1.803.0" dependencies = [ "async-stream", "async-trait", @@ -14864,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14877,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "argon2", @@ -15017,7 +15018,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15040,7 +15041,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15057,7 +15058,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15083,7 +15084,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.800.0" +version = "1.803.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15093,7 +15094,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15110,7 +15111,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15132,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15155,7 +15156,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15171,7 +15172,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15193,7 +15194,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15214,7 +15215,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15228,7 +15229,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -15263,7 +15264,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15288,7 +15289,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15316,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15338,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15358,7 +15359,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15396,7 +15397,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15424,7 +15425,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.800.0" +version = "1.803.0" dependencies = [ "lazy_static", "serde", @@ -15436,7 +15437,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.800.0" +version = "1.803.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15460,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15474,7 +15475,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.800.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15509,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.800.0" +version = "1.803.0" dependencies = [ "chrono", "lazy_static", @@ -15523,7 +15524,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15542,7 +15543,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.800.0" +version = "1.803.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15646,9 +15647,10 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.800.0" +version = "1.803.0" dependencies = [ "chrono", + "futures", "itertools 0.14.0", "lazy_static", "serde", @@ -15665,7 +15667,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.800.0" +version = "1.803.0" dependencies = [ "regex", "serde", @@ -15680,7 +15682,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15707,7 +15709,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "futures", @@ -15724,7 +15726,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.800.0" +version = "1.803.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15740,7 +15742,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -15761,7 +15763,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -15792,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "arc-swap", @@ -15817,7 +15819,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-stream", @@ -15851,7 +15853,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "futures", @@ -15869,7 +15871,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.800.0" +version = "1.803.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15878,7 +15880,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15890,7 +15892,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15902,7 +15904,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "gosyn", @@ -15914,7 +15916,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15926,7 +15928,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15938,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "nu-parser", @@ -15949,7 +15951,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15960,7 +15962,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15972,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15983,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16005,7 +16007,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -16017,7 +16019,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16031,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16048,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16061,7 +16063,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde", @@ -16073,7 +16075,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16091,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16107,7 +16109,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16123,7 +16125,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16137,7 +16139,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16176,7 +16178,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "const_format", @@ -16216,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.800.0" +version = "1.803.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16227,7 +16229,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16262,7 +16264,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16286,7 +16288,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16319,7 +16321,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16346,7 +16348,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16379,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16399,7 +16401,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16433,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16469,7 +16471,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16492,7 +16494,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16516,7 +16518,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -16540,7 +16542,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16575,7 +16577,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16603,7 +16605,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16628,7 +16630,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16647,7 +16649,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-once-cell", @@ -16764,7 +16766,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.800.0" +version = "1.803.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 966f820d57..f432f022c5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.800.0" +version = "1.803.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.800.0" +version = "1.803.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -115,7 +115,7 @@ strip = "none" [features] default = [] -private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] +private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer?/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"] local_reports = ["windmill-common/local_reports"] @@ -131,6 +131,10 @@ quickjs = ["windmill-worker/quickjs", "windmill-api/quickjs"] openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect", "windmill-object-store/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] +# `tantivy` must stay the only feature that enables windmill-indexer: it is the only one that +# also gives it `enterprise` + `parquet`, and its EE sources gate nearly everything on that +# pair, so a bare windmill-indexer is a crate of dead code that `-D warnings` rejects. Any +# other feature wanting one of its features has to use the optional `windmill-indexer?/` form. tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:windmill-runtime-nativets", "windmill-test-utils/deno_core"] diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index 0468ebf721..f5725d65a3 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -85,7 +85,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream | | EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation | | EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts | -| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | +| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers, script-controlled `wm_content_type`/`wm_headers` on `run_wait_result` and sync HTTP-route responses | stored user content → admin browser (same origin) | Admin session, account takeover | | EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | | EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | | EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS` now defaults to `true`; can still be overridden to `false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | @@ -106,7 +106,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | | T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | -| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | +| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, S3 download content-type, or a script-chosen `text/html` content type on `run_wait_result` / sync HTTP-route responses (GET-reachable with the `SameSite=Lax` session cookie) | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads and on every `result_to_response` composite result (inserted after `wm_headers`; hop-by-hop names such as `Connection` rejected so a proxy cannot strip them) | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0, WIN-2471 | | T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | | T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 | | T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 623450b3b2..9eb73313a3 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0fa1af247ea6534eafd5f188e00d9ca674b605e5 +6038847600dbc00993848e44e45cd5bcff50ac4c diff --git a/backend/migrations/20260831120907_skills_resource_type.down.sql b/backend/migrations/20260831120907_skills_resource_type.down.sql new file mode 100644 index 0000000000..5cc32da37d --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.down.sql @@ -0,0 +1,13 @@ +-- The up migration only ever added: `ai_skill` still holds every skill it copied, +-- so there is nothing to restore and nothing to delete. Removing the resources +-- would destroy any a user has since edited or created, and removing a folder +-- would take whatever else was put in it. +-- +-- The seeded resource type goes. `created_by` only distinguishes this migration's +-- row from one a user created by hand: a hub sync updates the schema in place and +-- leaves `created_by` alone, so a synced-over row is still removed here and the +-- next sync puts it back. +DELETE FROM resource_type +WHERE workspace_id = 'admins' + AND name = 'ai_skill' + AND created_by = 'system'; diff --git a/backend/migrations/20260831120907_skills_resource_type.up.sql b/backend/migrations/20260831120907_skills_resource_type.up.sql new file mode 100644 index 0000000000..4e8f9cfa95 --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.up.sql @@ -0,0 +1,88 @@ +-- AI chat skills move from the `ai_skill` table onto ordinary resources, so they +-- gain folder ACLs, version history, workspace export and git-sync. An `ai_skill` +-- resource holds the SKILL.md body in `value.content`; its description lives in +-- the resource's own `description` column and its name is the path basename. +-- +-- Nothing here is destructive. `ai_skill` is left in place, unread, for a later +-- release to drop once operators have confirmed the copy. That is what lets every +-- step below skip on conflict rather than resolve one: a skipped row is still in +-- the table, so it is not lost, and the migration needs no record of what it did +-- in order to be reversible. + +-- `format_extension` makes the resource editor render `value.content` as a plain +-- .md file. Seeded under 'admins' so every workspace sees it. +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, format_extension, edited_at) +VALUES ( + 'admins', + 'ai_skill', + '{"type": "object", "properties": {"content": {"type": "string"}}}', + 'A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.', + 'system', + 'md', + now() +) +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Shared home matching the admin-only upload these skills had. A workspace that +-- already has a `skills` folder keeps it untouched, ACL and all: adopting one +-- would hand its own grants — possibly write for everyone — over a set of +-- instructions the assistant follows. +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) +SELECT DISTINCT workspace_id, 'skills', 'Skills', ARRAY[]::TEXT[], '{"g/all": false}'::jsonb +FROM ai_skill +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Copied only where the destination is free and the folder matches the one above +-- exactly, owners included: a pre-existing folder carrying the same ACL but an +-- owner would hand that owner update and delete over skills the removed API let +-- only workspace admins touch. Anything else stays in `ai_skill` for an operator +-- to place deliberately. +-- +-- What was actually inserted is recorded rather than inferred. Inferring it from +-- "is there an ai_skill resource at the destination" reports nothing when the +-- blocker is itself an ai_skill with different instructions — the one case where +-- the skipped skill is least likely to be noticed. +CREATE TEMP TABLE ai_skill_copied AS +WITH inserted AS ( + INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) + SELECT + s.workspace_id, + 'f/skills/' || s.name, + jsonb_build_object('content', s.instructions), + s.description, + 'ai_skill', + s.edited_by, + s.edited_at + FROM ai_skill s + JOIN folder f + ON f.workspace_id = s.workspace_id + AND f.name = 'skills' + AND f.extra_perms = '{"g/all": false}'::jsonb + AND cardinality(f.owners) = 0 + ON CONFLICT (workspace_id, path) DO NOTHING + RETURNING workspace_id, path +) +SELECT workspace_id, path FROM inserted; + +-- Anything not copied is still in `ai_skill`, but nothing reads that table any +-- more, so from the app's side the skill is missing until an operator places it. +-- Name them rather than leaving that to be discovered. +DO $$ +DECLARE + leftover RECORD; +BEGIN + FOR leftover IN + SELECT s.workspace_id, s.name + FROM ai_skill s + WHERE NOT EXISTS ( + SELECT 1 FROM ai_skill_copied c + WHERE c.workspace_id = s.workspace_id + AND c.path = 'f/skills/' || s.name + ) + LOOP + RAISE WARNING 'ai_skill %/% was not copied to a resource (its destination or the f/skills folder is already taken); it remains in the ai_skill table', + leftover.workspace_id, leftover.name; + END LOOP; +END $$; + +DROP TABLE ai_skill_copied; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql new file mode 100644 index 0000000000..e62fc02975 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql @@ -0,0 +1,12 @@ +-- Drafts owned by a principal with no login account cannot exist under the constraint; drop them +-- before restoring it. +DELETE FROM draft +WHERE email IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM password WHERE password.email = draft.email); + +ALTER TABLE draft + ADD CONSTRAINT draft_password_fkey + FOREIGN KEY (email) + REFERENCES password(email) + ON DELETE CASCADE + ON UPDATE CASCADE; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql new file mode 100644 index 0000000000..65a6686e95 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql @@ -0,0 +1,3 @@ +-- The delete and rename this cascaded are now explicit, at the sites that remove or rename an +-- account; `windmill_common::user_drafts::delete_drafts_of_email` carries the reasoning. +ALTER TABLE draft DROP CONSTRAINT IF EXISTS draft_password_fkey; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index b5ed9dff9e..395e692623 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.800.0" +version = "1.803.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.800.0" +version = "1.803.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.800.0" +version = "1.803.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.800.0" +version = "1.803.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index ea7160821b..45c8b0d94e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.800.0" +version = "1.803.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index fa88cf6cdc..f84812151d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,14 +7,14 @@ */ use anyhow::Context; use monitor::{ - load_base_url, load_otel, reload_critical_alerts_on_db_oversize, - reload_delete_logs_periodically_setting, reload_indexer_config, - reload_instance_python_version_setting, reload_maven_repos_setting, + flush_pending_log_files_to_object_store, load_base_url, load_otel, + reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, + reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, reload_maven_settings_xml_setting, reload_no_default_maven_setting, reload_nuget_config_setting, reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, reload_workspace_registries_setting, - flush_pending_log_files_to_object_store, send_logs_to_object_store, WORKERS_NAMES, + send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::{Pool, Postgres}; @@ -53,9 +53,9 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, @@ -140,12 +140,12 @@ use crate::monitor::{ reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, - reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, - reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, - reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, - reload_service_log_retention_secs_setting, reload_smtp_config, + reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, + reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_sandbox_image_cache_max_setting, + reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, + reload_sandbox_image_pull_policy_setting, reload_sandbox_registry_auth_setting, + reload_scim_token_setting, reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -406,8 +406,12 @@ struct HubResourceTypeRaw { pub schema: Option, pub app: String, pub description: Option, + /// Absent from hubs predating the column, and from caches written before it. + #[serde(default)] + pub format_extension: Option, } + /// Processed resource type with parsed schema #[derive(serde::Deserialize, serde::Serialize, Clone)] pub struct HubResourceType { @@ -416,6 +420,18 @@ pub struct HubResourceType { pub schema: Option, pub app: String, pub description: Option, + /// Doubly optional on purpose. A cache written before this column has no key at + /// all (`None`) and must leave the stored extension alone; one written since + /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely + /// dropping it and must clear. A single `Option` conflates the two, and picking + /// either meaning breaks the other — as does plain serde, which folds `null` + /// into the outer `None`, hence the wrapping deserializer. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -462,6 +478,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { schema, app: rt.app, description: rt.description, + format_extension: Some(rt.format_extension), }) }) .collect(); @@ -503,9 +520,17 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh tracing::info!("Found {} cached resource types", cached_types.len()); - // Get existing resource types in admins workspace - let existing_types: Vec<(String, Option, Option)> = sqlx::query_as( - "SELECT name, schema, description FROM resource_type WHERE workspace_id = 'admins'", + // Get existing resource types in admins workspace. `format_extension` is part of + // the comparison below, so a type whose only change is gaining or losing it is + // not mistaken for unchanged; `is_fileset` decides whether it may take one. + let existing_types: Vec<( + String, + Option, + Option, + Option, + bool, + )> = sqlx::query_as( + "SELECT name, schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -513,19 +538,42 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option), + (Option, Option, Option, bool), > = existing_types .into_iter() - .map(|(name, schema, desc)| (name, (schema, desc))) + .map(|(name, schema, desc, format_extension, is_fileset)| { + (name, (schema, desc, format_extension, is_fileset)) + }) .collect(); let mut synced_count = 0; let mut skipped_count = 0; for rt in cached_types { - // Check if resource type already exists with same schema and description - if let Some((existing_schema, existing_desc)) = existing_map.get(&rt.name) { - if existing_schema == &rt.schema && existing_desc == &rt.description { + let existing = existing_map.get(&rt.name); + let is_fileset = existing.map(|(_, _, _, f)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _)| e.clone()); + // A fileset is a set of files, so it cannot also be one file. Create, update + // and the manual sync all reject the pair; this writer would otherwise + // persist it onto a same-named local fileset. + // + // A cache with no key at all leaves the stored value alone, so the target is + // what is already there — which is also what makes the comparison below + // agree with the write instead of re-upserting the row on every boot. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(from_cache) => from_cache.clone(), + None => stored_extension.clone(), + } + }; + + if let Some((existing_schema, existing_desc, _, _)) = existing { + if existing_schema == &rt.schema + && existing_desc == &rt.description + && stored_extension == format_extension + { skipped_count += 1; continue; } @@ -533,14 +581,19 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // Insert or update resource type sqlx::query( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // `format_extension` is resolved above rather than coalesced here: a + // COALESCE could never clear one, so a hub that dropped an extension + // would leave the stale value behind forever. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) + VALUES ('admins', $1, $2, $3, $4, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) + .bind(&format_extension) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; @@ -1960,6 +2013,9 @@ async fn process_notify_event( SERVICE_LOG_RETENTION_SECS_SETTING => { reload_service_log_retention_secs_setting(conn).await } + OTEL_TRACES_RETENTION_SECS_SETTING => { + reload_otel_traces_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ce09cebe71..86e75b7de9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -64,10 +64,11 @@ use windmill_common::{ JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, @@ -97,10 +98,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS, - JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, - METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, - OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, + DEFAULT_OTEL_TRACES_RETENTION_SECS, DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, + JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, + METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, + OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -514,6 +515,15 @@ pub async fn initial_load( Ordering::Relaxed, ) }); + pass.setting(OTEL_TRACES_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_otel_traces_retention_secs(parse_setting_value::( + v, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + )) + }); pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { STORE_AUDIT_LOGS_S3.store( parse_setting_value::( @@ -1682,6 +1692,57 @@ const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; /// across ticks rather than inside one, the way the neighbouring sweeps already do. const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; +/// One span per HTTP request made from a job script, so the table grows far faster than the +/// job table it is keyed against; batched for the same reason the service log sweep is. +const OTEL_TRACES_DELETE_BATCH: i64 = 10_000; +const OTEL_TRACES_DELETE_MAX_BATCHES: usize = 10; + +/// Delete HTTP request tracing spans older than `retention_secs`, returning how many went. +/// +/// `retention_secs` is a parameter rather than a read of the process-wide setting so a test can +/// pin a window without writing state the other tests in this binary run against concurrently. +async fn delete_expired_otel_traces(db: &DB, retention_secs: i64) -> u64 { + // `start_time_unix_nano` is the proto field stored verbatim, so the cutoff is built in that + // unit rather than compared against `now()`. Truncating the epoch to whole seconds first + // keeps the multiplication inside `bigint`. + // + // Batched on `ctid`, not on the `(trace_id, span_id)` primary key: with the key the planner + // hashes the LIMITed subquery and Seq Scans the whole table to probe it, which at the size + // this table reaches is the cost the batching exists to avoid. `ctid` plans as a Tid Scan, so + // each batch touches only the rows it deletes. Safe because the subquery and the delete share + // one snapshot, and spans are never updated after insert. + let mut deleted = 0; + for _ in 0..OTEL_TRACES_DELETE_MAX_BATCHES { + let batch = sqlx::query!( + "DELETE FROM otel_traces WHERE ctid IN ( + SELECT ctid FROM otel_traces + WHERE start_time_unix_nano < EXTRACT( + EPOCH FROM now() - ($1::bigint::text || ' s')::interval + )::bigint * 1000000000 + LIMIT $2 + )", + retention_secs, + OTEL_TRACES_DELETE_BATCH, + ) + .execute(db) + .await; + + match batch { + Ok(res) => { + deleted += res.rows_affected(); + if (res.rows_affected() as i64) < OTEL_TRACES_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting expired otel trace spans: {:?}", e); + break; + } + } + } + deleted +} + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1808,6 +1869,12 @@ pub async fn delete_expired_items(db: &DB) -> () { } } + let deleted_spans = + delete_expired_otel_traces(db, windmill_common::otel_traces_retention_secs()).await; + if deleted_spans > 0 { + tracing::info!("deleted {} expired otel trace spans", deleted_spans); + } + let audit_retention_days = audit_log_retention_days().await; let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; @@ -2927,6 +2994,21 @@ pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { } } +pub async fn reload_otel_traces_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_otel_traces_retention_secs(v), + Err(e) => tracing::error!("Error reloading otel traces retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, @@ -4839,7 +4921,7 @@ async fn poll_git_fork_branches( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces") .execute(db) .await?; Ok(()) @@ -6975,6 +7057,46 @@ mod zombie_worker_memory_pct_tests { } } +#[cfg(test)] +mod otel_traces_retention_tests { + use super::{delete_expired_otel_traces, DB}; + + async fn insert_span(db: &DB, id: u8, age_secs: i64) { + sqlx::query!( + "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano) + VALUES ($1, $2, 'GET /', 3, $3, $3)", + &[id; 16][..], + &[id; 8][..], + (chrono::Utc::now() - chrono::Duration::seconds(age_secs)) + .timestamp_nanos_opt() + .unwrap(), + ) + .execute(db) + .await + .unwrap(); + } + + /// The cutoff crosses two units: a retention configured in seconds against a column holding + /// nanoseconds. Getting that conversion wrong is silent in both directions — a window a + /// billion times too wide never deletes anything, one a billion times too narrow deletes + /// every span on the next tick — so pin it on either side of the boundary. + #[sqlx::test(migrations = "./migrations")] + async fn deletes_only_spans_past_the_window(db: DB) -> anyhow::Result<()> { + let day = 60 * 60 * 24; + insert_span(&db, 1, 60).await; + insert_span(&db, 2, 6 * day).await; + insert_span(&db, 3, 8 * day).await; + + assert_eq!(delete_expired_otel_traces(&db, 7 * day).await, 1); + + let kept = sqlx::query_scalar!("SELECT trace_id FROM otel_traces ORDER BY trace_id") + .fetch_all(&db) + .await?; + assert_eq!(kept, vec![vec![1u8; 16], vec![2u8; 16]]); + Ok(()) + } +} + #[cfg(test)] mod log_file_listing_tests { use super::{rotated_log_files, sorted_log_files}; diff --git a/backend/tests/fixtures/inline_preview_auth.sql b/backend/tests/fixtures/inline_preview_auth.sql index 59fe2dc917..67ca598bda 100644 --- a/backend/tests/fixtures/inline_preview_auth.sql +++ b/backend/tests/fixtures/inline_preview_auth.sql @@ -2,7 +2,9 @@ -- Layered on top of `base` (which provides test-workspace and the non-operator -- `test-user-2`/SECRET_TOKEN_2). Adds an Operator member so we can assert that -- Operators cannot reach the arbitrary-code inline preview path --- (`POST /jobs/run_inline/preview`). +-- (`POST /jobs/run_inline/preview`) with their own token, plus two deployed script +-- jobs of the operator: one running, so we can assert that its WM_TOKEN can, and +-- one queued but not yet pulled, so we can assert that "queued" is not enough. INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) VALUES ('operator@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Operator User'); @@ -12,3 +14,11 @@ INSERT INTO usr(workspace_id, email, username, is_admin, operator, role) VALUES INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('OPERATOR_TOKEN'::bytea), 'hex'), 'OPERATOR_T', 'OPERATOR_TOKEN', 'operator@windmill.dev', 'operator token', false); + +INSERT INTO v2_job(id, workspace_id, kind, runnable_path, created_by, permissioned_as, permissioned_as_email) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'); + +INSERT INTO v2_job_queue(id, workspace_id, scheduled_for, running) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', now(), true), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', now(), false); diff --git a/backend/tests/inline_preview_auth.rs b/backend/tests/inline_preview_auth.rs index 97b70fb545..8841719962 100644 --- a/backend/tests/inline_preview_auth.rs +++ b/backend/tests/inline_preview_auth.rs @@ -9,16 +9,33 @@ //! was the incomplete-fix residual of CVE-2026-22683, whose v1.615.0 patch only //! covered the entity-CRUD endpoints and left this direct inline-exec sink open. //! +//! The guard on both routes has one exemption: `wmill.datatable()` called from +//! inside a job the operator is running. Operators can only run deployed code, +//! so a request the job's WM_TOKEN authenticates comes from code a non-operator +//! authored, and the exemption is limited to the request shape the helper sends +//! (PostgreSQL against a `datatable://` database) so a leaked WM_TOKEN cannot +//! be replayed to run anything else. +//! //! This test pins down: -//! - an Operator is rejected by the operator guard (the core fix; pre-fix this -//! reached the inline executor instead of returning 401), and +//! - an Operator's own token is rejected by the operator guard (the core fix; +//! pre-fix this reached the inline executor instead of returning 401), //! - a regular non-operator passes the guard (the fix must not over-block the //! legitimate inline preview flow): in the test harness the worker inline //! utils are not registered, so a caller past the guard gets the distinct -//! "worker inline functions" error rather than the operator rejection. +//! "worker inline functions" error rather than the operator rejection, +//! - an Operator's job token passes the guard for a datatable query while its +//! job is running, on the inline route and on the `/jobs/run/preview` +//! fallback the SDKs use when the worker has no internal server, +//! - the same token is rejected for any other payload (in-process DuckDB, or a +//! `-- database` directive redirecting the query, whether written literally or +//! reached through a `WM_INTERNAL_DB` marker) and for a deferred run, +//! - an Operator's job token for a job that is not running, whether finished or +//! merely queued, is rejected. use serde_json::json; use sqlx::{Pool, Postgres}; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::Authed; use windmill_test_utils::*; fn client() -> reqwest::Client { @@ -38,11 +55,65 @@ fn inline_preview_body() -> serde_json::Value { }) } +/// The request `wmill.datatable("main")` sends: PostgreSQL against `datatable://main`. +fn datatable_query_body() -> serde_json::Value { + json!({ + "language": "postgresql", + "content": "SELECT 1 AS x;", + "args": { "database": "datatable://main" } + }) +} + +/// Mint the WM_TOKEN a job hands its own code: an internally-signed job JWT +/// (note the `job_id` claim) for the fixture's operator, exactly as the worker +/// issues it when the operator runs a deployed script. +async fn operator_job_token(job_id: uuid::Uuid) -> String { + let authed = Authed { + email: "operator@windmill.dev".to_string(), + username: "operator-user".to_string(), + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes: None, + token_prefix: None, + }; + create_jwt_token( + authed, + "test-workspace", + 3600, + Some(job_id), + Some("ephemeral-script".to_string()), + None, + None, + ) + .await + .expect("mint operator job token") +} + const OPERATOR_GUARD_MSG: &str = "Operators cannot run preview jobs"; +/// The fixture's deployed-script jobs of the operator: one running, one queued. +const RUNNING_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000001"; +const QUEUED_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000002"; + +async fn post(url: &str, token: &str, body: &serde_json::Value) -> (u16, String) { + let resp = authed(client().post(url), token) + .json(body) + .send() + .await + .expect("request"); + let status = resp.status().as_u16(); + let body = resp.text().await.expect("body"); + (status, body) +} + #[sqlx::test(fixtures("base", "inline_preview_auth"))] async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; + // The server decodes WM_TOKENs with the same in-process JWT secret, so + // setting it once lets us mint valid ones below. + set_jwt_secret().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -51,12 +122,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // 1. CORE REGRESSION: an Operator must be rejected by the operator guard. // Pre-fix this fell through to the inline executor (arbitrary code // execution); post-fix it returns 401 with the operator guard message. - let resp = authed(client().post(&url), "OPERATOR_TOKEN") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "OPERATOR_TOKEN", &inline_preview_body()).await; assert_eq!( status, 401, "Operator must be rejected from inline preview (got {status}): {body}" @@ -71,12 +137,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // the worker inline utils, so the request proceeds past the guard and // fails later with the distinct "worker inline functions" error — proving // the operator guard did not reject it. - let resp = authed(client().post(&url), "SECRET_TOKEN_2") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "SECRET_TOKEN_2", &inline_preview_body()).await; assert_ne!( status, 401, "non-operator must not be blocked by the operator guard (got {status}): {body}" @@ -86,5 +147,113 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result "non-operator must not hit the operator guard, got: {body}" ); + // 3. The WM_TOKEN of a deployed-script job the Operator is running passes the + // guard for a datatable query: this is `wmill.datatable()` called from + // inside that job. As in 2, the harness then fails with the "worker inline + // functions" error. + let running_job_token = + operator_job_token(uuid::Uuid::parse_str(RUNNING_JOB_ID).unwrap()).await; + let (status, body) = post(&url, &running_job_token, &datatable_query_body()).await; + assert_ne!( + status, 401, + "operator job token of a running job must pass the guard for a datatable query (got {status}): {body}" + ); + assert!( + !body.contains(OPERATOR_GUARD_MSG), + "operator job token of a running job must not hit the operator guard, got: {body}" + ); + + // 4. The same token is rejected for any other payload: the exemption covers + // the datatable request shape only, never in-process DuckDB, and never a + // `-- database` directive, which the executor honors over `args.database`. + let mut redirected = datatable_query_body(); + redirected["content"] = json!("-- database u/test-user/other_db\nSELECT 1 AS x;"); + let mut to_s3 = datatable_query_body(); + to_s3["content"] = json!("-- s3\nSELECT 1 AS x;"); + let mut resource_db = datatable_query_body(); + resource_db["args"]["database"] = json!("$res:u/test-user/other_db"); + // A marker is a single line the directive regexes cannot match; the directive only + // appears once the executor expands it, so the guard must check the expansion. + let mut marker = datatable_query_body(); + marker["content"] = json!(concat!( + r#"-- WM_INTERNAL_DB_SELECT {"table":"t","columnDefs":[{"field":"id","datatype":"int4"}],"#, + r#""whereClause":"true\n-- database u/test-user/other_db\n AND true"}"# + )); + for (label, payload) in [ + ("DuckDB", inline_preview_body()), + ("database directive", redirected), + ("s3 directive", to_s3), + ("resource database", resource_db), + ("marker-expanded database directive", marker), + ] { + let (status, body) = post(&url, &running_job_token, &payload).await; + assert_eq!( + status, 401, + "operator job token must be rejected for a {label} payload (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} payload must be the operator guard, got: {body}" + ); + } + + // 5. An Operator's job token whose job is not running is rejected like the + // operator's own token, whether the job is over (no queue row) or merely + // queued: a WM_TOKEN that leaked through logs cannot be replayed once the + // job is over. + for (label, job_id) in [ + ("finished", uuid::Uuid::new_v4()), + ("queued", uuid::Uuid::parse_str(QUEUED_JOB_ID).unwrap()), + ] { + let token = operator_job_token(job_id).await; + let (status, body) = post(&url, &token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token of a {label} job must be rejected (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} job must be the operator guard, got: {body}" + ); + } + + // 6. The SDKs fall back to `/jobs/run/preview` when the worker has no internal + // server (agent workers). The same exemption applies there: the running + // job's token queues the datatable query (201 with the job id), the + // operator's own token is still refused. + let fallback_url = format!("http://localhost:{port}/api/w/test-workspace/jobs/run/preview"); + let (status, body) = post(&fallback_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 201, + "operator job token of a running job must queue a datatable preview (got {status}): {body}" + ); + let (status, body) = post(&fallback_url, "OPERATOR_TOKEN", &datatable_query_body()).await; + assert_eq!( + status, 401, + "Operator must be rejected from the preview fallback (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection must be the operator guard, got: {body}" + ); + + // 7. A deferred run on the fallback would outlive the running job the + // exemption keys off, so the running job's token cannot schedule one. + for deferral in [ + "scheduled_in_secs=86400", + "scheduled_for=2099-01-01T00:00:00Z", + ] { + let deferred_url = format!("{fallback_url}?{deferral}"); + let (status, body) = post(&deferred_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token must not schedule a deferred preview with {deferral} (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for {deferral} must be the operator guard, got: {body}" + ); + } + Ok(()) } diff --git a/backend/tests/object_storage_test_ssrf.rs b/backend/tests/object_storage_test_ssrf.rs new file mode 100644 index 0000000000..aa9eaaf74c --- /dev/null +++ b/backend/tests/object_storage_test_ssrf.rs @@ -0,0 +1,97 @@ +//! `POST /api/settings/test_object_storage_config` runs the probe on the API server and reflects the +//! upstream response, so every non-super-admin must be rejected for private/loopback endpoints and +//! the Filesystem backend on every deployment (`CLOUD_HOSTED` is unset here), while a super admin's +//! Filesystem probe still round-trips. Requires the `parquet` feature, like the route. +#![cfg(feature = "parquet")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use windmill_test_utils::*; + +const SUPER_ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const USER_TOKEN: &str = "SECRET_TOKEN_2"; + +async fn test_object_storage( + url: &str, + token: &str, + body: serde_json::Value, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(url) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn object_storage_test_is_restricted_for_non_super_admins_off_cloud( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/settings/test_object_storage_config", + server.addr.port() + ); + + // A loopback "S3 endpoint" standing in for an internal service: the probe must be rejected + // before the server opens a connection to it. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let internal_port = listener.local_addr()?.port(); + let connected = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let connected = connected.clone(); + async move { + while listener.accept().await.is_ok() { + connected.store(true, Ordering::SeqCst); + } + } + }); + let internal_s3 = json!({ + "type": "S3", + "bucket": "bucket", + "region": "us-east-1", + "access_key": "key", + "secret_key": "secret", + "endpoint": format!("http://127.0.0.1:{internal_port}"), + "allow_http": true, + "path_style": true, + }); + let (status, body) = test_object_storage(&url, USER_TOKEN, internal_s3).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a loopback endpoint (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + assert!( + !connected.load(Ordering::SeqCst), + "the server must not connect to the rejected endpoint" + ); + + let tmp = tempfile::tempdir()?; + let filesystem = json!({ "type": "Filesystem", "root_path": tmp.path().to_str().unwrap() }); + let (status, body) = test_object_storage(&url, USER_TOKEN, filesystem.clone()).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a Filesystem backend (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + + // Super admins keep the unrestricted path. + let (status, body) = test_object_storage(&url, SUPER_ADMIN_TOKEN, filesystem).await?; + assert_eq!( + status, 200, + "super admin must be able to test a Filesystem backend (got {status}): {body}" + ); + Ok(()) +} diff --git a/backend/tests/relock_noop.rs b/backend/tests/relock_noop.rs new file mode 100644 index 0000000000..5cb8e3f51e --- /dev/null +++ b/backend/tests/relock_noop.rs @@ -0,0 +1,415 @@ +use sqlx::{Pool, Postgres}; +use tokio_stream::StreamExt; +use windmill_api_client::types::NewScript; +use windmill_common::scripts::{deploy_relocked_version, fetch_script_for_update}; +use windmill_test_utils::*; + +const W: &str = "test-workspace"; + +const A: &str = r#"export async function main() { return "a" }"#; +const A_COMMENTED: &str = r#"// same dependencies, different content +export async function main() { return "a" }"#; +const A_WITH_LODASH: &str = r#"import _ from "lodash@4.17.21"; +export async function main() { return _.trim(" a ") }"#; +const B: &str = r#"import { main as a } from "/f/rel/a.ts"; +export async function main() { return "b" + (await a()) }"#; +const C: &str = r#"import { main as b } from "/f/rel/b.ts"; +export async function main() { return "c" + (await b()) }"#; + +fn bun_script(path: &str, content: &str, parent_hash: Option) -> NewScript { + NewScript { + draft_only: None, + content: content.into(), + language: windmill_api_client::types::ScriptLang::Bun, + lock: None, + parent_hash, + path: path.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_secs: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + auto_kind: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + modules: None, + } +} + +#[derive(sqlx::FromRow, Debug)] +struct Version { + hash: i64, + archived: bool, + lock: Option, + created_at: chrono::DateTime, +} + +/// Every version of `path`, oldest first. +async fn versions(db: &Pool, path: &str) -> Vec { + sqlx::query_as( + "SELECT hash, archived, lock, created_at FROM script + WHERE workspace_id = $1 AND path = $2 ORDER BY created_at", + ) + .bind(W) + .bind(path) + .fetch_all(db) + .await + .unwrap() +} + +fn live(versions: &[Version]) -> &Version { + versions.iter().rev().find(|v| !v.archived).unwrap() +} + +/// `(path, status, logs)` of every dependency job created after `since`, in completion order. +async fn dependency_jobs_since( + db: &Pool, + since: chrono::DateTime, +) -> Vec<(String, String, String)> { + sqlx::query_as( + "SELECT j.runnable_path, c.status::text, COALESCE(l.logs, '') FROM v2_job_completed c + JOIN v2_job j ON j.id = c.id + LEFT JOIN job_logs l ON l.job_id = c.id + WHERE j.kind = 'dependencies' AND j.created_at > $1 + ORDER BY c.started_at", + ) + .bind(since) + .fetch_all(db) + .await + .unwrap() +} + +async fn wait_for_jobs( + completed: &mut (impl futures::Stream + Unpin), + count: usize, +) { + for _ in 0..count { + completed.next().await; + } + // Then let anything else that was queued run out, so a job the assertions say must not + // exist would have shown up here. + while let Ok(Some(_)) = + tokio::time::timeout(std::time::Duration::from_secs(2), completed.next()).await + {} +} + +/// A redeploy of an imported script whose dependencies did not move relocks its importer, +/// and that relock must deploy nothing: no new version, and no dependency job for the +/// importer's own importers. A redeploy that does change the dependencies still walks the +/// whole chain with a new version at each step. +#[sqlx::test(fixtures("base"))] +async fn relative_import_relock_deploys_only_when_the_lock_changed( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + // One at a time: each deploy's dependency job records the importer's edges, and an + // importer whose edges are recorded is what a later relock of it can skip on. + for (path, content) in [("f/rel/a", A), ("f/rel/b", B), ("f/rel/c", C)] { + client + .create_script(W, &bun_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + } + let b_before = versions(&db, "f/rel/b").await; + let c_before = versions(&db, "f/rel/c").await; + assert_eq!(b_before.len(), 1); + assert_eq!(c_before.len(), 1); + + // Content-only change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &bun_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!( + paths, + ["f/rel/a", "f/rel/b"], + "the leaf's own job and one no-op relock of its importer, and nothing for c" + ); + assert!( + jobs[1] + .2 + .contains("Lock unchanged: no new version deployed"), + "b's relock should have found its lock unchanged: {}", + jobs[1].2 + ); + let b_after = versions(&db, "f/rel/b").await; + let c_after = versions(&db, "f/rel/c").await; + assert_eq!( + b_after.len(), + 1, + "an unchanged relock must not mint a version" + ); + assert_eq!(live(&b_after).hash, live(&b_before).hash); + assert_eq!(c_after.len(), 1); + assert_eq!(live(&c_after).hash, live(&c_before).hash); + + // A dependency change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &bun_script("f/rel/a", A_WITH_LODASH, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/c"]); + for path in ["f/rel/b", "f/rel/c"] { + let vs = versions(&db, path).await; + assert_eq!( + vs.len(), + 2, + "{path}: a changed relock deploys a new version" + ); + assert!( + vs[0].archived && !vs[1].archived, + "{path}: parent archived, child live" + ); + assert!(vs[0].created_at < vs[1].created_at, "{path}: lineage order"); + assert!( + vs[1].lock.as_deref().unwrap_or("").contains("lodash"), + "{path}: the new version carries the new lock: {:?}", + vs[1].lock + ); + } + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock that has to wait on its head's row lock, because a deploy of the same path holds +/// it, must find the version that deploy left and requeue itself for it rather than fail. The +/// blocked statement re-checks only the row it selected, which the deploy archived, and comes +/// back empty; the successor is only visible to a fresh read. +#[sqlx::test(fixtures("base"))] +async fn relock_waiting_on_a_deploy_requeues_for_its_successor( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + for (path, content) in [("f/rel/a", A), ("f/rel/b", B)] { + client + .create_script(W, &bun_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + } + + // A deploy of b that holds its head's row lock for as long as this transaction lives. + let mut deploy = db.begin().await.unwrap(); + let head = fetch_script_for_update("f/rel/b", W, &mut *deploy) + .await + .unwrap() + .unwrap(); + + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &bun_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + + // b's relock skips generation and reaches its commit, where it waits on the lock. + let mut waiting = false; + for _ in 0..300 { + waiting = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' + AND query LIKE '%FROM script WHERE path = $1%FOR UPDATE%')", + ) + .fetch_one(&db) + .await + .unwrap(); + if waiting { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!(waiting, "b's relock never reached the row lock"); + + // The deploy lands: the head is archived and a successor with its own lock takes + // its place, while the relock is still waiting. + let lock = head.lock.clone().unwrap(); + let successor = + deploy_relocked_version(&mut deploy, head, None, Some(&lock), None, None) + .await + .unwrap(); + deploy.commit().await.unwrap(); + + // a's own job, the relock that waited, and the relock it queued for the successor. + wait_for_jobs(&mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/b"], "{jobs:?}"); + assert!( + jobs.iter().all(|(_, status, _)| status == "success"), + "no relock may fail on the wait: {jobs:?}" + ); + assert!( + jobs[1] + .2 + .contains("was deployed while this lock was generated"), + "the waiting relock should have seen the successor: {}", + jobs[1].2 + ); + assert!( + jobs[2] + .2 + .contains("Lock unchanged: no new version deployed"), + "the requeued relock should find the successor's lock current: {}", + jobs[2].2 + ); + let vs = versions(&db, "f/rel/b").await; + assert_eq!( + vs.len(), + 2, + "the deploy's successor and nothing else: {vs:?}" + ); + assert_eq!(live(&vs).hash, successor); + }, + port, + ) + .await; + + Ok(()) +} + +/// A multi-file importer: on a skipped relock each module gets its own last lock back, not the +/// parent script's, so an import's content-only redeploy leaves the importer alone as well. +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base"))] +async fn multi_file_importer_relock_is_a_no_op_too(db: Pool) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + let py = |path: &str, content: &str, parent_hash: Option, with_module: bool| { + let mut ns = bun_script(path, content, parent_hash); + ns.language = windmill_api_client::types::ScriptLang::Python3; + if with_module { + ns.modules = Some(std::collections::HashMap::from([( + "helper.py".to_string(), + serde_json::json!({ + "content": "def greet(x):\n return 'hi ' + x\n", + "language": "python3" + }), + )])); + } + ns + }; + async fn module_lock(db: &Pool) -> Option { + sqlx::query_scalar( + "SELECT modules->'helper.py'->>'lock' FROM script + WHERE workspace_id = $1 AND path = 'f/rel/pb' AND archived = false", + ) + .bind(W) + .fetch_one(db) + .await + .unwrap() + } + + in_test_worker( + &db, + async { + client + .create_script(W, &py("f/rel/pa", "def main():\n return 'a'\n", None, false)) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + client + .create_script( + W, + &py( + "f/rel/pb", + "from f.rel.pa import main as a\nfrom .helper import greet\n\ndef main():\n return greet(a())\n", + None, + true, + ), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + let lock_before = module_lock(&db).await; + assert!(lock_before.is_some(), "the module got a lock of its own on deploy"); + + let since = chrono::Utc::now(); + let pa_hash = live(&versions(&db, "f/rel/pa").await).hash; + client + .create_script( + W, + &py( + "f/rel/pa", + "# same dependencies\ndef main():\n return 'a'\n", + Some(format!("{pa_hash:016x}")), + false, + ), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/pa", "f/rel/pb"], "{jobs:?}"); + assert!( + jobs[1].2.contains("Lock unchanged: no new version deployed"), + "the multi-file importer's relock should be a no-op: {}", + jobs[1].2 + ); + assert_eq!(versions(&db, "f/rel/pb").await.len(), 1); + assert_eq!(module_lock(&db).await, lock_before, "the module keeps its own lock"); + }, + port, + ) + .await; + + Ok(()) +} diff --git a/backend/tests/relock_skip.rs b/backend/tests/relock_skip.rs index 8262a38cf4..bf5cb24cbf 100644 --- a/backend/tests/relock_skip.rs +++ b/backend/tests/relock_skip.rs @@ -266,7 +266,10 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + // Empty content leaves every importer's lock as it was, so only the five direct + // importers of the default deps run a job: an unchanged script relock deploys no + // version and so queues nothing for its own importers. + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; // Note: within a cascade, the same script may be triggered multiple times. // After the first trigger relocks and stores the hash, subsequent triggers skip. @@ -295,7 +298,7 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; assert!( diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index d99d955de9..05de1183cd 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -288,7 +288,6 @@ pub enum ScopeDomain { Configs, OAuth, AI, - AiSkills, AiEvals, // AI agent eval datasets Indexer, @@ -349,7 +348,6 @@ impl ScopeDomain { Self::Configs => "configs", Self::OAuth => "oauth", Self::AI => "ai", - Self::AiSkills => "ai_skills", Self::AiEvals => "ai_evals", Self::Capture => "capture", Self::Drafts => "drafts", @@ -405,7 +403,6 @@ impl ScopeDomain { "configs" => Some(Self::Configs), "oauth" => Some(Self::OAuth), "ai" => Some(Self::AI), - "ai_skills" => Some(Self::AiSkills), "ai_evals" => Some(Self::AiEvals), "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), @@ -1202,12 +1199,6 @@ mod tests { assert_eq!(domain, ScopeDomain::FlowConversations); assert_eq!(kind, None); assert_eq!(route_suffix, Some("flow_conversations/list".to_string())); - - let (domain, kind, route_suffix) = - extract_domain_from_route("/api/w/test_workspace/ai_skills/list").unwrap(); - assert_eq!(domain, ScopeDomain::AiSkills); - assert_eq!(kind, None); - assert_eq!(route_suffix, Some("ai_skills/list".to_string())); } #[test] @@ -1368,11 +1359,6 @@ mod tests { ScopeDomain::from_str("flow_conversations"), Some(ScopeDomain::FlowConversations) ); - assert_eq!( - ScopeDomain::from_str("ai_skills"), - Some(ScopeDomain::AiSkills) - ); - // Test canonical string conversion assert_eq!(ScopeDomain::Acls.as_str(), "acls"); assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); @@ -1381,41 +1367,6 @@ mod tests { ScopeDomain::FlowConversations.as_str(), "flow_conversations" ); - assert_eq!(ScopeDomain::AiSkills.as_str(), "ai_skills"); - } - - #[test] - fn test_ai_skills_scope_access() { - let read_scopes = vec!["ai_skills:read".to_string()]; - assert!( - check_route_access(&read_scopes, "/api/w/test_workspace/ai_skills/list", "GET").is_ok() - ); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/get/foo", - "GET" - ) - .is_ok()); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_err()); - - let write_scopes = vec!["ai_skills:write".to_string()]; - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_ok()); - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/delete/foo", - "DELETE" - ) - .is_ok()); } #[test] diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 6353cd429a..ecb7ba405b 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -157,7 +157,8 @@ async fn list_flows( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 5a7af05cbe..c6f88dace5 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -183,9 +183,9 @@ async fn add_granular_acl( if kind == "folder" { let change_type = if write.unwrap_or(false) { - "grant_read" - } else { "grant_write" + } else { + "grant_read" }; crate::folders::log_folder_permission_change( &mut *tx, diff --git a/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs new file mode 100644 index 0000000000..3e1035ff82 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs @@ -0,0 +1,186 @@ +//! Request headers reaching a runnable's preprocessor over MCP. +//! +//! The property this pins is structural rather than a filter: the model writes +//! the tool's arguments, which become `event.body`, while the server writes +//! `event.headers`. A model that guesses a header's name can only ever land in +//! `body`, so an identity read from `headers` is one prompt injection cannot +//! forge. Nothing else in the suite exercises MCP argument shaping end to end. +//! +//! Requires: bun runtime, live database (migrations applied by sqlx::test). +#![cfg(feature = "mcp")] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const SCRIPT_PATH: &str = "u/test-user/mcp_hdr_probe"; + +/// Echoes the two halves of the event separately, so the assertions can tell +/// which one a value arrived in. +const PREPROCESSOR_SCRIPT: &str = r#" +export async function preprocessor(event: any) { + return { + kind: event.kind, + from_headers: event.headers?.["x-user-id"] ?? "", + from_body: event.body?.x_user_id ?? "", + header_names: Object.keys(event.headers ?? {}).sort(), + }; +} + +export async function main(kind: string, from_headers: string, from_body: string, header_names: string[]) { + return { kind, from_headers, from_body, header_names }; +} +"#; + +async fn insert_mcp_token(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('MCP_TOKEN'::bytea), 'hex'), 'MCP_TOK', 'MCP_TOKEN', 'test@windmill.dev', 'mcp token', true, ARRAY['mcp:all'])", + ) + .execute(db) + .await?; + Ok(()) +} + +/// POST one JSON-RPC message. The endpoint answers either `application/json` or +/// a single-event SSE stream, so strip the `data: ` framing before parsing. +async fn mcp_post(port: u16, headers: &[(&str, &str)], body: Value) -> anyhow::Result { + let mut req = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/mcp/w/test-workspace/mcp" + )) + .header("Authorization", "Bearer MCP_TOKEN") + .header("Accept", "application/json, text/event-stream") + .json(&body); + for (k, v) in headers { + req = req.header(*k, *v); + } + let text = req.send().await?.text().await?; + let payload = text + .lines() + .find_map(|l| l.strip_prefix("data: ")) + .unwrap_or(text.trim()); + serde_json::from_str(payload).map_err(|e| anyhow::anyhow!("unparseable MCP body {text:?}: {e}")) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_preprocessor_receives_the_callers_headers( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + insert_mcp_token(&db).await?; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": SCRIPT_PATH, + "summary": "mcp header probe", + "description": "", + "content": PREPROCESSOR_SCRIPT, + "language": "bun", + "lock": "", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { "x_user_id": { "type": "string" } }, + "required": [] + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 201, + "create script: {}", + resp.text().await.unwrap_or_default() + ); + + // A script counts as deployed once it has a lock, which normally arrives from + // a dependency job. Planting an empty one keeps the test to the path under + // test instead of a bun resolution whose timing it does not control. + sqlx::query("UPDATE script SET lock = '' WHERE path = $1 AND workspace_id = 'test-workspace'") + .bind(SCRIPT_PATH) + .execute(&db) + .await?; + + let tools = mcp_post( + port, + &[], + json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}), + ) + .await?; + let tool_name = tools["result"]["tools"] + .as_array() + .and_then(|list| { + list.iter() + .filter_map(|t| t["name"].as_str()) + .find(|n| n.contains("mcp__hdr__probe")) + }) + .ok_or_else(|| anyhow::anyhow!("the deployed script was not listed as a tool: {tools}"))? + .to_string(); + + let result = in_test_worker( + db.clone(), + async { + mcp_post( + port, + // Every name the withheld list covers has to be on the wire, or + // asserting its absence proves nothing. `Authorization` is already + // set by `mcp_post`, and `extract_token` reads it before the + // cookie, so sending one does not disturb auth. + &[ + ("X-User-Id", "alice@corp.example"), + ("Cookie", "session=secret"), + ("Proxy-Authorization", "Basic Zm9v"), + ], + json!({ + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + // The model names the header it wants to spoof. Its value is an + // argument, so it can only ever reach `event.body`. + "params": { "name": tool_name, "arguments": { "x_user_id": "attacker@evil.test" } } + }), + ) + .await + }, + port, + ) + .await?; + + let text = result["result"]["content"][0]["text"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("tool call returned no text content: {result}"))?; + let out: Value = serde_json::from_str(text)?; + + assert_eq!(out["kind"], "mcp", "preprocessor event kind: {out}"); + assert_eq!( + out["from_headers"], "alice@corp.example", + "the caller's header must reach event.headers: {out}" + ); + assert_eq!( + out["from_body"], "attacker@evil.test", + "the model's argument must land in event.body, not overwrite the header: {out}" + ); + + let names: Vec<&str> = out["header_names"] + .as_array() + .map(|a| a.iter().filter_map(|v| v.as_str()).collect()) + .unwrap_or_default(); + assert!( + names.contains(&"x-user-id"), + "event.headers must carry the request's own headers: {names:?}" + ); + for withheld in ["authorization", "cookie", "proxy-authorization"] { + assert!( + !names.contains(&withheld), + "{withheld} is withheld from a preprocessor: {names:?}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index 8c88c53454..3a146add27 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -38,6 +38,85 @@ fn new_script(path: &str, summary: &str, content: &str) -> serde_json::Value { }) } +/// A supplied lock queues no dependency job, so if the create does not record its hash nothing +/// ever will, and every importer of this script relocks on each of its deploys forever after. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_script_persists_supplied_lock_hash(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/supplied_lock"; + let lock = r#"{"version":"4","remote":{}}"#; + let mut script = new_script( + path, + "Supplied lock", + "export async function main() { return 42; }", + ); + script["lock"] = json!(lock); + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + let stored_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(stored_hash, windmill_common::scripts::hash_script(lock)); + + // A script deployed before the create recorded hashes has no row, and pushing it unchanged + // creates no version to hang one off. Without the write on that path it would keep its + // importers relocking until someone edited it. + sqlx::query!( + "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .execute(&db) + .await?; + + // The no-op comparison covers every field, so the push has to carry what the first deploy + // filled in by itself; `auto_parent` both resolves the parent and keeps the hash distinct. + script["auto_parent"] = json!(true); + script["ws_error_handler_muted"] = json!(false); + script["assets"] = json!([]); + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create?skip_if_noop=true" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "no-op push: {}", resp.text().await?); + + let versions: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await? + .unwrap_or_default(); + assert_eq!(versions, 1, "no-op push must not create a version"); + + let repaired_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(repaired_hash, windmill_common::scripts::hash_script(lock)); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -797,10 +876,12 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( // What a deploy leaves behind: the old head archived, a new one live at the path. // Copied through a temp table so this does not have to restate every column. - sqlx::query("CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1") - .bind(head) - .execute(&mut *winner) - .await?; + sqlx::query( + "CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1", + ) + .bind(head) + .execute(&mut *winner) + .await?; sqlx::query("UPDATE superseding SET hash = $1, archived = false, parent_hashes = ARRAY[$2]") .bind(head + 1) .bind(head) @@ -818,7 +899,10 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( let resp = tokio::time::timeout(std::time::Duration::from_secs(20), update).await??; let status = resp.status(); let body = resp.text().await?; - assert_eq!(status, 400, "losing the race should not read as success: {body}"); + assert_eq!( + status, 400, + "losing the race should not read as success: {body}" + ); assert!( body.contains("deployed to concurrently"), "the loser must say it was superseded, not that the script is missing: {body}" diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index bec59ac45a..65d44509d7 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -917,3 +917,79 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a Ok(()) } + +/// An address with no `password` row can own a draft, and the account paths carry the delete and +/// rename that no foreign key does any more. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_drafts_follow_their_owner_without_a_fkey(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/users"); + + // The destination of the rename below already holds a draft of the same item — it belongs to + // an accountless principal, so `change_email`'s "address is free" check does not see it. + sqlx::query!( + "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'), + ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')" + ) + .execute(&db) + .await?; + + // A null username is how the legacy workspace-level row is encoded, so an owner nobody can + // name must be absent from the owner circles rather than pose as one. + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list?all_users=true" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let listed = resp.json::().await?; + let ext = listed + .as_array() + .unwrap() + .iter() + .find(|d| d["path"] == "u/ext/s") + .expect("the accountless owner's draft is listed"); + assert_eq!(ext.get("draft_users"), None); + + let resp = authed(client().post(format!("{global_base}/change_email/test2@windmill.dev"))) + .json(&json!({ "new_email": "renamed@windmill.dev" })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "change_email: {}", resp.text().await?); + let moved = sqlx::query!( + "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'" + ) + .fetch_all(&db) + .await?; + assert_eq!( + moved + .iter() + .map(|r| (r.email.as_deref(), r.summary.as_deref())) + .collect::>(), + vec![(Some("renamed@windmill.dev"), Some("moving"))], + "the moving account's draft wins the unique index it now collides on" + ); + + let resp = authed(client().delete(format!("{global_base}/delete/test3@windmill.dev"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete_user: {}", resp.text().await?); + let remaining = sqlx::query_scalar!("SELECT path FROM draft ORDER BY path") + .fetch_all(&db) + .await?; + assert_eq!( + remaining, + vec!["u/ext/s".to_string(), "u/two/s".to_string()], + "the deleted account's draft goes, the accountless owner's stays" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 65fe4241cc..25312cae27 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -889,6 +889,53 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +/// A workspace with no provider of its own is served the instance config, but the +/// `copilot_disabled` flag must still come from the workspace's own row. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_get_copilot_info_keeps_workspace_copilot_disabled_over_instance_fallback( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + sqlx::query("UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2") + .bind(json!({ "copilot_disabled": true })) + .bind("test-workspace") + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind("ai_config") + .bind(json!({ + "providers": { + "openai": { + "resource_path": "u/test-user/openai_instance", + "models": ["gpt-4o-mini"] + } + } + })) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{base}/get_copilot_info"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert_eq!( + settings["providers"]["openai"]["models"][0], "gpt-4o-mini", + "instance providers are still served" + ); + assert_eq!(settings["copilot_disabled"], true); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -941,7 +988,12 @@ async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyh .send() .await .unwrap(); - assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert_eq!( + resp.status(), + 200, + "disable on fork: {}", + resp.text().await? + ); assert!(!stored().await?); Ok(()) @@ -1044,9 +1096,11 @@ async fn test_create_service_account_drops_orphaned_group_memberships( .await?; // Same username, different workspace, and very much alive — must not be touched. - sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')") - .execute(&db) - .await?; + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')", + ) + .execute(&db) + .await?; sqlx::query( "INSERT INTO group_ (workspace_id, name, summary) VALUES ('other-workspace', 'all', 'All users'), diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index fbfd656cdf..e897bf5eaf 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -416,11 +416,31 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result let mut headers = HeaderMap::new(); + // A reverse proxy consumes hop-by-hop headers instead of forwarding them and + // drops every header named by `Connection`, so a script could use one to strip + // the sandbox headers this function adds before they reach the browser. + const HOP_BY_HOP_HEADERS: [&str; 9] = [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ]; + if let Some(windmill_headers) = windmill_headers { for (k, v) in windmill_headers { let k = HeaderName::from_str(k.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header name {k}: {err}")) })?; + if HOP_BY_HOP_HEADERS.contains(&k.as_str()) { + return Err(Error::ExecutionErr(format!( + "windmill_headers cannot set the hop-by-hop header \"{k}\"" + ))); + } let v = HeaderValue::from_str(v.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header value {v}: {err}")) })?; @@ -428,6 +448,22 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result } } + // The script controls the content type and body, and run_wait_result and sync + // HTTP routes are reachable by top-level GET navigation with the session cookie: + // sandbox the document into an opaque origin so HTML can never run with the + // viewer's session. Inserted after `wm_headers` so a script cannot override it. + headers.insert( + http::header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + headers.insert( + http::header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static( + "sandbox allow-scripts allow-forms allow-popups \ + allow-popups-to-escape-sandbox allow-downloads allow-modals", + ), + ); + if let Some(content_type) = windmill_content_type { let serialized_json_result = result_value .map(|val| val.get().to_owned()) @@ -1104,6 +1140,56 @@ mod result_to_response_tests { resp.headers().get(http::header::CONTENT_TYPE).unwrap(), "text/html" ); + assert_sandboxed(resp.headers()); assert_eq!(body_bytes(resp).await, b"

hi

"); } + + fn assert_sandboxed(headers: &HeaderMap) { + assert_eq!( + headers.get(http::header::X_CONTENT_TYPE_OPTIONS).unwrap(), + "nosniff" + ); + let csp = headers + .get(http::header::CONTENT_SECURITY_POLICY) + .expect("content-security-policy") + .to_str() + .unwrap(); + assert!(csp.starts_with("sandbox "), "csp: {csp}"); + assert!(!csp.contains("allow-same-origin"), "csp: {csp}"); + } + + #[tokio::test] + async fn custom_headers_cannot_override_sandbox() { + // wm_headers is script-controlled: a content-type set there replaces the JSON + // one even without wm_content_type, and the sandbox headers must survive an + // attempt to override them. + let resp = result_to_response( + raw( + r#"{"wm_headers":{"content-type":"text/html","content-security-policy":"default-src *","x-content-type-options":"none"},"result":"

hi

"}"#, + ), + true, + ) + .expect("response"); + + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html" + ); + assert_sandboxed(resp.headers()); + } + + #[tokio::test] + async fn hop_by_hop_custom_headers_are_rejected() { + // A proxy drops every header named by `Connection`, which would strip the + // sandbox headers on the way to the browser. + for name in ["connection", "Connection", "transfer-encoding", "upgrade"] { + let res = result_to_response( + raw(&format!( + r#"{{"wm_content_type":"text/html","wm_headers":{{"{name}":"content-security-policy, x-content-type-options"}},"result":"

hi

"}}"# + )), + true, + ); + assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); + } + } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 79c6c99b9b..ccc5815aa7 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,7 +39,7 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::process_relative_imports; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ @@ -216,12 +216,15 @@ async fn list_scripts( // a member of has no `usr` row, so fall back to their instance-derived username // (`password.username`), or their email when derivation is disabled — this keeps the // raw email out of the payload whenever a derived username exists. The genuine - // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL). + // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL), + // which is why an owner that resolves to no name at all — an external JWT's subject + // has neither row — is dropped: None is read as "legacy" downstream. "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() @@ -1070,6 +1073,14 @@ fn modules_eq( } } +/// Recorded for the empty lock a codebase or a language with no lock generation carries as well as +/// for a real one: the worker writes `hash_script("")` in the same situation, and a path going from +/// a real lock to an empty one has to stop matching what its importers recorded, or they wrongly +/// skip rather than merely relock too often. +fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { + [(path.to_string(), hash_script(lock))] +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1337,6 +1348,12 @@ async fn create_script_internal<'c>( parent_hash = %p_hash.0, "Skipping no-op script deploy (identical to parent)" ); + // The version is unchanged, but the row recording its lock's hash may never have + // been written — nothing else writes it for a supplied lock, and a path only ever + // pushed unchanged would otherwise keep its importers relocking forever. + if let Some(lock) = ps.lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } return Ok((p_hash.clone(), tx, None, Vec::new())); } @@ -1884,6 +1901,13 @@ async fn create_script_internal<'c>( .execute(&mut *tx) .await?; + // A lock that is not left to a dependency job queues none, so this is the only place its hash + // can be recorded. `try_skip_relock` treats a missing hash for an imported script as changed, + // so leaving the row out makes every importer of this path relock on every deploy of it. + if let Some(lock) = lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } + // Update ci_test_reference table for test scripts // Delete by both new and old path to handle renames let old_path = parent_hashes_and_perms.as_ref().map(|x| x.p_path.as_str()); diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index cead2f4b27..18a8d52ac0 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -284,15 +284,28 @@ pub async fn test_s3_bucket( use bytes::Bytes; use futures::StreamExt; - // The probe executes on the API server itself. On multi-tenant Cloud that is a shared control - // plane, so we constrain untrusted callers to remove the SSRF / credential-exfiltration / - // local-filesystem surface (see validate_object_storage_test). On self-hosted instances the - // object store usually lives on the local/private network and all authenticated users are - // trusted, so testing there stays unrestricted. Super admins keep the unrestricted path too. + // The probe executes on the API server itself and reflects the upstream response into the + // error, so any authenticated caller could otherwise use it as an SSRF / port-scan primitive + // against the server's network, exfiltrate its ambient credentials, or write to its local + // disk (see validate_object_storage_test). That holds on self-hosted instances as much as on + // Cloud, so only super admins get the unrestricted path. let is_super_admin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; - let restrict = !is_super_admin && *CLOUD_HOSTED; + let restrict = !is_super_admin; if restrict { - validate_object_storage_test(&test_s3_bucket).await?; + validate_object_storage_test(&test_s3_bucket) + .await + .map_err(|e| match e { + // A job token never counts as a super admin (it is capped at workspace admin), so + // a super admin calling this route from a script is told why rather than that + // they lack a privilege they hold. + error::Error::NotAuthorized(msg) if authed.job_id.is_some() => { + error::Error::NotAuthorized(format!( + "{msg} A job token ($WM_TOKEN) is never treated as a super admin; call \ + this route with a user token instead." + )) + } + e => e, + })?; } let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) @@ -355,8 +368,8 @@ pub async fn test_s3_bucket( } } -// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller on -// Cloud. The probe runs on the shared API server, so without these constraints an authenticated +// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller. +// The probe runs on the API server, so without these constraints an authenticated // user could coerce the server into connecting to arbitrary internal endpoints (SSRF), signing // requests with the instance role (credential exfiltration), or reading/writing the server's local // disk (filesystem object store). @@ -366,6 +379,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul opt.as_ref().is_some_and(|s| !s.is_empty()) } + // Every refusal names the way out: the resource usually works in jobs (workers reach the + // endpoint directly), so without it the refusal reads as a broken resource. + const ALTERNATIVE: &str = + "Ask a super admin to run it, or test the resource from a script, which runs on a worker."; + // Reject backends that rely on the server's identity or local filesystem, require explicit // credentials for the rest (so the server never falls back to its own ambient credentials), and // resolve the host the client will actually connect to. We derive the *effective* endpoint here @@ -376,20 +394,25 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul let effective_endpoint: Option = match settings { ObjectSettings::Filesystem(_) => { return Err(error::Error::NotAuthorized( - "Testing a local filesystem object store requires a super admin".to_string(), + "Testing a local filesystem object store requires a super admin: it runs on the \ + Windmill server and reads and writes the server's local disk. Ask a super admin \ + to run it." + .to_string(), )); } ObjectSettings::AwsOidc(_) => { - return Err(error::Error::NotAuthorized( - "Testing OIDC-based object storage requires a super admin".to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing OIDC-based object storage requires a super admin: it runs on the \ + Windmill server with the server's own identity. {ALTERNATIVE}" + ))); } ObjectSettings::S3(s3) => { if !(non_empty(&s3.access_key) && non_empty(&s3.secret_key)) { - return Err(error::Error::NotAuthorized( - "Testing S3 storage without explicit credentials requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing S3 storage without an explicit access key and secret key requires a \ + super admin: it runs on the Windmill server, which would use its own ambient \ + credentials. {ALTERNATIVE}" + ))); } let region = s3 .region @@ -413,10 +436,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul } ObjectSettings::Azure(azure) => { if !non_empty(&azure.access_key) { - return Err(error::Error::NotAuthorized( - "Testing Azure storage without an explicit access key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing Azure storage without an explicit access key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } Some( azure @@ -432,10 +456,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul // otherwise an untrusted caller could probe with the server's identity (the very // SSRF/credential-exfil this function guards against). if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { - return Err(error::Error::NotAuthorized( - "Testing GCS storage without a service account key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing GCS storage without a service account key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } // The service-account-key JSON can override the data-plane URL (`gcs_base_url`) and the // OAuth token endpoint (`token_uri`); the GCS client connects to whatever they point at. @@ -492,10 +517,15 @@ async fn validate_public_endpoint(endpoint: &str) -> error::Result<()> { // attempts (a name resolving to both a public and a private address). for addr in addrs { if is_forbidden_ip(addr.ip()) { - return Err(error::Error::NotAuthorized( - "Testing object storage at a private, loopback, or link-local endpoint requires a super admin" - .to_string(), - )); + // The resolved address stays out of the message: it is the server's resolver's + // answer, and this message is only ever shown to the caller being constrained. + return Err(error::Error::NotAuthorized(format!( + "Testing object storage at '{host}', which resolves to a private, loopback, or \ + link-local address, requires a super admin: this test runs on the Windmill \ + server, which is not allowed to probe internal addresses for non-super-admins. \ + Ask a super admin to run it, or test the resource from a script, which runs on \ + a worker." + ))); } } Ok(()) @@ -2004,6 +2034,15 @@ struct CachedResourceType { #[allow(dead_code)] app: String, description: Option, + /// Doubly optional, and read through a wrapping deserializer: this struct also + /// decodes the on-disk cache, where an absent key means "written before the + /// column, leave the stored extension alone" and an explicit null means the hub + /// dropped it. Plain serde folds both into `None`. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + format_extension: Option>, } #[derive(serde::Deserialize)] @@ -2013,6 +2052,8 @@ struct HubResourceTypeRaw { schema: Option, app: String, description: Option, + #[serde(default)] + format_extension: Option, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2054,6 +2095,7 @@ async fn fetch_resource_types_from_hub() -> error::Result = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), ) .fetch_one(&db) .await?; @@ -2120,13 +2164,27 @@ async fn sync_cached_resource_types( } sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // Whether the payload carried the key at all is what decides: present + // (even as null) is authoritative and may clear, absent means a cache + // written before the column and must leave the stored value alone. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) + VALUES ('admins', $1, $2, $3, $4, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + -- A fileset is a set of files, so it cannot also be one file. + -- Create and update reject the pair; this writer bypasses both, so + -- it declines the extension rather than persisting the forbidden + -- combination onto a same-named local fileset. + format_extension = CASE + WHEN resource_type.is_fileset THEN NULL + WHEN $5 THEN EXCLUDED.format_extension + ELSE resource_type.format_extension END, + edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), ) .execute(&db) .await?; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8c3a86746e..55aeb56219 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1239,6 +1239,7 @@ async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Resu sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &authed.email).await?; audit_log( &mut *tx, @@ -1661,6 +1662,7 @@ async fn delete_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?; // Read before deleting, so each membership's tenant can go first. A username // is scoped to one workspace, and so are the tenants naming it. @@ -1889,7 +1891,7 @@ async fn change_user_email( .execute(&mut *tx) .await?; - // ---- account ---- (draft.email follows through its ON UPDATE CASCADE fkey) + // ---- account ---- sqlx::query!( "UPDATE password SET email = $1 WHERE email = $2", &new_email, @@ -1903,6 +1905,7 @@ async fn change_user_email( } _ => e.into(), })?; + windmill_common::user_drafts::rename_drafts_of_email(&mut *tx, &old_email, &new_email).await?; sqlx::query!( "UPDATE usr SET email = $1 WHERE email = $2", @@ -3580,6 +3583,9 @@ async fn overwrite_global_users( require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; let mut tx = db.begin().await?; + // Replaces the account table, so — unlike the paths that remove one account — it deliberately + // does not call `delete_drafts_of_email`: the addresses are about to be reinstated, and + // dropping every draft on the instance to restore accounts would be pure collateral. sqlx::query!("DELETE FROM password") .execute(&mut *tx) .await?; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 72aa93d266..4b4a593a64 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,6 +12,7 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; +use windmill_dep_map::lock_hash::record_lock_hashes_for_workspace; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; @@ -6097,9 +6098,8 @@ async fn clone_workspace_data( // Clone the forker's own per-user drafts (plus the legacy NULL-email // workspace draft, if any) so they keep their pending edits in the // fork. Other users' drafts are intentionally NOT cloned — they don't - // own a `usr` row in the fork (see `clone_workspace_full`) so their - // drafts would dangle and the home-page `draft_users` aggregate would - // surface them as duplicate legacy entries. + // own a `usr` row in the fork (see `clone_workspace_full`), so those + // drafts would belong to someone the fork holds no membership for. clone_drafts(tx, source_workspace_id, target_workspace_id, &authed.email).await?; // Clone workspace runnable dependencies and dependency map @@ -7438,7 +7438,16 @@ async fn clone_workspace_runnable_dependencies( .execute(&mut **tx) .await?; - // Clone dependency_map to preserve import relationships + // Recorded so the clone's own relocks have something to match; with no row they record NULL + // and nothing in it ever skips. Hashed from the locks the clone holds rather than copied from + // the source's rows, which are only as current as the last write to them: one left stale by a + // supplied lock deployed before this was recorded names a lock the clone no longer has, and an + // importer that resolved against the real one would then skip a relock it needed. + record_lock_hashes_for_workspace(tx, target_workspace_id).await?; + + // Deliberately without `imported_lockfile_hash`: it records what an importer resolved against + // when it was last locked, which nothing here can establish for the version the clone got. + // Left NULL, every importer relocks once and re-anchors both sides to what the clone holds. sqlx::query!( "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) SELECT $1, importer_path, importer_kind, imported_path, importer_node_id diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 92eeaeaedb..3cba2b7da6 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -16532,202 +16532,6 @@ } } }, - "/w/{workspace}/ai_skills/list": { - "get": { - "summary": "list the workspace AI chat skills (name + description only)", - "operationId": "listAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "responses": { - "200": { - "description": "skill listing", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "required": [ - "name", - "description" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - } - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/get/{name}": { - "get": { - "summary": "get a workspace AI chat skill including its instructions", - "operationId": "getAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "skill", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "instructions": { - "type": "string" - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/upload": { - "post": { - "summary": "upsert workspace AI chat skills (admin only)", - "operationId": "uploadAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "skills" - ], - "properties": { - "skills": { - "type": "array", - "maxItems": 50, - "items": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9-]+$" - }, - "description": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "instructions": { - "type": "string", - "minLength": 1, - "maxLength": 65536 - } - } - } - } - } - } - } - } - }, - "responses": { - "200": { - "description": "uploaded", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/delete/{name}": { - "delete": { - "summary": "delete a workspace AI chat skill (admin only)", - "operationId": "deleteAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "deleted", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, "/w/{workspace}/apps/get_data/v/{secretWithExtension}": { "get": { "summary": "get raw app data by", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 235c80ccf9..d5a14a717d 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -17063,141 +17063,6 @@ paths: text/plain: schema: type: string - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - responses: - '200': - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: ^[a-z0-9-]+$ - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - '200': - description: uploaded - content: - text/plain: - schema: - type: string - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: deleted - content: - text/plain: - schema: - type: string /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fa0e90a3fb..a63c6ba63d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.800.0 + version: 1.803.0 title: Windmill API contact: @@ -10689,7 +10689,7 @@ paths: summary: run script by path operationId: runScriptByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -12767,133 +12767,6 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - responses: - "200": - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: "^[a-z0-9-]+$" - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - "200": - description: uploaded - content: - text/plain: - schema: - type: string - - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: deleted - content: - text/plain: - schema: - type: string - /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -14174,7 +14047,7 @@ paths: summary: run flow by path operationId: runFlowByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -27495,6 +27368,13 @@ components: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + copilot_disabled: + type: boolean + description: >- + Hides the Windmill AI assistant (chat, sessions, code generation, completion, + fixes) from the workspace UI. Read from the workspace's own settings even when + the providers served fall back to the instance config. AI agent steps and the + AI sandbox in flows are unaffected. FreeTierInfo: type: object @@ -29347,6 +29227,8 @@ components: - "igroup.delete" - "igroup.adduser" - "igroup.removeuser" + - "instance_groups.jit_adduser" + - "instance_groups.jit_removeuser" - "variables.decrypt_secret" - "workspaces.read_encryption_key" - "workspaces.edit_command_script" @@ -29845,6 +29727,12 @@ components: type: string is_fileset: type: boolean + format_extension: + type: string + nullable: true + description: >- + File extension for a type whose value is one file rather than a set + of fields. Omit to leave it unchanged; send null to clear it. TriggerHistoryEntry: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index c684830b27..e70cc19544 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -445,6 +445,12 @@ pub struct AIConfig { /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] pub model_pricing: Option>, + /// Hides the Windmill AI assistant (chat, sessions, generation, completion, fixes) from + /// the workspace UI. Only the workspace's own row is consulted: the flag holds even when + /// the providers served come from the instance config or the free tier. AI agent steps + /// and the AI sandbox are unaffected, so the providers stay in force. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub copilot_disabled: bool, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the diff --git a/backend/windmill-api/src/ai_skills.rs b/backend/windmill-api/src/ai_skills.rs deleted file mode 100644 index 11d99d55da..0000000000 --- a/backend/windmill-api/src/ai_skills.rs +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2026 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::db::{ApiAuthed, DB}; -use axum::{ - extract::{Extension, Json, Path}, - routing::{delete, get, post}, - Router, -}; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::require_admin, -}; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_skills)) - .route("/get/{name}", get(get_skill)) - .route("/upload", post(upload_skills)) - .route("/delete/{name}", delete(delete_skill)) -} - -/// Cheap listing surfaced in the AI chat system prompt — no `instructions` body. -#[derive(Serialize)] -pub struct SkillListItem { - pub name: String, - pub description: String, -} - -/// Full skill, including the SKILL.md body, fetched on demand by `read_skill`. -#[derive(Serialize)] -pub struct Skill { - pub name: String, - pub description: String, - pub instructions: String, -} - -#[derive(Deserialize)] -pub struct UploadSkills { - pub skills: Vec, -} - -#[derive(Deserialize)] -pub struct SkillUpload { - pub name: String, - pub description: String, - pub instructions: String, -} - -const MAX_SKILLS_PER_UPLOAD: usize = 50; -// Every stored skill's name + description is advertised in the global AI chat -// system prompt, so bound the total a workspace can accumulate across uploads. -const MAX_SKILLS_PER_WORKSPACE: usize = 100; -// `name` and `description` follow the Claude SKILL.md spec -// (https://platform.claude.com/docs/en/agents-and-tools/agent-skills): both are -// loaded into the AI chat system prompt and `name` is the model-facing skill id, -// so matching the upstream limits keeps skills portable with Claude Code. -const MAX_SKILL_NAME_CHARS: usize = 64; -const MAX_SKILL_DESCRIPTION_CHARS: usize = 1_024; -// Not a spec field — a payload bound on the SKILL.md body, so measured in bytes. -const MAX_SKILL_INSTRUCTIONS_BYTES: usize = 64 * 1024; - -fn validate_skill(skill: &SkillUpload) -> Result<()> { - let name = skill.name.trim(); - if name.is_empty() || name.chars().count() > MAX_SKILL_NAME_CHARS { - return Err(Error::BadRequest(format!( - "skill name must be between 1 and {MAX_SKILL_NAME_CHARS} characters, got {:?}", - skill.name - ))); - } - if !name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - return Err(Error::BadRequest(format!( - "skill name {name:?} must only contain lowercase letters, digits or '-'" - ))); - } - if skill.description.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} is missing a description (the SKILL.md frontmatter `description`)" - ))); - } - if skill.description.chars().count() > MAX_SKILL_DESCRIPTION_CHARS { - return Err(Error::BadRequest(format!( - "skill {name:?} description must be at most {MAX_SKILL_DESCRIPTION_CHARS} characters" - ))); - } - if skill.instructions.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} has an empty SKILL.md body" - ))); - } - if skill.instructions.len() > MAX_SKILL_INSTRUCTIONS_BYTES { - return Err(Error::BadRequest(format!( - "skill {name:?} instructions must be at most {MAX_SKILL_INSTRUCTIONS_BYTES} bytes" - ))); - } - Ok(()) -} - -/// Collect the trimmed skill names, rejecting duplicates within a single upload. -/// The insert upserts by name, so a duplicate would silently keep only the last -/// and make the reported/audited count wrong. -fn collect_upload_names(skills: &[SkillUpload]) -> Result> { - let mut names = Vec::with_capacity(skills.len()); - let mut seen = HashSet::with_capacity(skills.len()); - for skill in skills { - let name = skill.name.trim().to_string(); - if !seen.insert(name.clone()) { - return Err(Error::BadRequest(format!( - "duplicate skill name {name:?} in upload" - ))); - } - names.push(name); - } - Ok(names) -} - -/// Reject an upload that would push the workspace past `MAX_SKILLS_PER_WORKSPACE`. -/// Uploads upsert, so names already present (`replacing`) don't count as new. -fn check_workspace_skill_capacity( - existing_total: i64, - replacing: i64, - upload_count: usize, -) -> Result<()> { - let new_count = upload_count as i64 - replacing; - if existing_total + new_count > MAX_SKILLS_PER_WORKSPACE as i64 { - return Err(Error::BadRequest(format!( - "workspace cannot store more than {MAX_SKILLS_PER_WORKSPACE} skills" - ))); - } - Ok(()) -} - -async fn list_skills( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query!( - "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - &w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json( - rows.into_iter() - .map(|r| SkillListItem { name: r.name, description: r.description }) - .collect(), - )) -} - -async fn get_skill( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let row = sqlx::query!( - "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - row.map(|r| { - Json(Skill { name: r.name, description: r.description, instructions: r.instructions }) - }) - .ok_or_else(|| Error::NotFound(format!("no skill named {name:?} in workspace {w_id}"))) -} - -/// Bulk upsert the uploaded skills by name. Existing skills not in the payload -/// are left untouched — removal goes through `delete_skill`. -async fn upload_skills( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(payload): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - if payload.skills.is_empty() { - return Err(Error::BadRequest("no skills to upload".to_string())); - } - if payload.skills.len() > MAX_SKILLS_PER_UPLOAD { - return Err(Error::BadRequest(format!( - "cannot upload more than {MAX_SKILLS_PER_UPLOAD} skills at a time" - ))); - } - for skill in &payload.skills { - validate_skill(skill)?; - } - let names = collect_upload_names(&payload.skills)?; - - let mut tx = db.begin().await?; - let counts = sqlx::query!( - r#"SELECT - COUNT(*)::bigint AS "total!", - COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS "replacing!" - FROM ai_skill - WHERE workspace_id = $1"#, - &w_id, - &names - ) - .fetch_one(&mut *tx) - .await?; - check_workspace_skill_capacity(counts.total, counts.replacing, names.len())?; - - for (skill, name) in payload.skills.iter().zip(names.iter()) { - sqlx::query!( - r#"INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by) - VALUES ($1, $2, $3, $4, now(), $5) - ON CONFLICT (workspace_id, name) DO UPDATE - SET description = EXCLUDED.description, - instructions = EXCLUDED.instructions, - edited_at = now(), - edited_by = EXCLUDED.edited_by"#, - &w_id, - name, - skill.description, - skill.instructions, - &authed.username, - ) - .execute(&mut *tx) - .await?; - } - - let audit_resource = names.join(","); - audit_log( - &mut *tx, - &authed, - "ai_skills.upload", - ActionKind::Update, - &w_id, - Some(&audit_resource), - Some([("skill_count", &names.len().to_string()[..])].into()), - ) - .await?; - tx.commit().await?; - - Ok(format!( - "Uploaded {} skill(s) to workspace {}", - payload.skills.len(), - &w_id - )) -} - -async fn delete_skill( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - let deleted = sqlx::query_scalar!( - "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - - if deleted.is_none() { - tx.commit().await?; - return Err(Error::NotFound(format!( - "no skill named {name:?} in workspace {w_id}" - ))); - } - - audit_log( - &mut *tx, - &authed, - "ai_skills.delete", - ActionKind::Delete, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Deleted skill {name} from workspace {w_id}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn skill() -> SkillUpload { - SkillUpload { - name: "test-skill".to_string(), - description: "Useful for tests".to_string(), - instructions: "# Test\n\nDo the thing.".to_string(), - } - } - - #[test] - fn validate_skill_rejects_oversized_description() { - let mut skill = skill(); - skill.description = "x".repeat(MAX_SKILL_DESCRIPTION_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_instructions() { - let mut skill = skill(); - skill.instructions = "x".repeat(MAX_SKILL_INSTRUCTIONS_BYTES + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_name() { - let mut skill = skill(); - skill.name = "a".repeat(MAX_SKILL_NAME_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_non_slug_name() { - // Uppercase, underscore, space and punctuation are all outside the - // Claude SKILL.md `[a-z0-9-]` name charset. - for bad in ["My-Skill", "my_skill", "my skill", "skill!"] { - let mut skill = skill(); - skill.name = bad.to_string(); - - assert!( - matches!(validate_skill(&skill), Err(Error::BadRequest(_))), - "{bad:?} should be rejected" - ); - } - } - - #[test] - fn validate_skill_counts_description_in_characters() { - // 1024 two-byte chars exceed the byte limit but sit exactly on the - // character limit, so they must be accepted. - let mut skill = skill(); - skill.description = "é".repeat(MAX_SKILL_DESCRIPTION_CHARS); - - assert!(validate_skill(&skill).is_ok()); - } - - #[test] - fn workspace_capacity_allows_replacement_at_cap() { - // Already at the cap, but the upload only replaces an existing skill. - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(check_workspace_skill_capacity(at_cap, 1, 1).is_ok()); - } - - #[test] - fn workspace_capacity_rejects_new_skill_over_cap() { - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(matches!( - check_workspace_skill_capacity(at_cap, 0, 1), - Err(Error::BadRequest(_)) - )); - } - - #[test] - fn collect_upload_names_trims_and_collects() { - let names = collect_upload_names(&[skill()]).unwrap(); - assert_eq!(names, vec!["test-skill".to_string()]); - } - - #[test] - fn collect_upload_names_rejects_duplicates() { - // Names are compared after trimming, so whitespace can't smuggle a dup in. - let dup = SkillUpload { name: " test-skill ".to_string(), ..skill() }; - assert!(matches!( - collect_upload_names(&[skill(), dup]), - Err(Error::BadRequest(_)) - )); - } -} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 7bc995844d..0d29c1c369 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -491,7 +491,8 @@ async fn list_apps( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app')) as draft_users", + WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app') \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(app.workspace_id, app.path) as inherited_labels", ]) .left() diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index b44b0e64be..e5928f761e 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -214,6 +214,9 @@ fn list_drafts_query(all_users: bool) -> String { // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled (`password.username` is NULL). This // keeps the raw email out of the payload whenever a derived username exists. + // A null username means the legacy row downstream, so an owner that resolves to + // no name at all — an external JWT's subject has neither row — is dropped rather + // than surfaced as a second legacy entry. let draft_users = r#"CASE WHEN d.typ::text IN ('script', 'flow', 'app', 'raw_app') THEN ( SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END) NULLS LAST) @@ -221,6 +224,7 @@ fn list_drafts_query(all_users: bool) -> String { LEFT JOIN usr u ON u.workspace_id = du.workspace_id AND u.email = du.email LEFT JOIN password p ON p.email = du.email AND p.super_admin = true WHERE du.workspace_id = d.workspace_id AND du.path = d.path AND du.typ = d.typ + AND (du.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ) ELSE NULL END"#; // Default lists the user's own drafts AND the legacy NULL-email rows; with // `all_users` the filter is dropped to list every workspace draft. diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a30cffbfbf..53044ee020 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -18,6 +18,7 @@ use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; use sha2::{Digest, Sha256}; +use std::borrow::Cow; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; @@ -108,6 +109,7 @@ use windmill_common::{ flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, + query_builders, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, @@ -8131,6 +8133,79 @@ pub async fn run_wait_result_flow_by_version( .await } +/// Whether request-supplied SQL from an operator may run. Operators can only run deployed +/// code, so a request their job token (`WM_TOKEN`) authenticates comes from code a +/// non-operator authored. The job must still be running, and the request must have the +/// shape `wmill.datatable()` sends (PostgreSQL against a `datatable://` database), so a +/// WM_TOKEN that leaked into job logs cannot be replayed to reach another target while the +/// job lives, in particular DuckDB, which runs in-process in the worker. +/// +/// What it does permit is any statement against the workspace's data tables, writes and DDL +/// included: the helper's body is an unrestricted SQL template and data tables carry no +/// per-user ACL. Narrowing that is a separate decision from this exemption. +/// +/// The database argument is only half the target: the executor honors a `-- database` +/// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused. +/// Check them against the code the executor runs rather than the request's `content`, which +/// is not the same string once a `WM_INTERNAL_DB` marker expands. +async fn operator_may_run_datatable_query( + db: &DB, + w_id: &str, + job_id: Option, + language: Option<&ScriptLang>, + content: &str, + args: Option<&HashMap>>, +) -> error::Result { + let Some(job_id) = job_id else { + return Ok(false); + }; + if language != Some(&ScriptLang::Postgresql) { + return Ok(false); + } + // Parse the directives out of the code the executor actually runs: it expands a + // `WM_INTERNAL_DB` marker first, and a directive can be embedded in the expansion. + // An expansion that overrides the language would run something other than the SQL the + // language check above cleared, so it is refused along with a malformed marker. + let executed = + match query_builders::try_expand_internal_db_query(content, &ScriptLang::Postgresql) { + Some(Ok(expanded)) if expanded.language_override.is_none() => Cow::Owned(expanded.code), + Some(_) => return Ok(false), + None => Cow::Borrowed(content), + }; + if windmill_parser_sql::parse_db_resource(&executed).is_some() + || !matches!(windmill_parser_sql::parse_s3_mode(&executed), Ok(None)) + { + return Ok(false); + } + let targets_datatable = args + .and_then(|args| args.get("database")) + .and_then(|database| serde_json::from_str::(database.get()).ok()) + .is_some_and(|database| database.starts_with("datatable://")); + if !targets_datatable { + return Ok(false); + } + Ok(sqlx::query_scalar!( + "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} + +/// The refusal an operator gets from a preview route. Inside a job the caller never ran a +/// preview themselves, so name the one thing the job's token may do. +fn operator_preview_refusal(job_id: Option) -> error::Error { + let reason = if job_id.is_some() { + "Operators cannot run preview jobs for security reasons: from inside a job, an \ + operator may only run a wmill.datatable() query while that job is running" + } else { + "Operators cannot run preview jobs for security reasons" + }; + error::Error::NotAuthorized(reason.to_string()) +} + async fn run_preview_script( authed: ApiAuthed, Extension(db): Extension, @@ -8142,9 +8217,20 @@ async fn run_preview_script( #[cfg(feature = "enterprise")] check_license_key_valid().await?; if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // A deferred run would outlive the running job the exemption keys off. + if run_query.get_scheduled_for(&db).await?.is_some() + || !operator_may_run_datatable_query( + &db, + &w_id, + authed.job_id, + preview.language.as_ref(), + preview.content.as_deref().unwrap_or_default(), + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(authed.job_id)); + } } // Preview runs arbitrary, request-supplied code. require_path_read_access_for_preview // only checks folder/namespace *read* access (and is a no-op when path is null), so a @@ -8239,13 +8325,21 @@ async fn run_inline_preview_script( Path(w_id): Path, Json(preview): Json, ) -> error::Result { - // Same arbitrary-code class as run_preview_script: operators are blocked from - // running request-supplied code, and a narrowly-scoped token must not escape - // its scope through inline preview. - if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // Same arbitrary-code class as run_preview_script, and every worker and standalone + // server exposes this route, so an operator is refused on the same terms. A + // narrowly-scoped token must not escape its scope through inline preview either. + if authed.is_operator + && !operator_may_run_datatable_query( + &db, + &w_id, + job_id, + Some(&preview.language), + &preview.content, + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(job_id)); } check_scopes(&authed, || format!("jobs:run"))?; if let Some(job_id) = job_id { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fc98020d1b..054af0f2fa 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,7 +69,6 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; -mod ai_skills; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; @@ -453,15 +452,15 @@ pub async fn run_server( // unless they are allowed — hence a separate layer rather than widening the // one every other route shares. (`Mcp-Param-*` is only sent for tool inputs // annotated with `x-mcp-header`, which no tool here declares.) + // + // The request's own header list is mirrored rather than enumerated: a browser + // MCP client may send any custom name for a preprocessor to read, and no fixed + // list could cover them. Nothing is granted by echoing it: the origin is + // `Any`, so browsers never attach credentials, and the endpoint authenticates + // each request on its own. let mcp_cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST, http::Method::DELETE]) - .allow_headers([ - http::header::CONTENT_TYPE, - http::header::AUTHORIZATION, - http::HeaderName::from_static("mcp-protocol-version"), - http::HeaderName::from_static("mcp-method"), - http::HeaderName::from_static("mcp-name"), - ]) + .allow_headers(tower_http::cors::AllowHeaders::mirror_request()) // The 401 challenge is how a client discovers where to authorize (RFC 9728), // and it is not a safelisted response header, so without this a browser // client sees an empty one and has no way to begin the OAuth flow. @@ -713,7 +712,6 @@ pub async fn run_server( Router::new() }) .nest("/ai", ai::workspaced_service()) - .nest("/ai_skills", ai_skills::workspaced_service()) .nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service()) .nest( "/path_autocomplete", diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 568ad854f9..aa2415f664 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -878,7 +878,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runScriptByPath"), description: Cow::Borrowed("run script by path"), - instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/p/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ @@ -1419,7 +1419,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runFlowByPath"), description: Cow::Borrowed("run flow by path"), - instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/f/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 2b4350bcfe..480e3c0841 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -12,7 +12,9 @@ use windmill_mcp::common::transform::transform_property_keys; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend, PathFilter}; +use windmill_mcp::server::{ + BackendResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter, +}; use crate::auth::AuthCache; use crate::db::ApiAuthed; @@ -214,8 +216,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, false, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_script_by_path_internal( self.db.clone(), @@ -238,8 +243,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, true, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_flow_by_path_internal( self.db.clone(), diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 94271bb662..65c8069b85 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -11,15 +11,19 @@ use serde_json::Value; use sql_builder::prelude::*; use windmill_common::auth::create_jwt_token; use windmill_common::db::{Authed, UserDB}; +use windmill_common::error::Error; use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; +use windmill_common::triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}; use windmill_common::utils::{query_elems_from_hub, StripPath}; use windmill_common::worker::to_raw_value; use windmill_common::{DB, HUB_BASE_URL}; use windmill_mcp::server::{ - non_empty_body_fields, BackendResult, EndpointTool, ErrorData, PathFilter, + non_empty_body_fields, BackendResult, EndpointTool, ErrorData, McpRequest, PathFilter, }; use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; +use windmill_trigger::trigger_helpers::{get_runnable_format, RunnableId}; +use crate::args::build_headers; use crate::db::ApiAuthed; use crate::HTTP_CLIENT; @@ -641,7 +645,7 @@ fn selects_endpoint_tool(caller_scopes: &[String], tool: &str) -> bool { .is_ok_and(|config| config.endpoints.iter().any(|e| e == tool)) } -/// Create HTTP request with authentication +/// Create HTTP request with authentication. pub async fn create_http_request( method: &str, url: &str, @@ -702,17 +706,113 @@ pub async fn create_http_request( .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) } -/// Convert a JSON Value into PushArgsOwned for job execution -pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned { +/// The `kind` an MCP-invoked runnable sees on its preprocessor event, alongside +/// `webhook`, `http` and the trigger kinds. +const MCP_TRIGGER_KEY: &str = "mcp"; + +/// A preprocessor's view of the MCP request that ran it. Mirrors the HTTP +/// trigger event: `body` is what the model sent, everything else describes the +/// call itself. +#[derive(serde::Serialize)] +struct McpPreprocessorEvent<'a> { + kind: &'a str, + body: Box, + headers: HashMap>, + tool_name: &'a str, +} + +/// Headers withheld from a preprocessor because they authenticate the connection. +/// +/// Not a security boundary: a webhook preprocessor receives all three. Withheld +/// because nothing needs them yet, and releasing one later is additive while +/// withdrawing one after runnables read it is not. +const WITHHELD_FROM_PREPROCESSOR: &[&str] = &["authorization", "cookie", "proxy-authorization"]; + +/// Every header a preprocessor may see. +fn preprocessor_headers( + headers: &http::HeaderMap, +) -> HashMap> { + let mut selected = build_headers(headers, None, true); + selected.retain(|name, _| { + !WITHHELD_FROM_PREPROCESSOR + .iter() + .any(|withheld| withheld.eq_ignore_ascii_case(name)) + }); + selected +} + +/// Build the job arguments for a script or flow run as an MCP tool. +/// +/// Shaped by the runnable's own format: a preprocessor receives the request as +/// an event, and a runnable without one receives only what the model sent. +pub async fn prepare_push_args( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + args: Value, + request: &McpRequest<'_>, +) -> Result { + let mut main_args = HashMap::new(); if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); for (k, v) in map { - args_hash.insert(k, to_raw_value(&v)); + main_args.insert(k, to_raw_value(&v)); } - windmill_queue::PushArgsOwned { extra: None, args: args_hash } - } else { - windmill_queue::PushArgsOwned::default() } + + let runnable_id = if is_flow { + RunnableId::from_flow_path(path) + } else { + // Resolves a `hub/` path to the hub script on its own. + RunnableId::from_script_path(path) + }; + + // MCP is not one of the `TRIGGER_KIND` enum values and does not need to be: + // the per-kind arms of the no-preprocessor heuristic are payload-shape + // special cases for message triggers, and `Webhook` reaches the same generic + // arm MCP wants while sharing that kind's format cache. + let runnable_format = get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; + + Ok(match runnable_format { + // Without a preprocessor there is nowhere for a header to go that the + // model does not also write: its arguments *are* the runnable's + // parameters, so a header bound to one of them would be a value the model + // could set. The request is reachable through a preprocessor, where it + // arrives in a key of the event the model never fills. + RunnableFormat { has_preprocessor: false, .. } => { + windmill_queue::PushArgsOwned { args: main_args, extra: None } + } + RunnableFormat { has_preprocessor: true, version } => { + let headers = preprocessor_headers(request.headers); + match version { + RunnableFormatVersion::V2 => { + let event = McpPreprocessorEvent { + kind: MCP_TRIGGER_KEY, + body: to_raw_value(&main_args), + headers, + tool_name: request.tool_name, + }; + windmill_queue::PushArgsOwned { + args: HashMap::from([("event".to_string(), to_raw_value(&event))]), + extra: None, + } + } + RunnableFormatVersion::V1 => windmill_queue::PushArgsOwned { + args: main_args, + extra: Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": MCP_TRIGGER_KEY, + MCP_TRIGGER_KEY: { + "headers": headers, + "tool_name": request.tool_name, + } + })), + )])), + }, + } + } + }) } /// Parse an HTTP response body into a JSON Value diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 6ae384250b..737db81b3e 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -618,6 +618,7 @@ pub(crate) async fn offboard_global_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email).await?; sqlx::query!("DELETE FROM workspace_invite WHERE email = $1", &email) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index c145252251..ea6d0c5bba 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -261,7 +261,8 @@ fn branch_sqls() -> Branches { FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred}) as draft_users" + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred} \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users" ) }; diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index 2268a63894..a1a9206196 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -99,7 +99,6 @@ fn build_standard_scope_domains() -> Vec { ("configs", "Configs", "Configuration management", false), ("oauth", "OAuth", "OAuth management", false), ("ai", "AI", "AI feature management", false), - ("ai_skills", "AI Skills", "AI skill management", false), ( "ai_evals", "AI Evals", diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 76378e335a..da8532e9d1 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -146,6 +146,7 @@ async fn edit_copilot_config( .await?; let workspace_has_config = ai_config.has_providers(); + let copilot_disabled = ai_config.copilot_disabled; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -158,7 +159,7 @@ async fn edit_copilot_config( .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()) .filter(|c| c.has_providers()); - let effective_ai_config = if workspace_has_config { + let mut effective_ai_config = if workspace_has_config { ai_config } else if let Some(instance_config) = instance_config_with_providers { instance_config @@ -172,6 +173,7 @@ async fn edit_copilot_config( } else { AIConfig::default() }; + effective_ai_config.copilot_disabled = copilot_disabled; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -207,6 +209,9 @@ async fn get_copilot_info( )) })?; + let copilot_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.copilot_disabled); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -215,20 +220,23 @@ async fn get_copilot_info( // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the // free-tier fallback, matching the proxy and edit_copilot_config paths. .filter(|c| c.has_providers()); - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = instance_config { - Ok(Json(instance_config)) - } else if let Some(free_config) = - crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? - { - // Nothing configured: fall back to Windmill's free tier (EE-only). The config - // carries a `free_tier` marker even once the user's grant is spent — with no - // providers, but telling the client *why* AI is off. - Ok(Json(free_config)) - } else { - Ok(Json(AIConfig::default())) - } + let mut effective = + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + workspace_ai_config.0 + } else if let Some(instance_config) = instance_config { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + free_config + } else { + AIConfig::default() + }; + effective.copilot_disabled = copilot_disabled; + Ok(Json(effective)) } #[cfg(feature = "enterprise")] diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b1ad2edd01..d38396562b 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -94,6 +94,9 @@ pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation pub const DISABLE_WORKSPACE_INVITE_EMAILS_SETTING: &str = "disable_workspace_invite_emails"; pub const DISABLE_PASSWORD_LOGIN_SETTING: &str = "disable_password_login"; pub const AUTO_LOGIN_PROVIDER_SETTING: &str = "auto_login_provider"; +/// Name of the SAML attribute or OIDC userinfo claim carrying the user's IdP groups. Unset or +/// empty leaves instance-group membership entirely to SCIM. +pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; @@ -108,6 +111,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; +pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 2c21046523..2803868d5b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -148,43 +148,76 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; /// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower /// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` /// the cleanup queries build overflows Postgres' microsecond field. -const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; +const MAX_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; -/// Apply a configured service log retention, in seconds. +/// Clamp a configured retention window, in seconds, to one a cutoff can be built from. /// -/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a -/// cutoff. The two unusable directions are not the same mistake and must not share a landing -/// point: too large still says "keep these for a very long time", so it is capped and the -/// intent survives, whereas falling back would delete logs the operator meant to keep. A -/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at -/// or after `now` and the next sweep expires the entire history, rows and object-storage files -/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an -/// operator types by analogy with it, and what the settings UI writes into a field that was -/// merely focused — falls back to the default. -pub fn set_service_log_retention_secs(configured: i64) { - let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS { +/// Shared by the retention windows that have no "keep forever" spelling, so that an unusable +/// value can never reach a cutoff. The two unusable directions are not the same mistake and must +/// not share a landing point: too large still says "keep these for a very long time", so it is +/// capped and the intent survives, whereas falling back would delete data the operator meant to +/// keep. A non-positive value has no such reading — every cutoff is `now - retention`, so it +/// lands at or after `now` and the next sweep expires the entire history. `0` is both what an +/// operator types by analogy with job retention, where it does mean keep forever, and what the +/// settings UI writes into a field that was merely focused, so it falls back to the default. +fn clamp_retention_secs(configured: i64, default: i64, what: &str) -> i64 { + if configured > MAX_RETENTION_SECS { tracing::warn!( - "service log retention of {configured}s exceeds the maximum of \ - {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there" + "{what} retention of {configured}s exceeds the maximum of {MAX_RETENTION_SECS}s, \ + capping it there" ); - MAX_SERVICE_LOG_RETENTION_SECS + MAX_RETENTION_SECS } else if configured >= 1 { configured } else { tracing::warn!( - "service log retention of {configured}s would expire every service log, \ - falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s" + "{what} retention of {configured}s would expire the entire history, \ + falling back to the default of {default}s" ); - DEFAULT_SERVICE_LOG_RETENTION_SECS - }; + default + } +} + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`]. Expiry reaches every copy of a log line: +/// the row, the file on disk, and the object-storage object. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + "service log", + ); SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); } +/// Apply a configured OTEL trace retention, in seconds. +/// +/// The only way into [`OTEL_TRACES_RETENTION_SECS`]. +pub fn set_otel_traces_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_OTEL_TRACES_RETENTION_SECS, + "otel traces", + ); + OTEL_TRACES_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long an HTTP request tracing span stays in `otel_traces`, in seconds. +/// +/// Spans are keyed by the job they were captured for and read back by the job detail view, so +/// this is the outer bound on how far back that view can show a job's HTTP requests. It is +/// independent of job retention: a span can outlive its job, or be swept while the job remains. +pub fn otel_traces_retention_secs() -> i64 { + OTEL_TRACES_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// How long a service log line stays retrievable, in seconds. /// /// The outer bound on everything service-log: the `log_file` rows, the raw files in object @@ -423,6 +456,10 @@ lazy_static::lazy_static! { /// would expire every service log cannot reach a cutoff. Read it with /// [`service_log_retention_secs`]. static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); + /// Private on purpose, same as [`SERVICE_LOG_RETENTION_SECS`]: + /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the + /// only reader. + static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 31219621e9..d71bc50e60 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -358,31 +358,38 @@ pub async fn fetch_script_for_update<'a>( .map_err(crate::error::Error::from) } -pub struct ClonedScript { - pub old_script: NewScript, - pub new_hash: i64, -} -// TODO: What if dependency job fails, there is script with NULL in the lock -pub async fn clone_script<'c>( - path: &str, - w_id: &str, +/// Deploys the outcome of a relative-import relock as a new version of `head`, the path's live +/// version that the caller holds `FOR UPDATE`, and archives `head`. A `lock` of `None` records +/// a failed generation: the version carries `lock_error_logs` instead and runs keep resolving +/// to the last version that has a lock. A `modules` of `None` keeps the head's module locks. +/// +/// Writes whatever `head` names and checks nothing: callers are responsible for having +/// established access to its workspace and path, as a dependency job's push already has. +/// +/// `created_at` is stamped when the insert runs, not at transaction start. The row lock on +/// `head` is what orders one relock after another, and with `now()` a transaction that began +/// first but locked second commits a live child older than its archived parent, which every +/// "latest version" read then mis-orders. +pub async fn deploy_relocked_version( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + head: Script, deployment_message: Option, - db: &DB, -) -> crate::error::Result { - let mut tx = db.begin().await?; - let s = if let Some(s) = fetch_script_for_update(path, w_id, &mut *tx).await? { - s - } else { - return Err(crate::error::Error::NotFound(format!( - "Non-archived script with path '{}' not found", - path - ))); - }; + lock: Option<&str>, + modules: Option<&std::collections::HashMap>, + lock_error_logs: Option<&str>, +) -> crate::error::Result { + let s = head; + let w_id = s.workspace_id.as_str(); - let rs = runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut *tx) - .await?; + let rs = + runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut **tx) + .await?; let (debouncing_settings, concurrency_settings) = - runnable_settings::prefetch_cached_tx(&rs, &mut tx).await?; + runnable_settings::prefetch_cached_tx(&rs, &mut *tx).await?; + + // What the row stores is what the hash covers: the new module locks when there are any. + let modules = modules.cloned().or(s.modules); + let modules_json = modules.as_ref().map(serde_json::to_value).transpose()?; let ns = NewScript { path: s.path.clone(), @@ -392,7 +399,7 @@ pub async fn clone_script<'c>( content: s.content, schema: s.schema, is_template: s.is_template, - lock: None, + lock: lock.map(str::to_string), language: s.language, kind: Some(s.kind), tag: s.tag, @@ -424,7 +431,7 @@ pub async fn clone_script<'c>( on_behalf_of: s.on_behalf_of, preserve_on_behalf_of: None, assets: s.assets, - modules: s.modules, + modules, auto_parent: None, labels: s.labels, skip_draft_deletion: None, @@ -433,7 +440,7 @@ pub async fn clone_script<'c>( let new_hash = hash_script(&ns); tracing::debug!( - "cloning script at path {} from '{}' to '{}'", + "deploying relocked version of script at path {} from '{}' to '{}'", s.path, *s.hash, new_hash @@ -446,17 +453,19 @@ pub async fn clone_script<'c>( envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels) + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, \ + lock_error_logs, created_at) SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ - content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ + content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, \ envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, \ + $6::text, clock_timestamp() FROM script WHERE hash = $2 AND workspace_id = $3; - ", new_hash, s.hash.0, w_id).execute(&mut *tx).await?; + ", new_hash, s.hash.0, w_id, lock, modules_json, lock_error_logs).execute(&mut **tx).await?; // Archive base. sqlx::query!( @@ -464,9 +473,8 @@ pub async fn clone_script<'c>( *s.hash, w_id ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - tx.commit().await?; - Ok(ClonedScript { old_script: ns, new_hash }) + Ok(new_hash) } diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 84a0e8c3ea..115dae34e7 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -245,7 +245,9 @@ async fn fetch_other_drafts_users( // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled. Else a real teammate's draft renders // as a phantom "Legacy draft". The genuine NULL-email legacy row keeps - // `username = None` (no `usr`/`password` match and `d.email` is NULL). + // `username = None` (no `usr`/`password` match and `d.email` is NULL), which is + // why an owner that resolves to no name at all — an external JWT's subject has + // neither row — is dropped instead: `None` is taken to mean "legacy" downstream. let rows = sqlx::query_as!( OtherDraftUser, r#"SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as "username?", @@ -261,6 +263,7 @@ async fn fetch_other_drafts_users( AND d.path = $2 AND d.typ = $3 AND (d.email IS NULL OR d.email <> $4) + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ORDER BY d.email NULLS LAST"#, w_id, path, @@ -426,6 +429,67 @@ pub async fn overlay_or_draft_only( } } +/// Delete the drafts an address owns, across every workspace. +/// +/// `draft.email` carries no foreign key to `password`: a draft's owner is any principal the +/// instance authenticates, and an external JWT's subject never has a `password` row. Deleting an +/// account is therefore what has to delete its drafts — a delete path that skips this leaves them +/// behind forever, addressed to someone who no longer exists. Call it in the same transaction as +/// the account removal. +/// +/// No authorization of its own: it acts instance-wide on whatever address it is handed, so the +/// caller must already have authorized removing that account (superadmin, the account's own +/// holder, or SCIM). +pub async fn delete_drafts_of_email<'c>( + executor: impl sqlx::PgExecutor<'c>, + email: &str, +) -> Result<()> { + sqlx::query!("DELETE FROM draft WHERE email = $1", email) + .execute(executor) + .await?; + Ok(()) +} + +/// Move the drafts an address owns onto its new address, for the same reason +/// [`delete_drafts_of_email`] exists: no foreign key follows the rename, so drafts left behind are +/// stranded on an address that no longer authenticates. Same authorization contract, for a rename. +/// +/// The two addresses may each already hold a draft of the same item, since the destination can +/// belong to a principal with no account and so is not covered by the caller's "address is free" +/// check. `draft_pkey_with_user` admits only one, so the moving account's wins — which is also why +/// a rename onto the same address returns early: every row would collide with itself and be +/// cleared. Callers need not compare first (an IdP re-sending an unchanged `userName` does not). +pub async fn rename_drafts_of_email( + conn: &mut sqlx::PgConnection, + old_email: &str, + new_email: &str, +) -> Result<()> { + if old_email == new_email { + return Ok(()); + } + sqlx::query!( + "DELETE FROM draft dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.path = dest.path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Delete EVERY user's draft (and the legacy NULL-email row) at a path+kind. /// Use when the item is DELETED outright: it's gone for everyone, so leaving /// teammates' drafts behind would orphan them forever. Discarding one's OWN diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 00c5a04902..58453b05d2 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -175,7 +175,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -183,7 +183,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28910/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28930/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/backend/windmill-common/tests/user_drafts_rename.rs b/backend/windmill-common/tests/user_drafts_rename.rs new file mode 100644 index 0000000000..35d2b53858 --- /dev/null +++ b/backend/windmill-common/tests/user_drafts_rename.rs @@ -0,0 +1,28 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::user_drafts::rename_drafts_of_email; + +/// A rename onto the same address has to be a no-op: the helper clears a draft the destination +/// already holds at the same item, and every row would be its own destination. SCIM PATCH sends +/// `userName` unconditionally, so an IdP re-sending an unchanged one reaches this. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn renaming_onto_the_same_address_keeps_the_drafts(db: Pool) { + sqlx::query( + "INSERT INTO draft(workspace_id, path, typ, value, email) \ + VALUES ('test-workspace', 'u/test-user/s', 'script', '{}'::json, 'test@windmill.dev')", + ) + .execute(&db) + .await + .expect("failed to seed draft"); + + let mut conn = db.acquire().await.unwrap(); + rename_drafts_of_email(&mut conn, "test@windmill.dev", "test@windmill.dev") + .await + .unwrap(); + + let kept: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM draft WHERE email = 'test@windmill.dev'") + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(kept, 1); +} diff --git a/backend/windmill-dep-map/Cargo.toml b/backend/windmill-dep-map/Cargo.toml index 28b8552e8e..a93376d167 100644 --- a/backend/windmill-dep-map/Cargo.toml +++ b/backend/windmill-dep-map/Cargo.toml @@ -26,4 +26,5 @@ tracing.workspace = true lazy_static.workspace = true chrono.workspace = true itertools.workspace = true +futures.workspace = true uuid.workspace = true diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index ee6a3c3fd4..aab50795a9 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -1,6 +1,7 @@ pub mod ci_tests; #[cfg(feature = "private")] pub mod ci_tests_ee; +pub mod lock_hash; pub mod scoped_dependency_map; pub mod trigger_dependents; pub mod workspace_dependencies; @@ -127,6 +128,40 @@ pub fn extract_referenced_paths( } } +/// Re-records which paths `script_path` imports and what each one's lock hashes to right now. +/// That snapshot is what a later relock-skip check of this importer compares against, so it +/// has to move whenever the imports may have, whether or not the importer's own lock did. +/// +/// Writes for any path in `w_id` and checks nothing: callers are responsible for having +/// established access to that workspace and script, as a dependency job's push already has. +pub async fn refresh_dependency_map( + db: &sqlx::Pool, + w_id: &str, + script_path: &str, + parent_path: &Option, + code: &str, + script_lang: &Option, +) -> error::Result<()> { + use scoped_dependency_map::ScopedDependencyMap; + + let mut tx = db.begin().await?; + let mut dependency_map = + ScopedDependencyMap::fetch_maybe_rearranged(w_id, script_path, "script", parent_path, db) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + + dependency_map.dissolve(tx).await.commit().await?; + Ok(()) +} + pub async fn process_relative_imports( db: &sqlx::Pool, _job_id: Option, @@ -144,29 +179,7 @@ pub async fn process_relative_imports( use scoped_dependency_map::ScopedDependencyMap; use trigger_dependents::trigger_dependents_to_recompute_dependencies; - // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled - { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, - ) - .await?; - - tx = dependency_map - .patch( - extract_referenced_paths(&code, script_path, *script_lang), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - - dependency_map.dissolve(tx).await.commit().await?; - } + refresh_dependency_map(db, w_id, script_path, &parent_path, code, script_lang).await?; { let mut already_visited = args diff --git a/backend/windmill-dep-map/src/lock_hash.rs b/backend/windmill-dep-map/src/lock_hash.rs new file mode 100644 index 0000000000..50bd18a8ee --- /dev/null +++ b/backend/windmill-dep-map/src/lock_hash.rs @@ -0,0 +1,79 @@ +use std::collections::HashMap; + +use futures::TryStreamExt; +use sqlx::{Postgres, Transaction}; +use windmill_common::error::Result; +use windmill_common::scripts::hash_script; + +/// Records what the lock now at each path hashes to, which is one half of the comparison a relock +/// skip makes against what each importer resolved against. +/// +/// Writes any path in `w_id` and checks nothing: callers are responsible for having established +/// the caller's access to that workspace. A path repeated in `entries` keeps its last hash. +/// +/// Callers that write the lock itself in the same statement fold the upsert into that statement +/// instead; this is for the ones with nothing to fold it into. +pub async fn record_lock_hashes( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + entries: &[(String, i64)], +) -> Result<()> { + // Postgres rejects a whole statement that resolves a conflict on one key twice, so a path + // given more than once keeps its last hash, as it would if the two were written in order. + let mut deduped: HashMap<&str, i64> = HashMap::with_capacity(entries.len()); + for (path, hash) in entries { + deduped.insert(path.as_str(), *hash); + } + if deduped.is_empty() { + return Ok(()); + } + let (paths, hashes): (Vec, Vec) = deduped + .into_iter() + .map(|(path, hash)| (path.to_string(), hash)) + .unzip(); + // Recording a hash a path already has would still cut a row version, and the no-op push this + // is reached from is the mode a git-sync of an unchanged workspace runs in. + sqlx::query!( + "INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + SELECT $1, * FROM UNNEST($2::text[], $3::bigint[]) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash + WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + w_id, + &paths[..], + &hashes[..] + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Records the hash of every live lock in `w_id`, for a workspace whose scripts arrived without +/// going through a deploy — a clone, which copies their locks verbatim and so would otherwise hold +/// none of the hashes describing them. +/// +/// Carries the same caller obligation as [`record_lock_hashes`]. +/// +/// `script.lock` is unbounded and a workspace holds one per script, so the rows are streamed and +/// each lock is hashed and dropped before the next arrives; only the hashes accumulate. +pub async fn record_lock_hashes_for_workspace( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, +) -> Result<()> { + let mut entries: Vec<(String, i64)> = Vec::new(); + { + let mut rows = sqlx::query!( + "SELECT DISTINCT ON (path) path, lock FROM script + WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL + ORDER BY path, created_at DESC", + w_id + ) + .fetch(&mut **tx); + + while let Some(row) = rows.try_next().await? { + if let Some(lock) = row.lock { + entries.push((row.path, hash_script(&lock))); + } + } + } + record_lock_hashes(tx, w_id, &entries).await +} diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs index 3f88f7e781..d49fa468ae 100644 --- a/backend/windmill-mcp/src/common/schema.rs +++ b/backend/windmill-mcp/src/common/schema.rs @@ -101,7 +101,7 @@ fn apply_resource_enrichment( let resources_count = resource_cache.len(); let description = match resource_type { Some(rt) => format!( - "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", + "This is a resource named `{}` with the following description: `{}`.\nPass it as the bare string `$res:` — the whole value of this argument, never an object wrapper like {{\"$res\": \"\"}} and never a plain path.\n{}", rt.name, rt.description.as_deref().unwrap_or("No description"), if resources_count == 0 { @@ -138,7 +138,7 @@ fn apply_resource_enrichment( ) }) .collect::>() - .join("\\n"); + .join("\n"); let prior_description = prop_map .get("description") .and_then(Value::as_str) @@ -147,7 +147,7 @@ fn apply_resource_enrichment( prop_map.insert( "description".to_string(), Value::String(format!( - "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", + "{}\nHere are the available resources, one per line as `title: $res:path`. The title is only a label; pass the `$res:path` part verbatim as this argument's value:\n{}", prior_description, resources_description )), ); @@ -804,6 +804,10 @@ mod tests { let desc = node["description"].as_str().unwrap(); assert!(desc.contains("c_aws_account")); assert!(desc.contains("$res:f/platform/aws_dev")); + // MCP clients render this description verbatim, so the separators must be + // real newlines rather than the two-character escape. + assert!(desc.contains('\n')); + assert!(!desc.contains("\\n")); } #[test] diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index e42c0dacfc..7ca1eb08a1 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -16,6 +16,14 @@ use crate::server::endpoints::EndpointTool; /// Result type for backend operations using rmcp's ErrorData directly pub type BackendResult = Result; +/// What the backend needs about the HTTP request a tool call arrived on, in order +/// to hand a runnable the headers of the call that triggered it. +pub struct McpRequest<'a> { + pub headers: &'a http::HeaderMap, + /// The MCP tool name the caller invoked, reported to preprocessors. + pub tool_name: &'a str, +} + /// How a script/flow listing is narrowed by path at the SQL layer, *before* the /// `ITEMS_FETCH_MAX_LIMIT` cap applies. /// @@ -157,6 +165,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Run a flow and wait for result @@ -166,6 +175,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Call an endpoint tool (generated API endpoint) diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index b6fb7a5b0a..b97e374e98 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -12,7 +12,7 @@ pub mod tools; // Re-export main types pub use crate::common::types::{McpToken, MultiWorkspaceMcp, WorkspaceInfo}; -pub use backend::{BackendResult, McpAuth, McpBackend, PathFilter}; +pub use backend::{BackendResult, McpAuth, McpBackend, McpRequest, PathFilter}; pub use endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only, list_workspaces_tool, non_empty_body_fields, EndpointTool, diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index f407504c4b..e27eaa8470 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -9,8 +9,10 @@ use crate::common::transform::{ extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, }; -use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId}; -use crate::server::backend::{McpAuth, McpBackend, PathFilter}; +use crate::common::types::{ + McpToken, MultiWorkspaceMcp, ResourceInfo, SchemaType, ToolableItem, WorkspaceId, +}; +use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter}; use crate::server::endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; @@ -101,16 +103,24 @@ enum McpMode { Multi(String), } +/// Everything a request carries besides its MCP payload. +struct McpContext { + auth: A, + mode: McpMode, + headers: http::HeaderMap, +} + impl Runner { /// Create a new Runner with the given backend pub fn new(backend: B) -> Self { Self { backend: Arc::new(backend) } } - /// Extract authentication and the workspace mode from request context + /// Extract authentication, the workspace mode and the HTTP request itself + /// from the request context fn extract_context( context: &RequestContext, - ) -> Result<(B::Auth, McpMode), ErrorData> { + ) -> Result, ErrorData> { let http_parts = context.extensions.get::().ok_or_else(|| { tracing::error!("http::request::Parts not found"); ErrorData::internal_error("http::request::Parts not found", None) @@ -148,7 +158,7 @@ impl Runner { McpMode::Single(workspace_id) }; - Ok((auth.clone(), mode)) + Ok(McpContext { auth: auth.clone(), mode, headers: http_parts.headers.clone() }) } } @@ -391,6 +401,18 @@ fn authorize_endpoint_call( Ok(()) } +/// Map the model's argument keys back to the runnable's original parameter names. +fn transform_call_args(args: Value, item_schema: &Option) -> Value { + let Value::Object(map) = args else { + return args; + }; + let mut args_hash = HashMap::new(); + for (k, v) in map { + args_hash.insert(reverse_transform_key(&k, item_schema), v); + } + Value::Object(args_hash.into_iter().collect()) +} + fn find_matching_path(candidates: Vec, request_name: &str) -> Option { candidates .into_iter() @@ -427,7 +449,7 @@ impl ServerHandler for Runner { _request: Option, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, .. } = Self::extract_context(&context)?; // Parse MCP scopes to determine what to expose let scopes = auth.scopes().unwrap_or(&[]); @@ -455,7 +477,7 @@ impl ServerHandler for Runner { request: CallToolRequestParams, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, headers } = Self::extract_context(&context)?; // Parse MCP scopes for authorization let scopes = auth.scopes().unwrap_or(&[]); @@ -464,6 +486,7 @@ impl ServerHandler for Runner { let read_only = auth.read_only(); let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); + let mcp_request = McpRequest { headers: &headers, tool_name: request.name.as_ref() }; // Every tool here runs to completion in one round trip: none of them ask the // client for input, so the MRTR variants of `CallToolResponse` are never built. @@ -474,14 +497,22 @@ impl ServerHandler for Runner { &workspace_id, &scope_config, read_only, - request.name, + request.name.clone(), args, + &mcp_request, ) .await } McpMode::Multi(token) => { - self.call_tool_multi(&auth, &token, &scope_config, read_only, request.name, args) - .await + self.call_tool_multi( + &auth, + &token, + &scope_config, + read_only, + request.name.clone(), + args, + ) + .await } }?; Ok(result.into()) @@ -665,6 +696,7 @@ impl Runner { read_only: bool, name: std::borrow::Cow<'static, str>, args: Value, + request: &McpRequest<'_>, ) -> Result { // Check if this is an endpoint tool let endpoint_tools = self.backend.all_endpoint_tools(); @@ -777,17 +809,7 @@ impl Runner { .map_err(|e| ErrorData::internal_error(e.message, None))? }; - // Transform arguments back to original key names - let transformed_args = if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); - for (k, v) in map { - let original_key = reverse_transform_key(&k, &item_schema); - args_hash.insert(original_key, v); - } - Value::Object(args_hash.into_iter().collect()) - } else { - args - }; + let transformed_args = transform_call_args(args, &item_schema); let script_or_flow_path = if is_hub { format!("hub/{}", path) @@ -798,11 +820,23 @@ impl Runner { // Execute script or flow let result = if tool_type == "script" { self.backend - .run_script(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_script( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await } else { self.backend - .run_flow(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_flow( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await }; diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c7c119a5a7..8ac4fd390a 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4007,33 +4007,10 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { { let maybe_new_id = match j.kind { - JobKind::Dependencies => { - let deployment_message = j - .args - .clone() - .map(|hashmap| { - hashmap - .get("deployment_message") - .map(|map_value| serde_json::from_str::(map_value.get()).ok()) - .flatten() - }) - .flatten(); - - // This way we tell downstream which script we should archive when the resolution is finished. - // (not used at the moment) - j.args - .as_mut() - .map(|args| args.insert("base_hash".to_owned(), to_raw_value(&*base_hash))); - - windmill_common::scripts::clone_script( - j.runnable_path(), - &j.workspace_id, - deployment_message, - db, - ) - .await? - .new_hash - } + // A script gets its new version from the worker, once the generated lock is known + // to differ from the live version's: minting one here would deploy, and walk the + // importers of, a version whose lock turns out byte-identical to its parent's. + JobKind::Dependencies => *base_hash, JobKind::FlowDependencies => { sqlx::query_scalar!( "INSERT INTO flow_version diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 63b415c74b..eea1476c9d 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -130,6 +130,12 @@ pub struct EditResourceType { pub schema: Option, pub description: Option, pub is_fileset: Option, + /// Doubly optional so an edit can distinguish the two things a plain + /// `Option` conflates: an absent field leaves the extension alone, while an + /// explicit `null` clears it. A hub pull relies on both — a type that stops + /// being a file type has to stop being one locally too. + #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + pub format_extension: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -2874,10 +2880,42 @@ async fn update_resource_type( if let Some(is_fileset) = ns.is_fileset { sqlb.set("is_fileset", if is_fileset { "TRUE" } else { "FALSE" }); } + if let Some(format_extension) = ns.format_extension.clone() { + match format_extension { + Some(ext) => sqlb.set_str("format_extension", ext), + None => sqlb.set("format_extension", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + // Creation refuses the pair outright, so an edit must too — otherwise the same + // impossible type (a set of files that is also one file) is reachable by setting + // either half on an existing row. Whichever half the request omits is read from + // the row being edited, inside this transaction and with the row locked: read + // outside it, two concurrent edits each supplying one half would both pass. + let current = sqlx::query!( + "SELECT is_fileset, format_extension FROM resource_type + WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + &name, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + let effective_is_fileset = ns + .is_fileset + .unwrap_or_else(|| current.as_ref().map(|c| c.is_fileset).unwrap_or(false)); + let effective_format_extension = match &ns.format_extension { + Some(value) => value.clone(), + None => current.and_then(|c| c.format_extension), + }; + if effective_is_fileset && effective_format_extension.is_some() { + return Err(Error::BadRequest( + "A fileset resource type cannot have a format_extension".to_string(), + )); + } + sqlx::query(&sql).execute(&mut *tx).await?; audit_log( &mut *tx, diff --git a/backend/windmill-types/src/more_serde.rs b/backend/windmill-types/src/more_serde.rs index d4b648d2d5..f9cbe739c5 100644 --- a/backend/windmill-types/src/more_serde.rs +++ b/backend/windmill-types/src/more_serde.rs @@ -38,6 +38,25 @@ pub fn is_default(t: &T) -> bool { &T::default() == t } +pub fn maybe_number<'de, T, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: FromStr + serde::Deserialize<'de>, + ::Err: Display, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumericOrString { + String(String), + RawT(T), + } + + match NumericOrString::::deserialize(deserializer)? { + NumericOrString::String(s) => T::from_str(&s).map_err(serde::de::Error::custom), + NumericOrString::RawT(i) => Ok(i), + } +} + pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -66,3 +85,60 @@ where NumericOrNull::Null => Ok(None), } } + +/// Deserializer for a doubly-optional field, so a struct can tell an absent key +/// (`None`) from an explicit `null` (`Some(None)`). +/// +/// Plain serde collapses both into the outer `None`, which makes the distinction +/// unusable exactly where it matters: a payload that omits a field means "leave it +/// alone", while one that sends `null` means "clear it". +/// +/// ```ignore +/// #[serde(default, deserialize_with = "double_option", skip_serializing_if = "Option::is_none")] +/// pub field: Option>, +/// ``` +pub fn double_option<'de, T, D>(deserializer: D) -> Result>, D::Error> +where + T: serde::Deserialize<'de>, + D: serde::Deserializer<'de>, +{ + serde::Deserialize::deserialize(deserializer).map(Some) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + #[derive(Deserialize)] + struct WithMaybeNumber { + #[serde(deserialize_with = "super::maybe_number")] + n: i64, + } + + #[test] + fn maybe_number_accepts_number() { + let v: WithMaybeNumber = serde_json::from_value(serde_json::json!({ "n": 12345 })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_accepts_string() { + let v: WithMaybeNumber = + serde_json::from_value(serde_json::json!({ "n": "12345" })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_rejects_non_numeric_string() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": "abc" })).is_err() + ); + } + + #[test] + fn maybe_number_rejects_null() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": null })).is_err() + ); + } +} diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 414650e846..19f6a06021 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -16,7 +16,7 @@ use crate::{ runnable_settings::{ConcurrencySettings, DebouncingSettings}, }; -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct ScriptModule { pub content: String, pub language: ScriptLang, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 57b859075a..9bba9a99e6 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -19,14 +19,17 @@ use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::jobs::JobKind; use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; -use windmill_common::scripts::ScriptHash; +use windmill_common::scripts::{ + deploy_relocked_version, fetch_script_for_update, hash_script, ScriptHash, ScriptModule, +}; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; -use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; +use windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -38,8 +41,10 @@ use windmill_common::{ scripts::ScriptLang, DB, }; +use windmill_dep_map::lock_hash::record_lock_hashes; pub use windmill_dep_map::{ extract_referenced_paths, extract_relative_imports, process_relative_imports, + refresh_dependency_map, }; use windmill_git_sync::{ handle_deployment_metadata, tally_deployed_object_changes, DeployedObject, @@ -86,10 +91,13 @@ use crate::{ /// has the toolchain and, since the cache key is per OS/arch, the platform the runtime /// workers use. Deploys that supply their own lock never reach a dependency job at all and /// queue theirs from `create_script_internal` instead. -async fn maybe_queue_binary_prebuild(db: &DB, job: &MiniPulledJob, lock: &str) -> Result<()> { - let (Some(hash), Some(path), Some(lang)) = - (job.runnable_id, job.runnable_path.clone(), job.script_lang) - else { +async fn maybe_queue_binary_prebuild( + db: &DB, + job: &MiniPulledJob, + hash: ScriptHash, + lock: &str, +) -> Result<()> { + let (Some(path), Some(lang)) = (job.runnable_path.clone(), job.script_lang) else { return Ok(()); }; let Some(prebuild) = @@ -283,6 +291,13 @@ pub async fn handle_dependency_job( job.runnable_path() ); let script_path = job.runnable_path(); + let w_id = &job.workspace_id; + + let triggered_by_relative_import = job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default(); // A build pass reads the same script data but writes none of the deploy state below, // including the `lock_error_logs` stamp on a fetch failure: the version it builds is @@ -296,14 +311,48 @@ pub async fn handle_dependency_job( *deployment_tallied = true; } + // A relative-import relock deploys nothing until `commit_relock` says so, while the + // caller's fallback tally assumes a failed dependency job left a deployed version behind. + // Claim the tally here; the failure path hands it back once it has minted the version + // that carries the error. + if triggered_by_relative_import { + *deployment_tallied = true; + } + + // A relative-import relock locks the path's live version as of now, not the hash captured + // when the job was pushed: a deploy can land during the debounce delay, after which that + // hash names an archived version. What it generates is committed against the live version + // re-read under a row lock, so a deploy landing mid-generation is caught there too. + let target_hash = if triggered_by_relative_import { + Some(ScriptHash(live_head_hash(db, w_id, script_path).await?)) + } else { + job.runnable_id + }; + // `JobKind::Dependencies` job store either: // - A saved script `hash` in the `script_hash` column. // - Preview raw lock and code in the `queue` or `job` table. - let script_data = &match job.runnable_id { + let script_data = &match target_hash { + // Read straight from the database: the cache pins a version's data under its hash for + // as long as this worker lives, and the live version may still be waiting on its own + // dependency job's lock, which lands in place. A run resolving to it on this worker + // would then get no lock from the cache at all. + Some(hash) if triggered_by_relative_import => { + let raw = cache::script::fetch_script_from_db(db, hash, std::panic::Location::caller()) + .await?; + Cow::Owned(std::sync::Arc::new(cache::ScriptData { + lock: raw.lock, + code: raw.content, + modules: raw.modules, + })) + } Some(hash) => match cache::script::fetch(&Connection::from(db.clone()), hash).await { Ok(d) => Cow::Owned(d.0), Err(e) => { - if !is_build_job { + // The live version of a relative-import relock is what runs resolve to, and + // `lock_error_logs` on it takes it out of resolution; the job carries the + // error instead, since it deployed nothing. + if !is_build_job && !triggered_by_relative_import { let logs2 = sqlx::query_scalar!( "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2", &job.id, @@ -348,12 +397,6 @@ pub async fn handle_dependency_job( .await; } - let triggered_by_relative_import = job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default(); - // Extract temp_script_refs from job args (path -> hash mapping for temp storage) let temp_script_refs: Option> = job .args @@ -391,20 +434,17 @@ pub async fn handle_dependency_job( ) .await; + let (deployment_message, parent_path) = + get_deployment_msg_and_parent_path_from_args(job.args.clone()); + match content { Ok(content) => { - if job.runnable_id.is_none() { + let Some(current_hash) = target_hash else { // it a one-off raw script dependency job, no need to update the db return Ok(to_raw_value_owned( json!({ "status": "Successful lock file generation", "lock": content }), )); - } - - let current_hash = job.runnable_id.unwrap_or(ScriptHash(0)); - let w_id = &job.workspace_id; - - let (deployment_message, parent_path) = - get_deployment_msg_and_parent_path_from_args(job.args.clone()); + }; // Generate lockfiles for module files (if any). // @@ -441,7 +481,9 @@ pub async fn handle_dependency_job( occupancy_metrics, &raw_workspace_dependencies_o, module.lock.as_deref(), - triggered_by_relative_import, + // A module that was never locked has nothing a skip could hand + // back; the path's lock is the parent script's, not its own. + triggered_by_relative_import && module.lock.is_some(), script_path, None, "script", @@ -464,34 +506,83 @@ pub async fn handle_dependency_job( None }; - // We do not create new row for this update - // That means we can keep current hash and just update lock - // Also store lockfile hash for dependency change detection - let lockfile_hash = windmill_common::scripts::hash_script(&content); - let updated_modules_json = updated_modules - .as_ref() - .and_then(|m| serde_json::to_value(m).ok()); - sqlx::query!( - "WITH update_lock AS ( - UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + let deployed_hash = if triggered_by_relative_import { + match commit_relock( + db, + w_id, + script_path, + current_hash, + Some(&content), + updated_modules.as_ref(), + None, + deployment_message.clone(), ) - INSERT INTO lock_hash (workspace_id, path, lockfile_hash) - VALUES ($3, $4, $5) - ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - &content, - ¤t_hash.0, - w_id, - script_path, - &lockfile_hash, - updated_modules_json - ) - .execute(db) - .await?; + .await? + { + RelockOutcome::Deployed(hash) => hash, + RelockOutcome::Unchanged => { + let log_msg = "\nLock unchanged: no new version deployed"; + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + // The imports may have moved even though the result did not, and the + // map is what this importer's next skip check reads. + refresh_dependency_map( + db, + w_id, + script_path, + &parent_path, + &script_data.code, + &job.script_lang, + ) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock unchanged, no new version deployed", "lock": content }), + )); + } + RelockOutcome::Superseded(head) => { + let log_msg = format!( + "\nVersion {head} was deployed while this lock was generated; discarding it and queueing a relock of that version" + ); + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock generation superseded by a newer version", "lock": content }), + )); + } + } + } else { + // We do not create new row for this update + // That means we can keep current hash and just update lock + // Also store lockfile hash for dependency change detection + let lockfile_hash = windmill_common::scripts::hash_script(&content); + let updated_modules_json = updated_modules + .as_ref() + .and_then(|m| serde_json::to_value(m).ok()); + sqlx::query!( + "WITH update_lock AS ( + UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + ) + INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + VALUES ($3, $4, $5) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + &content, + ¤t_hash.0, + w_id, + script_path, + &lockfile_hash, + updated_modules_json + ) + .execute(db) + .await?; - // `lock` has been updated; invalidate the cache. - // Since only worker that ran this Dependency Job has the cache - // we do not need to think about invalidating cache for other workers. - cache::script::invalidate(current_hash); + // `lock` has been updated; invalidate the cache. + // Since only worker that ran this Dependency Job has the cache + // we do not need to think about invalidating cache for other workers. + cache::script::invalidate(current_hash); + current_hash + }; // The version only became runnable now, so this process still resolves the path to // the one before it. Only the runnable-hash cache: the import-side caches ignore the // lock, so evicting this process' half of that pair here would key a bundle by a @@ -504,7 +595,7 @@ pub async fn handle_dependency_job( &db, &w_id, DeployedObject::Script { - hash: current_hash, + hash: deployed_hash, path: script_path.to_string(), parent_path: parent_path.clone(), }, @@ -565,7 +656,7 @@ pub async fn handle_dependency_job( }); } - if let Err(e) = maybe_queue_binary_prebuild(db, job, &content).await { + if let Err(e) = maybe_queue_binary_prebuild(db, job, deployed_hash, &content).await { tracing::error!(%e, "error queueing the auto-build binary job for {script_path}"); } @@ -583,14 +674,49 @@ pub async fn handle_dependency_job( .await? .flatten() .unwrap_or_else(|| "no logs".to_string()); - sqlx::query!( - "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", - &format!("{logs2}\n{error}"), - &job.runnable_id.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; + let error_logs = format!("{logs2}\n{error}"); + if let (true, Some(hash)) = (triggered_by_relative_import, target_hash) { + // The same shape a failed deploy leaves: a version without a lock that carries + // the error, so it shows on the script while runs keep resolving to the last + // version that has one. Only that version is the caller's fallback to tally; + // one that landed meanwhile owns its own lock, and a commit that failed left + // nothing. + match commit_relock( + db, + w_id, + script_path, + hash, + None, + None, + Some(&error_logs), + deployment_message.clone(), + ) + .await + { + Ok(RelockOutcome::Deployed(_)) => *deployment_tallied = false, + Ok(RelockOutcome::Superseded(_)) => { + if let Err(e) = + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await + { + tracing::error!(%e, "error queueing a relock of {script_path}") + } + } + Ok(RelockOutcome::Unchanged) => {} + Err(e) => { + tracing::error!(%e, "error recording the failed relock of {script_path}") + } + } + } else { + sqlx::query!( + "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", + &error_logs, + &job.runnable_id.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + } Err(Error::ExecutionErr(format!( "Error locking file: {error}\n\nlogs:\n{}", remove_ansi_codes(&logs2) @@ -598,6 +724,136 @@ pub async fn handle_dependency_job( } } } + +/// The version of `script_path` that runs resolve to, which is what a relative-import relock +/// locks. `NotFound` when the path holds none, which a job pushed for a path since archived or +/// deleted reports as its own failure. +async fn live_head_hash(db: &DB, w_id: &str, script_path: &str) -> error::Result { + sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + )) + }) +} + +enum RelockOutcome { + /// A new version carrying the result is the live one. + Deployed(ScriptHash), + /// The live version already holds this lock and these module locks; nothing was written. + Unchanged, + /// The live version is no longer the one the lock was generated for; nothing was written. + Superseded(ScriptHash), +} + +/// Commits what a relative-import relock produced against the path's live version, read under +/// a row lock so relocks of one path serialize on it. +/// +/// A result equal to the live version's lock and module locks writes nothing: the importer's +/// dependencies did not move, and a new version would deploy byte-identical content and then +/// walk its own importers for nothing. A `lock` of `None` is a failed generation and always +/// deploys, as the version that carries the error. +async fn commit_relock( + db: &DB, + w_id: &str, + script_path: &str, + generated_for: ScriptHash, + lock: Option<&str>, + modules: Option<&HashMap>, + lock_error_logs: Option<&str>, + deployment_message: Option, +) -> error::Result { + let mut tx = db.begin().await?; + let mut head = None; + for _ in 0..4 { + head = fetch_script_for_update(script_path, w_id, &mut *tx).await?; + if head.is_some() { + break; + } + // Having waited on the live version's row lock, the statement re-checked that row + // once the holder committed, found it archived, and returned nothing: the successor + // the holder inserted is not in the statement's snapshot. A fresh statement sees it, + // unless yet another writer got there first, so this goes around a few times before + // concluding the path holds no live version. + } + let Some(head) = head else { + return Err(Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + ))); + }; + if head.hash != generated_for { + // A deploy landed while the lock was generated. It carried its own lock or queued its + // own dependency job, and this lock describes content that is no longer live. + return Ok(RelockOutcome::Superseded(head.hash)); + } + let lock_hash_entry = lock.map(|lock| (script_path.to_string(), hash_script(lock))); + if let Some(lock) = lock { + let modules_unchanged = modules.map_or(true, |m| head.modules.as_ref() == Some(m)); + if head.lock.as_deref() == Some(lock) && modules_unchanged { + // The hash row is still written, and under the same row lock: a version deployed + // before lock hashes were recorded has none, so its importers cannot skip until + // it does, and a deploy that takes the lock next must not have the hash it records + // overwritten by this one. + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + return Ok(RelockOutcome::Unchanged); + } + } + let new_hash = deploy_relocked_version( + &mut tx, + head, + deployment_message, + lock, + modules, + lock_error_logs, + ) + .await?; + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + Ok(RelockOutcome::Deployed(ScriptHash(new_hash))) +} + +/// Queues another relative-import relock of `script_path`, through the same push the fan-out +/// uses. The version live now was deployed while a lock was generated for its predecessor; +/// when that deploy was a sibling relock it queued nothing for this path, and the result just +/// discarded may have been the one generated against the current imports. +async fn requeue_relock( + db: &DB, + job: &MiniPulledJob, + script_path: &str, + deployment_message: Option, + parent_path: Option, +) -> error::Result<()> { + let already_visited = job + .args + .as_ref() + .and_then(|x| x.get("already_visited")) + .and_then(|v| serde_json::from_str::>(v.get()).ok()) + .unwrap_or_default(); + trigger_dependents_to_recompute_dependencies( + &job.workspace_id, + vec![DependencyDependent { + importer_path: script_path.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + }], + deployment_message, + parent_path, + &job.permissioned_as_email, + &job.created_by, + &job.permissioned_as, + db, + already_visited, + ) + .await +} + fn remove_ansi_codes(s: &str) -> String { lazy_static::lazy_static! { static ref ANSI_REGEX: regex::Regex = regex::Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap(); @@ -2892,8 +3148,12 @@ async fn try_skip_relock( } // Fetch existing lock based on runnable type - let lock = match runnable_type { - "script" => sqlx::query_scalar!( + let lock = match (runnable_type, existing_lock) { + // A script's module asks with the script's own type and hands over the lock it last + // deployed with. The path's lock below is the parent script's, and a module given + // that loses whatever it resolves on its own. + ("script", Some(module_lock)) => Some(module_lock.to_string()), + ("script", None) => sqlx::query_scalar!( "SELECT lock FROM script WHERE path = $1 AND workspace_id = $2 AND lock IS NOT NULL AND deleted = false ORDER BY created_at DESC LIMIT 1", base_path, @@ -2903,7 +3163,7 @@ async fn try_skip_relock( .await? .flatten(), - "flow" | "app" => existing_lock.map(|s| s.to_string()), + ("flow" | "app", existing_lock) => existing_lock.map(|s| s.to_string()), _ => None, }; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f9584c3091..6a165901e4 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.800.0"; +export const VERSION = "v1.803.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index b45666a1e1..2dac8639e2 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -603,7 +603,7 @@ async function dev(opts: DevOptions, appFolder?: string) { build.onLoad( { filter: /.*/, namespace: "wmill-virtual" }, (args: any) => { - const contents = wmillTs(port); + const contents = wmillTs(); log.info( colors.yellow( `[wmill-virtual] Loading virtual module: ${args.path}`, diff --git a/cli/src/commands/app/wmillTsDev.ts b/cli/src/commands/app/wmillTsDev.ts index 7cb2ea328b..821060d8e4 100644 --- a/cli/src/commands/app/wmillTsDev.ts +++ b/cli/src/commands/app/wmillTsDev.ts @@ -1,5 +1,5 @@ //comment this line and last to dev -export function wmillTsDev(port: number) { return ` +export function wmillTsDev() { return ` let reqs: Record = {} let ws: WebSocket | null = null let wsReady: Promise @@ -10,7 +10,7 @@ function initWebSocket() { wsReadyResolve = resolve }) - ws = new WebSocket('ws://localhost:${port}') + ws = new WebSocket((window.location.protocol === 'https:' ? 'wss:' : 'ws:') + '//' + window.location.host) ws.onopen = () => { console.log('[wmill] WebSocket connected') @@ -157,4 +157,4 @@ export function streamJob( ws?.send(JSON.stringify({ jobId, type: 'streamJob', reqId })) }) } -`} \ No newline at end of file +`} diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 6ace1eaf48..7a383fcfeb 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -1144,7 +1144,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", @@ -1162,7 +1162,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index 1a1f144de1..f752147811 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -17,6 +17,9 @@ interface HubResourceType { app: string; description: string; is_fileset?: boolean; + // Absent from hubs predating the column, so a missing value is "ordinary type", + // not "unset it". + format_extension?: string | null; } export async function pull(opts: GlobalOptions) { @@ -116,7 +119,8 @@ export async function pull(opts: GlobalOptions) { typeof y.schema !== "string" && deepEqual(y.schema, x.schema) && y.description === x.description && - (y.is_fileset ?? false) === (x.is_fileset ?? false) + (y.is_fileset ?? false) === (x.is_fileset ?? false) && + (y.format_extension ?? null) === (x.format_extension ?? null) ) ) { log.info("skipping " + x.name + " (same as current)"); diff --git a/cli/src/commands/resource-type/resource-type.ts b/cli/src/commands/resource-type/resource-type.ts index 96c8429eb4..fd4b72108e 100644 --- a/cli/src/commands/resource-type/resource-type.ts +++ b/cli/src/commands/resource-type/resource-type.ts @@ -25,6 +25,9 @@ export interface ResourceTypeFile { schema?: any; description?: string; is_fileset?: boolean; + // Extension for a type whose value is one file rather than a set of fields; it + // is what makes the resource editor a file editor for that language. + format_extension?: string | null; } export async function pushResourceType( diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index edbedc2612..2eb6ba42b8 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -106,7 +106,8 @@ export async function pushSchedule( path: string, schedule: Schedule | ScheduleFile | undefined, localSchedule: ScheduleFile, - permissionedAsContext?: PermissionedAsContext + permissionedAsContext?: PermissionedAsContext, + enabledOwnedByParent?: boolean ): Promise { path = removeType(path, "schedule").replaceAll(SEP, "/"); log.debug(`Processing local schedule ${path}`); @@ -123,6 +124,21 @@ export async function pushSchedule( // Strip CLI-only boolean marker before sending to API delete (localSchedule as any).has_permissioned_as; + // In a fork, the file's `enabled` is the parent's for a path the parent + // also has (see sync push's `parentOwnedScheduleEnabled`): the fork's own + // flag stays as it is. + if (enabledOwnedByParent && schedule) { + if ( + localSchedule.enabled !== undefined && + localSchedule.enabled !== schedule.enabled + ) { + log.warnAlways( + `Schedule ${path} stays ${schedule.enabled ? "enabled" : "disabled"}: the file says ${localSchedule.enabled ? "enabled" : "disabled"}, but in a fork that flag is the parent workspace's` + ); + } + delete localSchedule.enabled; + } + const preserveFields: { permissioned_as?: string; preserve_permissioned_as?: boolean } = {}; if (permissionedAsContext?.userIsAdminOrDeployer) { if (schedule) { @@ -153,13 +169,9 @@ export async function pushSchedule( ...preserveFields, }, }); - // Tarball export from a fork strips `enabled` from schedule YAMLs so - // the fork→parent git-sync round-trip can't flip the parent's state. - // Skip the secondary setScheduleEnabled call when the local YAML - // doesn't carry `enabled` — sending `{ enabled: undefined }` would - // serialize to `{}` and the backend (`SetEnabled.enabled` is required) - // would reject the request. Preserving the target's existing flag is - // exactly the round-trip-safe behavior. + // No `enabled` in the file (absent from the YAML, or set aside above) + // leaves the remote flag alone: `SetEnabled.enabled` is required, so + // `{ enabled: undefined }` would be rejected rather than ignored. if ( localSchedule.enabled !== undefined && localSchedule.enabled !== schedule.enabled @@ -167,13 +179,12 @@ export async function pushSchedule( log.info(colors.bold.yellow( `Schedule ${path} is ${localSchedule.enabled ? "enabled" : "disabled"} locally but not on remote, updating remote` )); - await wmill.setScheduleEnabled({ - workspace: workspace, + await setEnabledUnlessParentOwned( + workspace, path, - requestBody: { - enabled: localSchedule.enabled, - }, - }); + localSchedule.enabled, + schedule.enabled + ); } } catch (e) { console.error((e as any).body); @@ -194,6 +205,44 @@ export async function pushSchedule( console.error((e as any).body); throw e; } + // A create in a fork lands disabled whatever the request says. A fork-only + // path the file wants enabled is enabled here, so one push converges; a + // parent-owned one stays disabled. + if (enabledOwnedByParent !== undefined && localSchedule.enabled === true) { + if (enabledOwnedByParent) { + log.warnAlways( + `Schedule ${path} created disabled: the file says enabled, but in a fork that flag is the parent workspace's` + ); + } else { + await setEnabledUnlessParentOwned(workspace, path, true, false); + } + } + } +} + +// The parent listing behind `enabledOwnedByParent` sees only what the pusher +// may read; the backend's `fork-conflict` refusal is the last word, so a path +// it says the parent has keeps the fork's flag rather than failing the push. +async function setEnabledUnlessParentOwned( + workspace: string, + path: string, + enabled: boolean, + remoteEnabled: boolean +): Promise { + try { + await wmill.setScheduleEnabled({ + workspace, + path, + requestBody: { enabled }, + }); + } catch (e) { + const conflict = parseForkConflict(e); + if (!conflict) { + throw e; + } + log.warnAlways( + `Schedule ${path} left ${remoteEnabled ? "enabled" : "disabled"}: the parent workspace '${conflict.parentWorkspaceId}' has the same schedule, so its flag is the parent's to set` + ); } } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 8c93bdb963..fada434e0d 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -2213,7 +2213,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", @@ -2231,7 +2231,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index cf44d5ec80..bab9967b3c 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -25,7 +25,7 @@ import { } from "yaml"; import JSZip from "jszip"; import { minimatch } from "minimatch"; -import { yamlParseContent } from "../../utils/yaml.ts"; +import { yamlParseContent, yamlParseFile } from "../../utils/yaml.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { @@ -1138,7 +1138,7 @@ export function rawAppPathWithinFolder( return resolved; } -function ZipFSElement( +export function ZipFSElement( zip: JSZip, useYaml: boolean, defaultTs: "bun" | "deno", @@ -1146,6 +1146,14 @@ function ZipFSElement( resourceTypeToIsFileset: Record, ignoreCodebaseChanges: boolean, stripOnBehalfOf: boolean, + // Names a flow's rendered inline-script files after the checkout's own + // `!inline` references (module id -> file). The export carries script + // source, never a reference, so without a checkout to defer to every file + // is named from the step summary, and a file the checkout names otherwise + // reads as a delete + add on every push while the resolved flows are equal. + localFlowInlineMapping?: ( + flowDir: string, + ) => Promise>, ): DynFSElement { // Pre-scan: find zip base paths of scripts that have modules. // These scripts use the folder layout: {basePath}__mod/script.{ext} @@ -1249,59 +1257,71 @@ function ZipFSElement( log.error(`Failed to parse flow.yaml at path: ${p}`); throw error; } - let inlineScripts; + let inlineScripts: InlineScript[]; try { - const assigner = newPathAssigner(defaultTs, { - skipInlineScriptSuffix: getNonDottedPaths(), - }); - // Preserve original !inline filenames from the flow to avoid phantom renames - const inlineMapping = extractCurrentMapping( - flow.value.modules as any, - {}, - flow.value.failure_module, - flow.value.preprocessor_module, - ); - inlineScripts = extractInlineScriptsForFlows( - flow.value.modules as any, - inlineMapping, - SEP, - defaultTs, - assigner, - { + // Extraction rewrites the modules' content into `!inline` refs, + // so each attempt works on its own copy of the flow. + const render = ( + source: OpenFlow, + inlineMapping: Record, + ): [OpenFlow, InlineScript[]] => { + const f: OpenFlow = structuredClone(source); + const assigner = newPathAssigner(defaultTs, { + skipInlineScriptSuffix: getNonDottedPaths(), + }); + const options = { skipInlineScriptSuffix: getNonDottedPaths(), failOnInlineDirective: true, - }, - ); - if (flow.value.failure_module) { - inlineScripts.push( - ...extractInlineScriptsForFlows( - [flow.value.failure_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { - skipInlineScriptSuffix: getNonDottedPaths(), - failOnInlineDirective: true, - }, - ), - ); - } - if (flow.value.preprocessor_module) { - inlineScripts.push( - ...extractInlineScriptsForFlows( - [flow.value.preprocessor_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { - skipInlineScriptSuffix: getNonDottedPaths(), - failOnInlineDirective: true, - }, - ), + }; + const scripts = extractInlineScriptsForFlows( + f.value.modules as any, + inlineMapping, + SEP, + defaultTs, + assigner, + options, ); + if (f.value.failure_module) { + scripts.push( + ...extractInlineScriptsForFlows( + [f.value.failure_module], + inlineMapping, + SEP, + defaultTs, + assigner, + options, + ), + ); + } + if (f.value.preprocessor_module) { + scripts.push( + ...extractInlineScriptsForFlows( + [f.value.preprocessor_module], + inlineMapping, + SEP, + defaultTs, + assigner, + options, + ), + ); + } + return [f, scripts]; + }; + const inlineMapping = localFlowInlineMapping + ? await localFlowInlineMapping(finalPath) + : {}; + let rendered = render(flow, inlineMapping); + // The assigner keeps the names it hands out unique, not the + // checkout's: one of those equal to another step's + // summary-derived name would leave two files at one path, so + // such a flow renders the export's way. + if ( + new Set(rendered[1].map((s) => s.path)).size !== + rendered[1].length + ) { + rendered = render(flow, {}); } + [flow, inlineScripts] = rendered; } catch (error) { log.error( `Failed to extract inline scripts for flow at path: ${p}`, @@ -2581,7 +2601,7 @@ export function preservePendingScriptLocks( } } -async function compareDynFSElement( +export async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, ignore: (path: string, isDirectory: boolean) => boolean, @@ -2594,6 +2614,9 @@ async function compareDynFSElement( branchOverride?: string, isEls1Remote?: boolean, caseInsensitiveFs?: boolean, + // Which schedule files carry an `enabled` that is not the target's to set + // (see push's `parentOwnedScheduleEnabled`): those compare without it. + parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, ): Promise<{ changes: Change[]; localMap: Record }> { let [m1, m2] = els2 ? await Promise.all([ @@ -2785,12 +2808,28 @@ async function compareDynFSElement( ); throw error; } + if ( + parentOwnsScheduleEnabled && + getTypeStrFromPath(k) === "schedule" && + parentOwnsScheduleEnabled(k) + ) { + delete parsedV?.enabled; + delete parsedM2?.enabled; + } if (deepEqual(parsedV, parsedM2)) { continue; } } else if (k.endsWith(".yaml")) { const before = parseYaml(k, m2[k]); const after = parseYaml(k, v); + if ( + parentOwnsScheduleEnabled && + getTypeStrFromPath(k) === "schedule" && + parentOwnsScheduleEnabled(k) + ) { + delete before?.enabled; + delete after?.enabled; + } if (deepEqual(before, after)) { continue; } @@ -3174,16 +3213,41 @@ export function untrackedDatatableMigrationDeletions< ); } +/** + * Whether a pull change removes a local dbt descriptor. A dbt project's + * descriptor is optional and the remote spells "this project names none" as + * empty content, so its removal reaches the apply loop as an add or an edit + * whose content is `""` — a `deleted` change is never produced for it. + */ +function removesDbtDescriptor(change: Change): boolean { + return ( + isDbtDescriptorPath(change.path) && + ((change.name === "added" && change.content === "") || + (change.name === "edited" && change.after === "")) + ); +} + /** * `--keep-deleted`: strip every deletion from the changeset, in place, so the * sync only adds and updates. A path missing on one side is not on its own * evidence that it should go from the other — a partial clone, a scoped * checkout or an item authored in the UI all read as deletions here. + * + * A dbt descriptor the remote no longer names counts too, but only on a pull, + * where applying it removes a local file — `added` as much as `edited`, since a + * stateful pull compares `.wmill` and absence from that map says nothing about + * the working tree the apply loop deletes from. A push removes nothing: the + * descriptor is a script's content, so an empty one updates the script in place. */ -function dropDeletions(changes: Change[], keptOn: "local" | "remote"): void { - const deletions = changes.filter((c) => c.name === "deleted"); +export function dropDeletions( + changes: Change[], + keptOn: "local" | "remote", +): void { + const isDeletion = (c: Change) => + c.name === "deleted" || (keptOn === "local" && removesDbtDescriptor(c)); + const deletions = changes.filter(isDeletion); if (deletions.length === 0) return; - const kept = changes.filter((c) => c.name !== "deleted"); + const kept = changes.filter((c) => !isDeletion(c)); changes.length = 0; changes.push(...kept); log.info( @@ -3808,22 +3872,17 @@ export async function pull( const target = path.join(process.cwd(), targetPath); const stateTarget = path.join(process.cwd(), ".wmill", targetPath); - // An empty dbt descriptor is not a file: the remote spells "this project - // named no descriptor" as empty content, and writing that would put a - // Windmill file inside a project that has none. ABSENCE is the state to - // reach, so both copies are removed if present and their being missing — - // a project pulled for the first time — is the goal, not an error. The - // `.wmill` copy goes too, or the same change is reported on every pull. + // ABSENCE is the state to reach, never an empty file: writing one would + // put a Windmill file inside a project that has none. So both copies are + // removed if present, and their being missing — a project pulled for the + // first time — is the goal, not an error. The `.wmill` copy goes too, or + // the same change is reported on every pull. // // `force` covers the missing file and NOTHING else: a permission or // read-only-filesystem failure has to surface, or the pull reports // success while the old descriptor — its warehouse, its command, its // arguments — is still what runs locally. - if ( - isDbtDescriptorPath(change.path) && - ((change.name === "added" && change.content === "") || - (change.name === "edited" && change.after === "")) - ) { + if (removesDbtDescriptor(change)) { await rm(target, { force: true }); if (opts.stateful) { await rm(stateTarget, { force: true }); @@ -4523,6 +4582,86 @@ async function checkServerLockJobs( } } +// The checkout's `!inline` references of one flow (module id -> file), as +// `ZipFSElement`'s `localFlowInlineMapping` names the remote render. Empty +// when the flow has no local flow.yaml. +export async function checkoutInlineNames( + flowYamlPath: string, +): Promise> { + let flow: any; + try { + flow = await yamlParseFile(flowYamlPath); + } catch { + return {}; + } + const mapping = extractCurrentMapping( + flow?.value?.modules, + {}, + flow?.value?.failure_module, + flow?.value?.preprocessor_module, + ); + // A reference that leaves the flow folder would render the remote step + // onto another item's path; such a step keeps its summary-derived name. + for (const [id, ref] of Object.entries(mapping)) { + if (path.isAbsolute(ref) || ref.split(/[\\/]/).includes("..")) { + delete mapping[id]; + } + } + return mapping; +} + +// For a path the parent also has, a fork's export writes the parent's +// `enabled` and the backend refuses to enable the fork's copy: the file's flag +// is the parent's. A parent that cannot be listed (a fork-scoped job token) +// may own every path. Undefined when the target is not a fork. +async function parentOwnedScheduleEnabled( + workspaceId: string, +): Promise<((scheduleFilePath: string) => boolean) | undefined> { + let parentWorkspaceId: string | null | undefined; + let known = false; + try { + const { workspaces } = await wmill.listUserWorkspaces(); + const entry = workspaces?.find((w) => w.id === workspaceId); + known = entry !== undefined; + parentWorkspaceId = entry?.parent_workspace_id; + } catch { + // A fork-scoped token cannot list workspaces. + } + // No parent on record (a fork whose parent was deleted keeps its + // `wm-fork-` id): nothing defers to a parent any more. + if (known && !parentWorkspaceId) { + return undefined; + } + // Without the listing only the `wm-fork-` prefix says fork: a dev + // workspace (custom id) reached with a fork-scoped token counts as none. + if (!isForkWorkspace(workspaceId, parentWorkspaceId)) { + return undefined; + } + let parentPaths: Set | undefined; + if (parentWorkspaceId) { + try { + parentPaths = new Set(); + const perPage = 100; + for (let page = 1; ; page++) { + const batch = await wmill.listSchedules({ + workspace: parentWorkspaceId, + page, + perPage, + }); + batch.forEach((s) => parentPaths!.add(s.path)); + if (batch.length < perPage) break; + } + } catch { + parentPaths = undefined; + } + } + return (scheduleFilePath) => + parentPaths === undefined || + parentPaths.has( + removeType(scheduleFilePath, "schedule").replaceAll(SEP, "/"), + ); +} + export async function push( opts: GlobalOptions & SyncOptions & { @@ -4613,6 +4752,9 @@ export async function push( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + const parentOwnsScheduleEnabled = opts.includeSchedules + ? await parentOwnedScheduleEnabled(workspace.workspaceId) + : undefined; if (opts.lint) { log.info("Running lint validation before push..."); @@ -4676,6 +4818,10 @@ export async function push( // ignore } + // See ZipFSElement's `localFlowInlineMapping`. + const localFlowInlineMapping = (flowDir: string) => + checkoutInlineNames(path.join(process.cwd(), flowDir, "flow.yaml")); + const remote = ZipFSElement( (await downloadZip( workspace, @@ -4701,6 +4847,7 @@ export async function push( resourceTypeToIsFileset, false, parseSyncBehavior(opts.syncBehavior) >= 1, + localFlowInlineMapping, ); const local = await FSFSElement( @@ -4721,6 +4868,7 @@ export async function push( wsNameForFiles, false, // els1 (local) is not the remote source await isCaseInsensitiveFilesystem(process.cwd()), + parentOwnsScheduleEnabled, ); // Detect resources/variables that the local config flags as ws_specific @@ -5787,6 +5935,9 @@ export async function push( originalLocalPath: originalWorkspaceSpecificPath, permissionedAsContext, wsSpecific: isWsSpecific ? true : undefined, + enabledOwnedByParent: parentOwnsScheduleEnabled?.( + change.path, + ), keyPushOpts: { noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, @@ -5922,6 +6073,9 @@ export async function push( originalLocalPath: localFilePath, permissionedAsContext, wsSpecific: isAddedWsSpecific ? true : undefined, + enabledOwnedByParent: parentOwnsScheduleEnabled?.( + change.path, + ), keyPushOpts: { noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 0af4e3d4e7..7017136a66 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.800.0"; +export const VERSION = "1.803.0"; diff --git a/cli/src/core/log.ts b/cli/src/core/log.ts index e78450597f..c4cf256e9d 100644 --- a/cli/src/core/log.ts +++ b/cli/src/core/log.ts @@ -41,6 +41,12 @@ export function warnStderr(msg: unknown) { console.error(`\x1b[33m${String(msg)}\x1b[39m`); } +// A notice that must reach a log even in silent (`--json-output`) mode: +// stderr keeps stdout parseable, and a job that runs the CLI records both. +export function warnAlways(msg: unknown) { + console.error(`\x1b[33m${String(msg)}\x1b[39m`); +} + export function error(msg: unknown) { console.error(`\x1b[31m${String(msg)}\x1b[39m`); } diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 100bea6e53..2a9bb34fe0 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -115,6 +115,8 @@ Local previews exist for every entity type and don't deploy: - \`wmill flow preview -d ''\` — run a local flow.yaml. - \`wmill app dev\` — live-reload dev server for raw apps. +An argument typed as a resource takes the bare string \`"$res:"\` as its whole value (a variable takes \`"$var:"\`) — never an object wrapper like \`{"$res": ""}\`, and never a plain path. See the \`resources\` skill. + Argument shapes and per-language details live in the \`write-script-\`, \`write-flow\`, and \`raw-app\` skills. ## Keeping metadata in sync diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 4b9091ad09..8e825483fd 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5130,6 +5130,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute \`wmill flow preview -d ''\` directly — pick plausible args from the flow's input schema. +An input typed as a resource (\`format: resource-\` in the schema) takes the bare string \`"$res:"\` as its whole value — \`-d '{"db": "$res:f/databases/postgres_prod"}'\`, not \`{"db": {"$res": "..."}}\` and not a plain path. Same for a variable, with \`"$var:"\`. See the \`resources\` skill. + \`wmill flow preview\` is safe to run yourself (it does not deploy). \`wmill generate-metadata\` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's \`AGENTS.md\` opts into automatic metadata. After running it, check the regenerated \`.lock\` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only \`wmill sync push\` deploys; run it only when the user explicitly asks. ### Visual preview @@ -6202,6 +6204,41 @@ Reference other resources: } \`\`\` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema \`format: resource-\`) is passed as +the **bare string** \`$res:\` — the whole argument value. Same for a variable, with +\`$var:\`. This applies everywhere job arguments are supplied: \`wmill script run/preview\`, +\`wmill flow run/preview\`, the \`runScriptByPath\` / \`runFlowByPath\` API, a schedule's \`args\`, a +trigger's configured static args. + +\`\`\`json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +\`\`\` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with \`$res:\` / \`$var:\`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +\`\`\`json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +\`\`\` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(\`{ "gh_auth": { "token": "$var:g/all/gh_token" } }\`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL @@ -7088,11 +7125,11 @@ flow related commands - \`flow push \` - push a local flow spec. This overrides any remote versions. - \`--message \` - Deployment message - \`flow run \` - run a flow by path. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`--step \` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -7488,11 +7525,11 @@ script related commands - \`--json\` - Output as JSON (for piping to jq) - \`script show \` - show a script's content (alias for get) - \`script run \` - run a script by path - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script preview \` - preview a local script without deploying it. Supports both regular and codebase scripts. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other than the final output. Useful for scripting. - \`--tag \` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script new \` - create a new script diff --git a/cli/src/types.ts b/cli/src/types.ts index c9244a8f4c..9c517ff3c7 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -189,6 +189,8 @@ export interface PushObjOptions { keyPushOpts?: PushWorkspaceKeyOptions; /** TypeScript runtime a bare `.ts` denotes, for raw-app runnables */ defaultTs?: "bun" | "deno"; + /** schedule push into a fork: the file's `enabled` is the parent's */ + enabledOwnedByParent?: boolean; } /** @@ -217,6 +219,7 @@ export async function pushObj( wsSpecific, keyPushOpts, defaultTs, + enabledOwnedByParent, } = opts; const typeEnding = getTypeStrFromPath(p); @@ -250,7 +253,7 @@ export async function pushObj( } else if (typeEnding === "resource-type") { await pushResourceType(workspace, p, befObj, newObj); } else if (typeEnding === "schedule") { - await pushSchedule(workspace, p, befObj, newObj, permissionedAsContext); + await pushSchedule(workspace, p, befObj, newObj, permissionedAsContext, enabledOwnedByParent); } else if (typeEnding === "http_trigger") { await pushTrigger("http", workspace, p, befObj, newObj, permissionedAsContext); } else if (typeEnding === "websocket_trigger") { diff --git a/cli/test/dbt_optional_descriptor_unit.test.ts b/cli/test/dbt_optional_descriptor_unit.test.ts index f44bdc6103..685dc20e8f 100644 --- a/cli/test/dbt_optional_descriptor_unit.test.ts +++ b/cli/test/dbt_optional_descriptor_unit.test.ts @@ -7,7 +7,11 @@ import { expect, test, describe, beforeEach, afterEach } from "bun:test"; import * as fs from "node:fs"; import * as path from "node:path"; import * as os from "node:os"; -import { FSFSElement, elementsToMap } from "../src/commands/sync/sync.ts"; +import { + FSFSElement, + dropDeletions, + elementsToMap, +} from "../src/commands/sync/sync.ts"; import { listWorkspacePaths } from "../src/commands/dev/dev.ts"; import { DbtPathCollisionError, @@ -293,6 +297,47 @@ describe("a dbt project without a descriptor", () => { }); }); +// Unguarded, `--keep-deleted` lets a pull remove the very file it promises to +// keep — the descriptor's warehouse and run arguments with it. +describe("--keep-deleted and a descriptor the remote no longer names", () => { + const DESCRIPTOR = "f/analytics/analytics__dbt/wm_dbt.yaml"; + const emptied = { + name: "edited", + path: DESCRIPTOR, + before: "warehouse: wh\n", + after: "", + }; + const unknownToTheMap = { name: "added", path: DESCRIPTOR, content: "" }; + const changeset = (descriptor: unknown) => + [ + descriptor, + { name: "edited", path: "f/other/script.ts", before: "a", after: "b" }, + ] as Parameters[0]; + + test("survives a pull", () => { + const changes = changeset(emptied); + dropDeletions(changes, "local"); + expect(changes.map((c) => c.path)).toEqual(["f/other/script.ts"]); + }); + + // A stateful pull compares `.wmill`, so a descriptor missing from the compared + // map can still be a working-tree file holding real settings. + test("survives a pull that compared a map without it", () => { + const changes = changeset(unknownToTheMap); + dropDeletions(changes, "local"); + expect(changes.map((c) => c.path)).toEqual(["f/other/script.ts"]); + }); + + test("is still pushed, deleting no remote item", () => { + const changes = changeset(emptied); + dropDeletions(changes, "remote"); + expect(changes.map((c) => c.path)).toEqual([ + DESCRIPTOR, + "f/other/script.ts", + ]); + }); +}); + // `wmill dev` walks basenames, and a dbt script is a DIRECTORY whose descriptor // may not exist — so it is recognized by the project folder or not at all. The // walk must also stop there: the project's own `.sql` models match the script diff --git a/cli/test/push_diff_convergence_unit.test.ts b/cli/test/push_diff_convergence_unit.test.ts new file mode 100644 index 0000000000..14e248c03e --- /dev/null +++ b/cli/test/push_diff_convergence_unit.test.ts @@ -0,0 +1,298 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import JSZip from "jszip"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, sep } from "node:path"; +import { + checkoutInlineNames, + compareDynFSElement, + ZipFSElement, +} from "../src/commands/sync/sync.ts"; + +// The differ also reads the working tree (shared lockfiles, dependency files); +// an empty one keeps that out of the picture. +const originalCwd = process.cwd(); +beforeAll(() => { + process.chdir(mkdtempSync(join(tmpdir(), "wmill-push-diff-"))); +}); +afterAll(() => { + process.chdir(originalCwd); +}); + +// A push is only useful when a second run of it finds nothing left to do. +// These pin the two shapes that used to be listed on every run of a push into +// a fork while the push itself either applied nothing or aborted. + +type Mock = { + isDirectory: boolean; + path: string; + getContentText(): Promise; + getChildren(): AsyncIterable; +}; + +// Both sides of the differ use the OS separator; fixtures are written with +// "/" and rows are read back the same way. +const osPath = (p: string) => p.split("/").join(sep); +const slashPath = (p: string) => p.split(sep).join("/"); + +function local(files: Record): Mock { + return { + isDirectory: true, + path: "", + async getContentText() { + return ""; + }, + async *getChildren() { + for (const [path, content] of Object.entries(files)) { + yield { + isDirectory: false, + path: osPath(path), + async getContentText() { + return content; + }, + async *getChildren() {}, + }; + } + }, + }; +} + +const noIgnore = () => false; + +async function diff( + localEl: Mock, + remoteEl: Mock, + skips: Record, + parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, +) { + const { changes } = await compareDynFSElement( + localEl as any, + remoteEl as any, + noIgnore, + false, + skips as any, + true, + [], + false, + undefined, + undefined, + false, + false, + parentOwnsScheduleEnabled, + ); + return changes.map((c) => `${c.name} ${slashPath(c.path)}`); +} + +const SCHEDULE = (enabled: string) => + `summary: nightly\nargs: {}\nenabled: ${enabled}\nis_flow: true\nschedule: 0 0 0 * * *\nscript_path: f/mail/flow\ntimezone: UTC\n`; + +test("push into a fork: a schedule the parent also has compares without `enabled`", async () => { + const remote = local({ "f/mail/nightly.schedule.yaml": SCHEDULE("true") }); + const skips = { includeSchedules: true }; + // The parent has `f/mail/nightly`; any other schedule is the fork's own. + const parentHas = (filePath: string) => + slashPath(filePath) === "f/mail/nightly.schedule.yaml"; + + // Not a fork: `enabled` is compared like any other field. + expect( + await diff( + local({ "f/mail/nightly.schedule.yaml": SCHEDULE("false") }), + remote, + skips, + ), + ).toEqual(["edited f/mail/nightly.schedule.yaml"]); + expect( + await diff( + local({ "f/mail/nightly.schedule.yaml": SCHEDULE("false") }), + remote, + skips, + parentHas, + ), + ).toEqual([]); + // The key being absent is the same case as it differing. + expect( + await diff( + local({ + "f/mail/nightly.schedule.yaml": SCHEDULE("false").replace( + "enabled: false\n", + "", + ), + }), + remote, + skips, + parentHas, + ), + ).toEqual([]); + // Only `enabled` is set aside. + expect( + await diff( + local({ + "f/mail/nightly.schedule.yaml": SCHEDULE("false").replace( + "0 0 0 * * *", + "0 0 1 * * *", + ), + }), + remote, + skips, + parentHas, + ), + ).toEqual(["edited f/mail/nightly.schedule.yaml"]); + // A schedule only the fork has keeps toggling from the file. + expect( + await diff( + local({ "f/mail/fork_only.schedule.yaml": SCHEDULE("true") }), + local({ "f/mail/fork_only.schedule.yaml": SCHEDULE("false") }), + skips, + parentHas, + ), + ).toEqual(["edited f/mail/fork_only.schedule.yaml"]); +}); + +const SUMMARY = "process one mail end-to-end (spam check, classify)"; + +function remoteFlow(content: string) { + const zip = new JSZip(); + zip.file( + "f/mail/flow_v2.flow.json", + JSON.stringify({ + summary: "Flow V2", + description: "", + value: { + modules: [ + { + id: "a", + summary: SUMMARY, + value: { + type: "rawscript", + content, + input_transforms: {}, + language: "python3", + }, + }, + ], + }, + schema: { type: "object", properties: {} }, + }), + // The backend's archive carries no directory entries. + { createFolders: false }, + ); + return zip; +} + +function localFlow(content: string) { + return local({ + "f/mail/flow_v2.flow/flow.yaml": `summary: Flow V2\ndescription: ''\nvalue:\n modules:\n - id: a\n summary: ${SUMMARY}\n value:\n type: rawscript\n content: '!inline process_mail.inline_script.py'\n input_transforms: {}\n language: python3\nschema:\n type: object\n properties: {}\n`, + "f/mail/flow_v2.flow/process_mail.inline_script.py": content, + }); +} + +// The checkout's `!inline` references, as `push` reads them from its flow.yaml. +const checkoutNames = async (flowDir: string) => + slashPath(flowDir) === "f/mail/flow_v2.flow" + ? { a: "process_mail.inline_script.py" } + : {}; + +test("push: an inline script the checkout names differently from the step summary is not a rename", async () => { + const skips = { includeSchedules: false }; + const render = (content: string, withCheckout: boolean) => + ZipFSElement( + remoteFlow(content), + true, + "bun", + {}, + {}, + false, + true, + withCheckout ? checkoutNames : undefined, + ) as any; + + // Same content, file named by hand: three rows before, none after. + expect( + await diff( + localFlow("def main():\n return 1\n"), + render("def main():\n return 1\n", false), + skips, + ), + ).toEqual([ + "deleted f/mail/flow_v2.flow/process_one_mail_end-to-end_(spam_check,_classify).inline_script.py", + "edited f/mail/flow_v2.flow/flow.yaml", + "added f/mail/flow_v2.flow/process_mail.inline_script.py", + ]); + expect( + await diff( + localFlow("def main():\n return 1\n"), + render("def main():\n return 1\n", true), + skips, + ), + ).toEqual([]); + + // A real edit is still one. + expect( + await diff( + localFlow("def main():\n return 2\n"), + render("def main():\n return 1\n", true), + skips, + ), + ).toEqual(["edited f/mail/flow_v2.flow/process_mail.inline_script.py"]); +}); + +test("push: a checkout name that collides with another step's summary-derived name keeps two files", async () => { + const zip = new JSZip(); + zip.file( + "f/mail/flow_v2.flow.json", + JSON.stringify({ + summary: "Flow V2", + description: "", + value: { + modules: [ + { + id: "a", + summary: SUMMARY, + value: { + type: "rawscript", + content: "a", + input_transforms: {}, + language: "python3", + }, + }, + { + id: "b", + summary: "process_mail", + value: { + type: "rawscript", + content: "b", + input_transforms: {}, + language: "python3", + }, + }, + ], + }, + schema: { type: "object", properties: {} }, + }), + { createFolders: false }, + ); + // Nothing local: every rendered file is a "deleted" row, one per path. + const rows = await diff( + local({}), + ZipFSElement(zip, true, "bun", {}, {}, false, true, checkoutNames) as any, + { includeSchedules: false }, + ); + expect(rows.filter((r) => r.endsWith(".py")).sort()).toEqual([ + "deleted f/mail/flow_v2.flow/process_mail.inline_script.py", + "deleted f/mail/flow_v2.flow/process_one_mail_end-to-end_(spam_check,_classify).inline_script.py", + ]); +}); + +test("push: checkout inline names stay inside the flow folder", async () => { + const flowYaml = join(process.cwd(), "flow.yaml"); + writeFileSync( + flowYaml, + `summary: x\nvalue:\n modules:\n - id: a\n value:\n type: rawscript\n content: '!inline a.inline_script.py'\n language: python3\n - id: b\n value:\n type: rawscript\n content: '!inline ../shared/b.py'\n language: python3\n - id: c\n value:\n type: rawscript\n content: '!inline /tmp/c.py'\n language: python3\n`, + ); + expect(await checkoutInlineNames(flowYaml)).toEqual({ + a: "a.inline_script.py", + }); + expect( + await checkoutInlineNames(join(process.cwd(), "missing.yaml")), + ).toEqual({}); +}); diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 5d7f0b2c29..5cef3c9364 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,9 +4,9 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 28 registered actions across fourteen features (`ai_session`, `ai_chat`, +It currently carries 32 registered actions across fifteen features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`, -`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`). Nearly all of the +`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, `sso_groups_claim`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 20592d0371..6fd6413d59 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.800.0", + "version": "1.803.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.800.0", + "version": "1.803.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 08d8c427ff..f8f952ede6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.800.0", + "version": "1.803.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/aiStore.test.ts b/frontend/src/lib/aiStore.test.ts index fed6147b62..be8980e93a 100644 --- a/frontend/src/lib/aiStore.test.ts +++ b/frontend/src/lib/aiStore.test.ts @@ -35,6 +35,24 @@ describe('setCopilotInfo legacy /thinking migration', () => { expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) }) + it('keeps the models but turns the assistant off when the workspace disabled it', () => { + setCopilotInfo({ + providers: { + anthropic: { + resource_path: 'u/admin/anthropic', + models: ['claude-sonnet-4-6'] + } + }, + copilot_disabled: true + }) + + const info = get(copilotInfo) + expect(info.enabled).toBe(false) + expect(info.workspaceDisabled).toBe(true) + // The providers still describe what AI agent steps can run on. + expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) + }) + it('defaults provider web search on unless explicitly disabled', () => { setCopilotInfo({ providers: { diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 2d6a98c625..6ac98d69e8 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -41,6 +41,10 @@ export const copilotSessionModel = writable( export const copilotInfo = writable<{ enabled: boolean + // The workspace hid the assistant (`ai_config.copilot_disabled`). `enabled` is then false + // whatever the providers say, and the AI entry points that nudge "configure AI" when + // `enabled` is off render nothing at all instead. + workspaceDisabled: boolean codeCompletionModel?: AIProviderModel defaultModel?: AIProviderModel metadataModel?: AIProviderModel @@ -56,6 +60,7 @@ export const copilotInfo = writable<{ freeTier?: FreeTierInfo }>({ enabled: false, + workspaceDisabled: false, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, @@ -71,7 +76,7 @@ export const copilotInfo = writable<{ aiUserDisabled.subscribe((disabled) => { copilotInfo.update((info) => ({ ...info, - enabled: info.aiModels.length > 0 && !disabled + enabled: info.aiModels.length > 0 && !disabled && !info.workspaceDisabled })) }) @@ -126,9 +131,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { return model }) + const workspaceDisabled = aiConfig.copilot_disabled === true copilotInfo.set({ - // Providers are configured; the per-user opt-out is the only thing that can gate it off. - enabled: !get(aiUserDisabled), + // Providers are configured; only the workspace or per-user opt-outs can gate it off. + enabled: !workspaceDisabled && !get(aiUserDisabled), + workspaceDisabled, // Strip the deprecated /thinking suffix from the configured model slots too, // otherwise a workspace whose default still carries it sends an invalid model id. codeCompletionModel: stripModelSuffix(aiConfig.code_completion_model), @@ -146,6 +153,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { copilotInfo.set({ enabled: false, + workspaceDisabled: aiConfig.copilot_disabled === true, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, diff --git a/frontend/src/lib/components/FolderEditor.svelte b/frontend/src/lib/components/FolderEditor.svelte index dede17b953..e8f37a625c 100644 --- a/frontend/src/lib/components/FolderEditor.svelte +++ b/frontend/src/lib/components/FolderEditor.svelte @@ -6,19 +6,23 @@ FolderService, UserService, GranularAclService, - GroupService + GroupService, + type User } from '$lib/gen' - import TableCustom from './TableCustom.svelte' + import DataTable from './table/DataTable.svelte' + import Head from './table/Head.svelte' + import Row from './table/Row.svelte' + import Cell from './table/Cell.svelte' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' - import { Alert, Button, Drawer, DrawerContent } from './common' + import { Alert, Button } from './common' import Skeleton from './common/skeleton/Skeleton.svelte' - import GroupEditor from './GroupEditor.svelte' + import GroupEditorDrawer from './GroupEditorDrawer.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import { ArrowDown, ArrowUp, Eye, Plus, Trash } from 'lucide-svelte' + import { ArrowDown, ArrowUp, Eye, Pen, Plus, Trash } from 'lucide-svelte' import Label from './Label.svelte' import { sendUserToast } from '$lib/toast' - import { createEventDispatcher, untrack } from 'svelte' + import { onMount, tick, untrack } from 'svelte' import Select from './select/Select.svelte' import { safeSelectItems } from './select/utils.svelte' import TextInput from './text_input/TextInput.svelte' @@ -28,97 +32,251 @@ import CollapseLink from './CollapseLink.svelte' import LabelsInput from './LabelsInput.svelte' import Badge from './common/badge/Badge.svelte' + import InputError from './InputError.svelte' + import Popover from './meltComponents/Popover.svelte' + import { deepEqual } from 'fast-equals' + import { + folderPermissionDiff, + isFolderDraftDirty, + type FolderDraft, + type FolderRole + } from '$lib/folderDraft' + + const VALID_FOLDER_NAME = /^[a-zA-Z_0-9-]+$/ + + const ROLE_TOOLTIPS = { + viewer: + 'A viewer of a folder has read-only access to all the elements (scripts/flows/apps/schedules/resources/variables) inside the folder', + writer: + 'A writer of a folder has read AND write access to all the elements (scripts/flows/apps/schedules/resources/variables) inside the folder', + admin: + 'An admin of a folder has read AND write access to all the elements inside the folders and can manage the permissions as well as add new admins' + } + + const MEMBERS_EXPLAINER = + "A member is a user or group with a role on this folder. The role applies to every script, flow, app, resource, variable and schedule inside it: viewers can read them, writers can also edit them, and admins can additionally manage the folder's members." + + // Edits mutate `draft` only; `save()` is the sole writer to the backend, and `baseline` is + // what the folder held when it was loaded, so comparing the two gives both the dirty state + // and the permission calls to replay. Both live in `folderDraft.ts`, with tests. + type Role = FolderRole interface Props { + /** In `new` mode this is the name being typed, hence bindable. */ name: string + mode?: 'edit' | 'new' + /** Drives the parent drawer's Save button, which lives above this component. */ + onCanSaveChange?: (canSave: boolean) => void + /** Drives the parent drawer's discard confirmation on close. Unlike `canSave` + * this stays true for edits that cannot be saved yet (an invalid rule, a name + * already taken) — closing would still throw them away. */ + onUnsavedChange?: (unsaved: boolean) => void + /** False while Save would create rather than update, which an `edit` drawer reaches + * when the folder turns out not to exist. The drawer labels itself from this. */ + onExistsChange?: (exists: boolean) => void + /** Edit a folder of this workspace rather than the active one. The folder picker + * can be aimed elsewhere (the project import wizard picks a destination workspace + * before entering it), and the folder must be written where it was listed. */ + workspace?: string } - let { name }: Props = $props() - let can_write = $state(false) + let { + name = $bindable(), + mode = 'edit', + onCanSaveChange, + onUnsavedChange, + onExistsChange, + workspace + }: Props = $props() - type Role = 'viewer' | 'writer' | 'admin' - let folder: Folder | undefined - let perms: { owner_name: string; role: Role }[] | undefined = $state(undefined) - let usernames: string[] = $state([]) - let groups: string[] = $state([]) - let ownerItem: string = $state('') + const targetWorkspace = $derived(workspace ?? $workspaceStore ?? '') + const aimedElsewhere = $derived(!!workspace && workspace !== $workspaceStore) - let newGroup: Drawer | undefined = $state(undefined) - let viewGroup: Drawer | undefined = $state(undefined) - - async function loadUsernames(): Promise { - usernames = await UserService.listUsernames({ workspace: $workspaceStore! }) - } - - async function loadGroups(): Promise { - groups = await GroupService.listGroupNames({ workspace: $workspaceStore! }) - } - - async function load() { - loadUsernames() - loadGroups() - await loadFolder() - } - - async function addToFolder() { - await GranularAclService.addGranularAcls({ - workspace: $workspaceStore ?? '', - path: name, - kind: 'folder', - requestBody: { - owner: (ownerKind == 'user' ? 'u/' : 'g/') + ownerItem - } - }) - ownerItem = '' - loadFolder() - } - - let folderNotFound: boolean | undefined = $state(undefined) - - async function loadFolder(): Promise { - try { - folder = await FolderService.getFolder({ workspace: $workspaceStore!, name }) - summary = folder.summary ?? '' - labels = [...(folder.labels ?? [])] - defaultPermissionedAs = (folder.default_permissioned_as ?? []).map((r) => ({ ...r })) - can_write = - $userStore != undefined && - (folder?.owners.includes('u/' + $userStore.username) || - ($userStore.is_admin ?? false) || - ($userStore.is_super_admin ?? false) || - $userStore.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1) - - perms = Array.from( - new Set( - Object.entries(folder?.extra_perms ?? {}) - .map((x) => x[0]) - .concat(folder?.owners ?? []) - ) - ).map((x) => { - return { - owner_name: x, - role: getRole(x) + // `$userStore` describes the workspace the app is *in*. Aimed at another one it answers + // the wrong question — a folder admin there would get read-only controls, and a + // non-member would get write ones — so resolve the membership of the workspace being + // edited. `whoami` returns group names unprefixed; `owners` holds them `g/`-prefixed. + let targetUser: User | undefined = $state(undefined) + const membership = $derived.by(() => { + if (!aimedElsewhere) { + return $userStore + ? { + username: $userStore.username, + is_admin: $userStore.is_admin ?? false, + is_super_admin: $userStore.is_super_admin ?? false, + pgroups: $userStore.pgroups ?? [], + groups: $userStore.groups ?? [] + } + : undefined + } + return targetUser + ? { + username: targetUser.username, + is_admin: targetUser.is_admin ?? false, + is_super_admin: targetUser.is_super_admin ?? false, + pgroups: (targetUser.groups ?? []).map((g) => 'g/' + g), + groups: targetUser.groups ?? [] } - }) - reloadHistory++ - } catch (e) { - folderNotFound = true + : undefined + }) + + async function loadTargetUser(): Promise { + if (!aimedElsewhere || !workspace) return + try { + targetUser = await UserService.whoami({ workspace }) + } catch { + // Not a member, or the call failed: no membership means read-only controls, + // which is the safe reading — the write would be refused anyway. + targetUser = undefined } } - // --- default_permissioned_as rules editor --- - let defaultPermissionedAs: FolderDefaultPermissionedAs = $state([]) + let can_write = $state(false) + let folder: Folder | undefined + let usernames: string[] = $state([]) + let groups: string[] = $state([]) + let folderNames: string[] = $state([]) + let ownerItem: string = $state('') + + let groupEditorDrawer: GroupEditorDrawer | undefined = $state(undefined) + let addMemberPopover: Popover | undefined = $state(undefined) + let nameInput: TextInput | undefined = $state(undefined) + + let baseline: FolderDraft | undefined = $state(undefined) + // Empty, not `emptyDraft()`: that one seeds the caller as an admin, which is true of a + // folder being created and a lie about one whose read failed. Every path that wants the + // seeded row calls `emptyDraft()` itself. + let draft: FolderDraft = $state({ + summary: '', + labels: [], + defaultPermissionedAs: [], + perms: [] + }) + let labelsInput: LabelsInput | undefined = $state() + let pendingLabel = $state('') + let folderNotFound: boolean | undefined = $state(undefined) + let loaded = $state(false) + + // A name typed in `new` mode, and one whose folder turned out not to exist, both + // end up at `createFolder` on save. + const isNew = $derived(mode === 'new' || folderNotFound === true) + + function emptyDraft(): FolderDraft { + return { + summary: '', + labels: [], + defaultPermissionedAs: [], + // The backend makes the creator an owner whatever we send, so the table + // shows that from the start rather than after the first reload. + perms: membership ? [{ owner_name: 'u/' + membership.username, role: 'admin' as Role }] : [] + } + } + + function setDraft(value: FolderDraft) { + baseline = structuredClone(value) + draft = structuredClone(value) + } + + async function loadUsernames(): Promise { + usernames = await UserService.listUsernames({ workspace: targetWorkspace }) + } + + async function loadGroups(): Promise { + groups = await GroupService.listGroupNames({ workspace: targetWorkspace }) + } + + async function loadFolderNames(): Promise { + folderNames = await FolderService.listFolderNames({ workspace: targetWorkspace }) + } + + /** Fills a picker or a validation list. The editor is usable before these land, so they + * run alongside the folder read — but a rejection has to be reported: unhandled, it + * leaves the list silently empty and duplicate names stop being caught. */ + function loadAside(load: () => Promise): void { + load().catch((e) => sendUserToast(e?.body ?? String(e), true)) + } + + async function load() { + loadAside(loadUsernames) + loadAside(loadGroups) + // Before the folder read: `can_write` is computed from this membership. + await loadTargetUser() + if (mode === 'new') { + loadAside(loadFolderNames) + can_write = true + setDraft(emptyDraft()) + loaded = true + } else { + await loadFolder() + } + } + + function grant(close: () => void) { + const owner = (ownerKind == 'user' ? 'u/' : 'g/') + ownerItem + if (!draft.perms.some((p) => p.owner_name === owner)) { + draft.perms.push({ owner_name: owner, role: newMemberRole }) + } + ownerItem = '' + close() + } + + /** `baselineOnly` re-reads the folder without touching the draft: after a save that + * committed some of its calls and then failed, the baseline must become what the server + * actually holds while the draft stays the user's intent — the applied changes then stop + * counting as dirty, and the ones still missing stay dirty and retryable. */ + async function loadFolder(opts?: { baselineOnly?: boolean }): Promise { + const apply = (value: FolderDraft) => + opts?.baselineOnly ? (baseline = structuredClone(value)) : setDraft(value) + try { + folder = await FolderService.getFolder({ workspace: targetWorkspace, name }) + folderNotFound = false + can_write = + membership != undefined && + (folder?.owners.includes('u/' + membership.username) || + membership.is_admin || + membership.is_super_admin || + membership.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1) + + apply({ + summary: folder.summary ?? '', + labels: [...(folder.labels ?? [])], + defaultPermissionedAs: (folder.default_permissioned_as ?? []).map((r) => ({ ...r })), + perms: Array.from( + new Set( + Object.entries(folder?.extra_perms ?? {}) + .map((x) => x[0]) + .concat(folder?.owners ?? []) + ) + ).map((x) => ({ owner_name: x, role: getRole(x) })) + }) + reloadHistory++ + } catch (e) { + // Only a folder that is genuinely absent may replace the draft — it can be created + // from here, so the editor opens on an empty one rather than a dead end. Any other + // failure (network, 5xx) must leave the draft alone: overwriting it here would + // discard the user's edits and clear `unsaved` with them. + if (e?.status === 404) { + folderNotFound = true + can_write = true + apply(emptyDraft()) + } else { + sendUserToast(e?.body ?? String(e), true) + } + } finally { + loaded = true + } + } const restricted = $derived( - isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + isDemoWorkspaceRestricted(targetWorkspace, membership?.is_admin, membership?.is_super_admin) ) const canEditDefaults = $derived( can_write && !restricted && - ($userStore?.is_admin || - $userStore?.is_super_admin || - ($userStore?.groups ?? []).includes('wm_deployers')) + (membership?.is_admin || + membership?.is_super_admin || + (membership?.groups ?? []).includes('wm_deployers')) ) function isValidGlob(glob: string): boolean { @@ -135,58 +293,47 @@ return /^[ug]\/.+/.test(value) || value.includes('@') } - // Split a permissioned_as value like "u/alice" or "g/prod" into its kind and name. - function ruleKind(value: string): 'user' | 'group' { + // Split an owner value like "u/alice" or "g/prod" into its kind and name. + function ownerKindOf(value: string): 'user' | 'group' { return value.startsWith('g/') ? 'group' : 'user' } - function ruleName(value: string): string { + function ownerNameOf(value: string): string { if (value.startsWith('u/') || value.startsWith('g/')) return value.slice(2) return value } function setRulePermissionedAs(idx: number, kind: 'user' | 'group', name: string) { const prefix = kind === 'user' ? 'u/' : 'g/' - defaultPermissionedAs[idx].permissioned_as = prefix + name + draft.defaultPermissionedAs[idx].permissioned_as = prefix + name } + // Only blocks a save for someone who can see the rules. The backend accepts values this + // rejects (`u/` alone passes `validate_default_permissioned_as`), so a folder admin who + // is not a workspace admin could otherwise meet a permanently disabled Save with no rule + // on screen to explain it. const defaultRulesInvalid = $derived( - defaultPermissionedAs.some( - (r) => !isValidGlob(r.path_glob) || !isValidPermissionedAs(r.permissioned_as) - ) + canEditDefaults && + draft.defaultPermissionedAs.some( + (r) => !isValidGlob(r.path_glob) || !isValidPermissionedAs(r.permissioned_as) + ) ) function addDefaultRule() { - defaultPermissionedAs = [...defaultPermissionedAs, { path_glob: '**', permissioned_as: '' }] + draft.defaultPermissionedAs = [ + ...draft.defaultPermissionedAs, + { path_glob: '**', permissioned_as: '' } + ] } function removeDefaultRule(idx: number) { - defaultPermissionedAs = defaultPermissionedAs.filter((_, i) => i !== idx) + draft.defaultPermissionedAs = draft.defaultPermissionedAs.filter((_, i) => i !== idx) } function moveDefaultRule(idx: number, delta: -1 | 1) { - const next = [...defaultPermissionedAs] + const next = [...draft.defaultPermissionedAs] const target = idx + delta if (target < 0 || target >= next.length) return ;[next[idx], next[target]] = [next[target], next[idx]] - defaultPermissionedAs = next - } - - async function saveDefaultRules() { - if (defaultRulesInvalid) { - sendUserToast('Some rules have invalid globs or permissioned_as values', true) - return - } - try { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { default_permissioned_as: defaultPermissionedAs } - }) - sendUserToast('Default permissioned_as rules updated') - dispatch('update') - loadFolder() - } catch (e) { - sendUserToast(e.body ?? String(e), true) - } + draft.defaultPermissionedAs = next } function getRole(x: string): Role { @@ -204,51 +351,208 @@ } let ownerKind: 'user' | 'group' = $state('user') - let groupCreated: string | undefined = $state(undefined) - let newGroupName: string = $state('') - let summary: string = $state('') - let labels: string[] | undefined = $state(undefined) + let newMemberRole: Role = $state('viewer') - async function saveLabels() { + // Set when the group editor is opened from the add-member form, so that saving returns + // there. Opened from a member row instead, that group is already a member and reopening + // the form on it would offer to add it twice. + let groupEditorInterruptedPicker = false + + function openGroupEditor(groupName: string, fromPicker: boolean) { + groupEditorInterruptedPicker = fromPicker + if (groupName) groupEditorDrawer?.initEdit(groupName) + else groupEditorDrawer?.initNew() + } + + async function onGroupSaved(groupName: string) { + // The group has to be in `groups` before the picker reopens, or the value set below + // has no matching item to show. try { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { labels: labels ?? [] } - }) - sendUserToast('Folder labels updated') - dispatch('update') + await loadGroups() } catch (e) { - sendUserToast(e.body ?? String(e), true) - loadFolder() + sendUserToast(e?.body ?? String(e), true) + } + if (!groupEditorInterruptedPicker) return + // Editing a group was a detour from adding a member: come back to the form on that + // group so the interrupted job can be finished. + ownerKind = 'group' + ownerItem = groupName + addMemberPopover?.open() + } + + // Guarded on `mode`, not `isNew`: the name field is rendered only in `new` mode, so on the + // not-found branch there is no input to annotate and no name the user could correct. + const nameError = $derived( + mode !== 'new' + ? '' + : !name + ? '' + : !VALID_FOLDER_NAME.test(name) + ? 'Folder name can only contain alphanumeric characters, underscores, and hyphens' + : folderNames.includes(name) + ? 'A folder with this name already exists' + : '' + ) + + // `create_folder` folds the caller into `owners` with write whatever the payload says, so + // on create their own row is fixed: offering to demote or remove it would be a change the + // backend silently discards. + // An invalid rule disables Save, so the section holding it is held open rather than merely + // opened once: collapsing it would hide the only explanation for the disabled button. + let defaultRulesOpen = $state(false) + + function isFixedCreatorRow(owner: string): boolean { + return isNew && owner === 'u/' + membership?.username + } + + // The label input holds typed text until Enter or a blur, and that text is an edit like + // any other: it has to count as dirty here, or Save stays disabled when it is the only + // change and closing drops it without asking. `save()` flushes it into `draft.labels`. + const dirty = $derived(isFolderDraftDirty(draft, baseline) || pendingLabel !== '') + // A typed name is progress too, even before any other field is touched. + const unsaved = $derived(dirty || (mode === 'new' && !!name)) + + $effect(() => { + onCanSaveChange?.( + isNew + ? loaded && !!name && !nameError && !restricted && !defaultRulesInvalid + : can_write && dirty && !defaultRulesInvalid + ) + }) + + $effect(() => { + onUnsavedChange?.(unsaved) + }) + + $effect(() => { + onExistsChange?.(!isNew) + }) + + /** Replays the permission rows the user changed. `updateFolder` could write + * `owners`/`extra_perms` wholesale in the same call as the settings, but it only + * logs a single "update owners"/"update acl" entry, so the permission history + * would stop naming who was granted what. The diff itself is in `folderDraft.ts`. */ + async function applyPermissionChanges(next: FolderDraft['perms'], prev: FolderDraft['perms']) { + const workspace = targetWorkspace + const callerOwners = membership + ? ['u/' + membership.username, ...(membership.pgroups ?? [])] + : [] + for (const call of folderPermissionDiff(prev, next, callerOwners)) { + switch (call.kind) { + case 'grantAdmin': + await FolderService.addOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner } + }) + break + case 'demoteAdmin': + await FolderService.removeOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner, write: call.write } + }) + break + case 'setAcl': + await GranularAclService.addGranularAcls({ + workspace, + path: name, + kind: 'folder', + requestBody: { owner: call.owner, write: call.write } + }) + break + case 'remove': + // Sequential, and `removeowner` first: the write policy refuses it when the + // member being removed is the caller's last admin handle. Failing there leaves + // the folder untouched, where the other order strands a member with no grant + // but still in `owners`. + await FolderService.removeOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner } + }) + await GranularAclService.removeGranularAcls({ + workspace, + path: name, + kind: 'folder', + requestBody: { owner: call.owner } + }) + break + } } } - async function addGroup() { - await GroupService.createGroup({ - workspace: $workspaceStore ?? '', - requestBody: { name: newGroupName } - }) - groupCreated = newGroupName - $userStore?.folders?.push(newGroupName) - loadGroups() - ownerItem = newGroupName + export async function save(): Promise<{ name: string; created: boolean } | undefined> { + // Clicking Save blurs the label input, which commits its text on a delay — after the + // snapshot below. Take the label first or it is dropped as the drawer closes. + labelsInput?.flushPendingLabel() + const next = $state.snapshot(draft) as FolderDraft + const prev = baseline as FolderDraft + // Captured before the write: an edit-branch save reloads, which clears `folderNotFound`. + const created = isNew + try { + if (created) { + await FolderService.createFolder({ + workspace: targetWorkspace, + requestBody: { + name, + summary: next.summary, + labels: next.labels, + default_permissioned_as: next.defaultPermissionedAs, + owners: next.perms.filter((p) => p.role === 'admin').map((p) => p.owner_name), + extra_perms: Object.fromEntries( + next.perms.map((p) => [p.owner_name, p.role !== 'viewer']) + ) + } + }) + sendUserToast(`Folder ${name} created`) + } else { + const requestBody: { + summary?: string + labels?: string[] + default_permissioned_as?: FolderDefaultPermissionedAs + } = {} + if (next.summary !== prev.summary) requestBody.summary = next.summary + if (!deepEqual(next.labels, prev.labels)) requestBody.labels = next.labels + if (!deepEqual(next.defaultPermissionedAs, prev.defaultPermissionedAs)) { + requestBody.default_permissioned_as = next.defaultPermissionedAs + } + if (Object.keys(requestBody).length > 0) { + await FolderService.updateFolder({ workspace: targetWorkspace, name, requestBody }) + } + await applyPermissionChanges(next.perms, prev.perms) + await loadFolder() + sendUserToast('Folder updated') + } + return { name, created } + } catch (e) { + sendUserToast(e.body ?? String(e), true) + // A failed create is not proof the folder is absent: `create_folder` commits before a + // git-sync step that can still fail the request. Only the name conflict says it was + // never written. Report rather than resolve — a folder found by name may be someone + // else's, and adopting it would send this draft's writes there. + const nameTaken = String(e?.body ?? '').includes('already exists') + if (created && !nameTaken) { + sendUserToast(`Folder ${name} may have been created anyway — reopen it to check`, true) + } + // Reconcile after any edit-path failure rather than tracking which calls landed: + // these handlers commit before a git-sync step that can still fail the request, so + // a rejection is not proof nothing was written. The baseline moves to what the + // server now holds and the draft stays, so a retry re-sends only what is missing. + if (!created) await loadFolder({ baselineOnly: true }) + return undefined + } } - const dispatch = createEventDispatcher() - - async function updateFolder() { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { summary } - }) - sendUserToast('Folder summary updated') - dispatch('update') - loadFolder() - } + // The stores are read only to wait until they are populated, and the load runs once: this + // editor holds an unsaved draft, and the layout re-`set`s `$userStore` periodically — a + // second `load()` would overwrite the draft with the server's state and lose the edits + // silently, `unsaved` included. The drawer remounts this component per opening. + let loadStarted = false $effect.pre(() => { + if (loadStarted) return if ($workspaceStore && $userStore) { + loadStarted = true untrack(() => { load() }) @@ -256,47 +560,41 @@ }) let reloadHistory = $state(0) + + onMount(async () => { + if (mode !== 'new') return + // The editor is remounted per drawer opening, so mount is the moment the + // create form appears; the input only exists after the first render. + await tick() + nameInput?.focus() + }) - - { - newGroup?.closeDrawer() - groupCreated = undefined - }} - > - {#if !groupCreated} -
- - -
- {:else} - - {/if} - - - - - - - - +
- + {/if} + +
{#if can_write} - + (pendingLabel = v)} + /> {:else}
- {#each labels ?? [] as label (label)} + {#each draft.labels as label (label)} {label} {:else} No labels @@ -319,257 +622,261 @@
- - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + /> + + {:else} +
+

Enable Windmill AI in the workspace settings

+
+ {/if} + + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index e61ab5cbf2..f399726988 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,5 +1,5 @@ - - {#snippet trigger()} - +{#if !$copilotInfo.workspaceDisabled} + + {#snippet trigger()} - - {:else} -
-

Enable Windmill AI in the workspace settings

-
- {/if} - - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + > + Generate + + + {:else} +
+

Enable Windmill AI in the workspace settings

+
+ {/if} + + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ScriptFix.svelte b/frontend/src/lib/components/copilot/ScriptFix.svelte index d6ccb26ab8..f586ad1aaf 100644 --- a/frontend/src/lib/components/copilot/ScriptFix.svelte +++ b/frontend/src/lib/components/copilot/ScriptFix.svelte @@ -77,7 +77,7 @@ const sessionScopedManager = getContext('aiChatManager') -{#if SUPPORTED_LANGUAGES.has(lang)} +{#if SUPPORTED_LANGUAGES.has(lang) && !$copilotInfo.workspaceDisabled} {#if sessionScopedManager} - {:else} + {:else if !$copilotInfo.workspaceDisabled} togglePanel() })} -{:else} +{:else if !$copilotInfo.workspaceDisabled} {#snippet trigger()} {@render button({ onPress: () => togglePanel() })} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index e4dba770eb..fd95613f2e 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -44,8 +44,12 @@ const isAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) const hasCopilot = $derived($copilotInfo.enabled) + // Another tab is running a turn on this session: transcript stays readable, + // composer locks, and the chat re-reads the shared record when the turn ends. + const runHeldElsewhere = $derived(aiChatManager.runHeldElsewhere) const disabled = $derived( forceDisabled || + runHeldElsewhere || !hasCopilot || (aiChatManager.mode === AIMode.SCRIPT && aiChatManager.scriptEditorOptions?.lang && @@ -58,19 +62,25 @@ const disabledMessage = $derived( forceDisabled ? forceDisabledMessage - : freeTierExhausted - ? '' - : !hasCopilot - ? $aiUserDisabled - ? 'Windmill AI is disabled in your account settings' - : isAdmin - ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` - : 'Ask an admin to enable Windmill AI in this workspace to use this chat' - : aiChatManager.mode === AIMode.SCRIPT && - aiChatManager.scriptEditorOptions?.lang && - !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) - ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` - : '' + : runHeldElsewhere + ? // The typing indicator and the composer placeholder already carry + // this state; a footer note would say it a third time. + '' + : freeTierExhausted + ? '' + : !hasCopilot + ? $copilotInfo.workspaceDisabled + ? 'Windmill AI is hidden in this workspace' + : $aiUserDisabled + ? 'Windmill AI is disabled in your account settings' + : isAdmin + ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` + : 'Ask an admin to enable Windmill AI in this workspace to use this chat' + : aiChatManager.mode === AIMode.SCRIPT && + aiChatManager.scriptEditorOptions?.lang && + !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) + ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` + : '' ) const suggestions = [ diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 2ab29499ce..f610dd5ff4 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -7,6 +7,7 @@ AlertTriangle, ArrowDown, AtSign, + BookOpen, ChevronDown, ChevronsRight, CheckIcon, @@ -35,6 +36,7 @@ import ContextUsageIndicator from './ContextUsageIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' import McpConnections from './McpConnections.svelte' + import SkillsPicker from './SkillsPicker.svelte' import ChatMode from './ChatMode.svelte' import DatatableCreationPolicy from './DatatableCreationPolicy.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' @@ -206,6 +208,7 @@ let aiChatInput: AIChatInput | undefined = $state() let mcpConnections: McpConnections | undefined = $state() + let skillsPicker: SkillsPicker | undefined = $state() let plusMenuOpen = $state(false) let editingMessageIndex = $state(null) @@ -312,7 +315,10 @@ } }) - const showTypingIndicator = $derived(aiChatManager.loading) + // Also shown for a run held by another tab, labeled with where it is: the + // dots say a turn is in flight even before the reader reaches the footer + // note. Remote runs pause nothing and offer no Stop — this tab can't cancel. + const showTypingIndicator = $derived(aiChatManager.loading || aiChatManager.runHeldElsewhere) // The manual `@` context-picker button. Shown in SCRIPT/FLOW (workspace items + // code blocks) and APP (datatables, frontend files). Hidden in GLOBAL — there @@ -568,8 +574,14 @@ (aiChatManager.flowAiChatHelpers?.hasPendingChanges() ?? false) && !aiChatManager.autoAcceptEditsActive ) + // A disabled state with no message (a remote hold, a spent free grant) keeps + // the footer toolbar in place — swapping it for an empty strip would make + // the model/mode row flash out and back on every remote turn. A state with + // a real message (archived, AI off) still shows it, hold or not, matching + // the precedence disabledMessage itself encodes. + const footerMessageShown = $derived(disabled && disabledMessage !== '') const showFooterLeftControls = $derived( - !disabled && + !footerMessageShown && (showContextPicker || showAutonomyModeSelector || (aiChatManager.mode === AIMode.SCRIPT && hasDiff)) @@ -670,10 +682,14 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#each pastChats as chat (chat.id)} + {/snippet} + + {#if listNotice} + + {listNotice} + + {/if} + + {#if forkPending} + + Skills are read-only until the first message creates this session's fork. Editing or + selecting one now would apply to the parent workspace and stop applying once the fork is + created. + + {/if} + + + + Drop a folder of SKILL.md files to import, or click to choose + one + + + + {#if loading} +
Loading skills…
+ {:else if loadError} +
+ Failed to load skills: {loadError} +
+ {:else if skills.length === 0} +
+ No skills in this workspace yet. Paste a SKILL.md or import a folder of them. +
+ {:else} +
+ {#each skills as skill (skill.path)} +
+ +
+
+ {ambiguous.has(skill.name) ? skill.path : skill.name} +
+ {#if skill.description} +
{skill.description}
+ {/if} +
+ await toggle(skill.path, e.detail)} + /> + openSkill(skill, skill.canWrite ? 'edit' : 'view') + }, + { + displayName: 'Delete', + icon: Trash2, + type: 'delete', + disabled: !skill.canWrite || forkPending, + action: () => (toDelete = skill) + } + ]} + /> +
+ {/each} +
+ {/if} + + { + const skill = toDelete + toDelete = undefined + if (skill) await remove(skill) + }} + onCanceled={() => (toDelete = undefined)} + > + + This deletes the resource at {toDelete?.path}, so + everyone who selected it loses the skill. + + + + { + const toImport = [ + ...pendingNew.map((skill) => ({ skill, overwrite: false })), + ...pendingConflicts + .filter((s) => overwriteChoices[s.name]) + .map((skill) => ({ skill, overwrite: true })) + ] + const skipped = pendingSkipped + pendingImport = undefined + pendingSkipped = [] + overwriteChoices = {} + if (toImport.length) await importSkills(toImport, skipped) + else sendUserToast('No skills imported.') + }} + onCanceled={() => { + pendingImport = undefined + pendingSkipped = [] + overwriteChoices = {} + }} + > +
+ + Skills are added under {defaultOwner()}. Move one to a + shared folder from the resources page to share it. + + {#if pendingNew.length} +
+ Add {pendingNew.length} new skill(s): + {pendingNew.map((s) => s.name).join(', ')} +
+ {/if} + {#if pendingConflicts.length} +
+ + {pendingConflicts.length} skill(s) already exist — choose which to overwrite: + +
+ {#each pendingConflicts as conflict (conflict.name)} +
+ {conflict.name} + +
+ {/each} +
+
+ {/if} + {#if pendingSkipped.length} + {pendingSkipped.length} file(s) will be skipped. + {/if} +
+ + + + + + {#snippet headerRight()} + {#if editing} + + {#snippet children({ item })} + + + {/snippet} + + {/if} + {/snippet} +
+ {#if detailMode === 'view'} + {#if parsed.description} +

{parsed.description}

+ {/if} +
+ +
+ {:else} + + +
+ +
+
+ {contentError ?? ''} +
+ + +
+
+ {/if} +
+ diff --git a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte index 02612492a5..db2a0e22c4 100644 --- a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte @@ -28,7 +28,10 @@ import MqttIcon from '$lib/components/icons/MqttIcon.svelte' import AmqpIcon from '$lib/components/icons/AmqpIcon.svelte' import NatsIcon from '$lib/components/icons/NatsIcon.svelte' - import { runToolDisplayAction } from './createdResourceActions.svelte' + import { + hasToolDisplayActionHandler, + runToolDisplayAction + } from './createdResourceActions.svelte' import type { CreatedResourceTriggerKind, ToolDisplayAction } from './shared' interface Props { @@ -122,17 +125,21 @@
{card.title}
{card.subtitle}
- + + {#if hasToolDisplayActionHandler(action.type)} + + {/if} {/each} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts index b8e543a720..2455948e22 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts @@ -90,6 +90,15 @@ export class SessionArtifactsStore { await this.#load() } + /** Re-read the loaded session's artifacts from the store, for records another + * tab wrote after this one loaded. Forces the read setSession skips: that + * skip protects local edits whose best-effort persist failed, while a tab + * catching up on another tab's finished turn wants the store's truth. */ + async resyncFromStore(): Promise { + if (this.#sessionId === undefined) return + await this.#load() + } + async #load(): Promise { const token = ++this.#seq const id = this.#sessionId diff --git a/frontend/src/lib/components/copilot/chat/context.ts b/frontend/src/lib/components/copilot/chat/context.ts index bf915515db..f57862c179 100644 --- a/frontend/src/lib/components/copilot/chat/context.ts +++ b/frontend/src/lib/components/copilot/chat/context.ts @@ -331,6 +331,13 @@ export function contextElementKey(c: ContextElement): string { return c.type === 'app_dom_selector' ? `dom:${c.appPath}:${c.selector}` : `${c.type}:${c.title}` } +/** An `@`-mentioned workspace item, attached to one message the way a DOM pick + * is. Membership only: this does not gate on mode, so every caller must. Outside + * GLOBAL these stay selected as chips the user removes by hand. */ +export function isMentionContext(c: ContextElement): boolean { + return c.type === 'workspace_script' || c.type === 'workspace_flow' || c.type === 'workspace_app' +} + export function isSameContextElement(a: ContextElement, b: ContextElement): boolean { if (a.type !== b.type) return false if (a.type === 'app_dom_selector' && b.type === 'app_dom_selector') { diff --git a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts index 725385d3ee..bd0f3b3d0e 100644 --- a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts @@ -25,6 +25,15 @@ export function registerToolDisplayActionHandler( } } +/** + * Reactive: reads the `$state` registry, so a component re-renders when a page mounts or + * unmounts its handler. Offering an action without checking this yields an affordance whose + * only outcome is the unavailable-action toast. + */ +export function hasToolDisplayActionHandler(type: ToolDisplayAction['type']): boolean { + return toolDisplayActionHandlers[type] !== undefined +} + export async function runToolDisplayAction(action: ToolDisplayAction): Promise { const handler = toolDisplayActionHandlers[action.type] if (!handler) { diff --git a/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts new file mode 100644 index 0000000000..5290528dba --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts @@ -0,0 +1,74 @@ +import { get } from 'svelte/store' +import { userStore } from '$lib/stores' + +/** + * A set of workspace-object paths the chat may act through, remembered per + * workspace and per account. + * + * Being able to read a resource is not the same as wanting the chat to use it: a + * resource in a shared folder is readable by a whole team, and each enabled entry + * costs something on every turn — an MCP server puts its tool descriptions in the + * model's context and reaches an external system, a skill puts its description + * there. So an entry is off until it is turned on. + * + * Stored per browser, like the chat's other per-user preferences, but keyed by + * email as well as workspace: browser storage outlives a logout, and inheriting + * the previous account's selection would hand the next person capabilities they + * never turned on. Workspace ids cannot contain `:`, so the composite key is + * unambiguous. + */ +export type EnabledPathsPreference = { + enabledPaths: (workspace: string) => string[] + isEnabled: (workspace: string, path: string) => boolean + /** Returns false when there is no account to record the preference against, so + * a caller that just created the object can say it did not stay on. */ + setEnabled: (workspace: string, path: string, enabled: boolean) => boolean +} + +export function createEnabledPathsPreference(storageKey: string): EnabledPathsPreference { + function scope(workspace: string): string | undefined { + const email = get(userStore)?.email + return email ? `${workspace}:${email}` : undefined + } + + function read(): Record { + if (typeof localStorage === 'undefined') return {} + try { + return JSON.parse(localStorage.getItem(storageKey) ?? '{}') + } catch { + return {} + } + } + + function write(all: Record) { + try { + localStorage.setItem(storageKey, JSON.stringify(all)) + } catch (e) { + console.error(`Failed to persist ${storageKey}`, e) + } + } + + function enabledPaths(workspace: string): string[] { + const key = scope(workspace) + return key ? (read()[key] ?? []) : [] + } + + return { + enabledPaths, + isEnabled: (workspace, path) => enabledPaths(workspace).includes(path), + setEnabled: (workspace, path, enabled) => { + const key = scope(workspace) + if (!key) return false + const all = read() + const current = new Set(all[key] ?? []) + if (enabled) { + current.add(path) + } else { + current.delete(path) + } + all[key] = [...current] + write(all) + return true + } + } +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 90301f5493..19050bb7da 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -210,7 +210,8 @@ vi.mock('$lib/gen', async () => { }), createResource: vi.fn(async () => 'created'), updateResource: vi.fn(async () => 'updated'), - deleteResource: vi.fn(async () => 'deleted') + deleteResource: vi.fn(async () => 'deleted'), + getResourceValue: vi.fn(async () => ({ content: 'skill body' })) }), VariableService: wrapService(actual.VariableService, { existsVariable: vi.fn(async () => false), @@ -5435,6 +5436,18 @@ describe('session-only preview tools gating', () => { }) }) +describe('read_skill', () => { + it('refuses a path the user has not selected, without reading it', async () => { + localStorage.clear() + userStore.set({ username: 'bob', email: 'bob@windmill.dev', workspace_id: WORKSPACE } as any) + + const res = await callGlobalTool('read_skill', { path: 'u/someone/private-notes' }) + + expect(res).toContain('not one of the skills selected') + expect(vi.mocked(ResourceService.getResourceValue)).not.toHaveBeenCalled() + }) +}) + describe('update_user_instructions', () => { function makeHelpers(initial = '') { let value = initial diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 52704e7cb8..e919769e7a 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -17,8 +17,7 @@ import { ScriptService, SqsTriggerService, VariableService, - WebsocketTriggerService, - WorkspaceService + WebsocketTriggerService } from '$lib/gen' import { createTwoFilesPatch } from 'diff' import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' @@ -83,6 +82,16 @@ import { } from '../flow/inlineScriptsUtils' import { searchNpmPackagesTool } from '../script/core' import type { McpServer } from './mcpTools' +import { logFeatureUsage } from '$lib/utils/featureUsage' +import { enabledSkillPaths } from '../skills/enabledSkills' +import { + listSkillResources, + readSkillBody, + skillNameFromPath, + truncateChars, + truncateForPrompt +} from '../skills/skillResources' +import { MAX_SKILL_DESCRIPTION_LENGTH, MAX_SKILL_INSTRUCTIONS_LENGTH } from '../skills/skillMd' import { getDatatableSdkReference, getFlowPrompt, @@ -636,7 +645,7 @@ const writeVariableSchema = variableRequestSchema.extend({ .string() .optional() .describe( - 'The value of the variable. Omit it to leave the value alone — required only when creating a new variable, or when changing a secret variable into a non-secret one. Never invent or guess the value of an existing variable: you cannot read it, and a "$var:..." reference is NOT a valid value (that syntax only references a variable from inside a resource). Omitting it keeps whatever the draft already holds, so a value you set earlier in this conversation stays set; discard_local_draft abandons it.' + 'The value of the variable. Omit it to leave the value alone — required only when creating a new variable, or when changing a secret variable into a non-secret one. Never invent or guess the value of an existing variable: you cannot read it, and a "$var:..." reference is NOT a valid value (a variable cannot reference itself). Omitting it keeps whatever the draft already holds, so a value you set earlier in this conversation stays set; discard_local_draft abandons it.' ), is_secret: z .boolean() @@ -845,7 +854,9 @@ const testRunArgsSchema = z .record(z.string(), z.any()) .nullable() .optional() - .describe('Arguments to pass to the runnable. Omit or pass null when no arguments are needed.') + .describe( + 'Arguments to pass to the runnable. Omit or pass null when no arguments are needed. An argument typed as a resource (format "resource-" in the input schema) takes the bare string "$res:" as its whole value — never an object wrapper like {"$res": ""}, and never a plain path, both of which reach the runnable unresolved. Same for a variable, with "$var:". The prefixed string can also sit in a nested field, e.g. {"gh_auth": {"token": "$var:g/all/gh_token"}}.' + ) const backgroundArgSchema = z .boolean() @@ -1362,9 +1373,9 @@ Data Tables: ? ` Skills: -- Skills are reusable instruction sets curated for this workspace, each covering a specific kind of task. The available skills are listed below by name and description. -- When a user's request matches a skill's description, call read_skill with its exact name to load the full instructions BEFORE acting, then follow them. -${skills.map((s) => `- ${s.name}: ${s.description}`).join('\n')}` +- Skills are reusable instruction sets the user selected for this chat, each covering a specific kind of task. The available skills are listed below by resource path and description. +- When a user's request matches a skill's description, call read_skill with its exact path to load the full instructions BEFORE acting, then follow them. +${skills.map((s) => `- ${s.path}: ${s.description}`).join('\n')}` : '' }${ mcpServers.length > 0 @@ -2205,7 +2216,7 @@ function getResourceInstructions(): string { - Reading a variable returns \`{ type: 'variable', path, summary?, isSecret, isDraft }\` — never its value, secret or not. \`isSecret\` tells you whether the value is encrypted. - \`write_variable\` takes \`{ path, value?, is_secret?, description?, account?, is_oauth?, expires_at?, labels? }\`. Creating a variable needs \`value\` and \`is_secret\`; editing one needs only the fields you are changing. Omitting \`value\` keeps the stored value, which is the only way to edit a secret variable — you cannot read its value, so passing any \`value\` you did not get from the user destroys it. - For secret fields in a resource value, do NOT inline the raw secret. Create a Variable first with \`is_secret: true\`, then in the resource value reference it as \`"$var:path/to/variable"\`. -- Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. These are references FROM a resource value; never store a \`$var:\` string as a variable's own value. +- Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. The same strings are also how a resource or variable is passed as a run argument (see the run-argument rule in the resource reference below); what they are never valid as is a variable's own value. - When deploying drafts that depend on each other (e.g., a resource and the variables it references), deploy the variables first. - Use \`search_resource_types\` to discover valid \`resource_type\` names and their JSON Schemas. Match the resource value to that schema. - For OAuth resources, the \`is_oauth: true\` flag is managed by Windmill's OAuth flow; global mode generally creates manual resources, not OAuth ones. @@ -2253,7 +2264,9 @@ function getInstructions( } } -export type AiSkillListItem = { name: string; description: string } +/** A skill the user turned on, as the prompt and the `/` picker see it. `path` + * is the `ai_skill` resource and the model-facing id; `name` is its basename. */ +export type AiSkillListItem = { path: string; name: string; description: string } /** Live session facts appended to the GLOBAL system prompt for session chats. * Provided by the session runtime as a resolver (copilot must not import the @@ -2316,15 +2329,43 @@ export function getSessionContextPromptSection(ctx: SessionPromptContext): strin return lines.join('\n') } -/** `/` picker entry: a workspace skill or a built-in session action. The kind - * drives the picker's category grouping; entries without one are ungrouped. */ -export type ChatCommandItem = AiSkillListItem & { kind?: 'action' | 'skill' } +/** `/` picker entry: a selected skill or a built-in session action. The kind + * drives the picker's category grouping; entries without one are ungrouped. + * Only skills carry a `path` — built-in actions run locally and have no resource. */ +export type ChatCommandItem = { + name: string + description: string + path?: string + kind?: 'action' | 'skill' +} -/** Fetch the workspace's AI skills (name + description) for the global system prompt. */ +/** + * The skills this user turned on in this workspace, for the global system prompt. + * A readable `ai_skill` resource is only a candidate — enabling one is a personal + * choice, since each enabled skill spends context on every turn. + */ export async function loadWorkspaceSkills(workspace: string): Promise { if (!workspace) return [] try { - return await WorkspaceService.listAiSkills({ workspace }) + const enabled = new Set(enabledSkillPaths(workspace)) + if (enabled.size === 0) return [] + // Filtered against what is actually readable now, so a skill that was + // deleted or whose folder access was revoked drops out instead of being + // advertised to the model as something read_skill can load. + // A truncated listing still carries most of the workspace, and the drawer is + // where that is surfaced; dropping everything here would silently empty the + // Skills section instead. + return (await listSkillResources(workspace)).skills + .filter((s) => enabled.has(s.path)) + .map(({ path, name, description }) => ({ + path, + name, + // Every description goes into the system prompt on every turn, and any + // resource of this type can be selected — including ones written through + // git sync or the resource editor, which never saw the authoring form's + // bounds. One unbounded description would crowd out the conversation. + description: truncateChars(description, MAX_SKILL_DESCRIPTION_LENGTH) + })) } catch (e) { console.error('Failed to load AI skills', e) return [] @@ -2332,32 +2373,52 @@ export async function loadWorkspaceSkills(workspace: string): Promise = { def: createToolDef( readSkillSchema, 'read_skill', - 'Load the full instructions for a workspace AI skill by name. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' + 'Load the full instructions for a selected AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' ), planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readSkillSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${parsed.name}"...` }) + const name = skillNameFromPath(parsed.path) + // The prompt lists only selected skills, but the tool takes a path the model + // composed, so the selection is enforced here too rather than assumed. Without + // it the tool reads any resource holding a string `content` — the user's own + // access, but not what "load a selected skill" says it does. + if (!enabledSkillPaths(workspace).includes(parsed.path)) { + toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not selected` }) + return `"${parsed.path}" is not one of the skills selected for this chat. Only the paths listed under "Skills" in the system prompt can be read.` + } + toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${name}"...` }) try { - const skill = await WorkspaceService.getAiSkill({ workspace, name: parsed.name }) - toolCallbacks.setToolStatus(toolId, { content: `Read skill "${parsed.name}"` }) - return `Skill: ${skill.name}\nDescription: ${skill.description}\n\nInstructions:\n${skill.instructions}` + // Bounded here rather than in the reader: any `ai_skill` resource can be + // selected, including ones written through git sync or the resource editor + // that never passed the authoring form's limits, and an unbounded body + // would exhaust the context on one tool call. The editor reads the same + // resource untruncated, so opening a long skill cannot rewrite it short. + const instructions = truncateForPrompt( + await readSkillBody(workspace, parsed.path), + MAX_SKILL_INSTRUCTIONS_LENGTH + ) + toolCallbacks.setToolStatus(toolId, { content: `Read skill "${name}"` }) + // Whether a selected skill is actually reached for. No key: the path is + // workspace-authored text. + logFeatureUsage('ai_session', 'skill_read', { workspace }) + return `Skill: ${parsed.path}\n\nInstructions:\n${instructions}` } catch (e) { const msg = e instanceof Error ? e.message : String(e) toolCallbacks.setToolStatus(toolId, { - content: `Error reading skill "${parsed.name}"`, + content: `Error reading skill "${name}"`, error: msg }) - return `Failed to read skill "${parsed.name}": ${msg}. Check the name against the Skills list in the system prompt.` + return `Failed to read skill "${parsed.path}": ${msg}. Check the path against the Skills list in the system prompt.` } } } @@ -5046,7 +5107,7 @@ function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promis // is always the model echoing the reference syntax back instead of a real value. if (args.value === `$var:${args.path}`) { throw new Error( - `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. The "$var:" syntax only references a variable from inside a resource value. Omit value to keep the current one.` + `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. Omit value to keep the current one.` ) } return writeDraft(VARIABLE_SPEC, 'variable', args.path, args, ctx, { override: args.override }) diff --git a/frontend/src/lib/components/copilot/chat/global/gate.ts b/frontend/src/lib/components/copilot/chat/global/gate.ts index 3321d5a0cd..8fb650aa36 100644 --- a/frontend/src/lib/components/copilot/chat/global/gate.ts +++ b/frontend/src/lib/components/copilot/chat/global/gate.ts @@ -9,8 +9,8 @@ * * When the beta ends, replace every call to `isGlobalAiEnabled()` with `true` * and delete this file. The references are intentionally narrow (chat mode - * visibility, custom prompt settings, the `change_mode` tool enum, and the - * AI skills workspace settings tab) so the rip-out is a small grep. + * visibility, custom prompt settings, and the `change_mode` tool enum) so the + * rip-out is a small grep. */ import { logFeatureUsage } from '$lib/utils/featureUsage' diff --git a/frontend/src/lib/components/copilot/chat/itemPreview.ts b/frontend/src/lib/components/copilot/chat/itemPreview.ts new file mode 100644 index 0000000000..ef9f14ddc9 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/itemPreview.ts @@ -0,0 +1,30 @@ +// The session preview panel's action, kept out of `shared.ts` so the chat message render +// path can import it at runtime without pulling in that module's graph and risking the +// chunk cycles docs/frontend-import-cycles.md exists to prevent. Keep this file import-free. + +/** Item kinds a session preview can host: the three live editors, which are also the + * subset a write tool can land. */ +export type PreviewCardKind = 'script' | 'flow' | 'raw_app' + +// Dispatched by a preview card on a tool call that created or updated a workspace item, +// and by a path link in a chat message. Opens the item's live editor in the session side +// panel — or focuses the tab if it is already open. The handler is registered by the +// sessions page (the only surface with a preview panel). +export type OpenItemPreviewAction = { + id: string + type: 'open_item_preview' + label: string + previewKind: PreviewCardKind + path: string +} + +/** Build the action a preview card or path link dispatches from its (kind, path). */ +export function openItemPreviewAction(kind: PreviewCardKind, path: string): OpenItemPreviewAction { + return { + id: `open-item-preview:${kind}:${path}`, + type: 'open_item_preview', + label: `Open ${kind === 'raw_app' ? 'app' : kind} preview`, + previewKind: kind, + path + } +} diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index b485be51b3..6d6794b56c 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -7,6 +7,12 @@ import type { UserDraftItemKind } from '$lib/gen' // The gate's two refusals, from a module that holds prose and one size limit: under the // shallow-import rule below, the rest of plan mode is not reachable from here. import { PLAN_MODE_MESSAGES } from './planModeMessages' +// Import-free leaf, so it satisfies the shallow-import rule below. +import { + openItemPreviewAction, + type OpenItemPreviewAction, + type PreviewCardKind +} from './itemPreview' // The tool modules that import this one (workspaceTools, flow/core, global/core, ...) // call createToolDef and read SPECIAL_MODULE_IDS at *module scope*, so if a chunk cycle @@ -526,35 +532,11 @@ export type NavigateAction = { page: string } -/** Kinds of previewable item a write tool can land — the subset of draft item - * kinds a session preview can host. */ -export type PreviewCardKind = 'script' | 'flow' | 'raw_app' - -// A discrete card shown on a tool call that created or updated a workspace item. -// Clicking it opens the item's live preview in the session side panel — or focuses -// the tab if it is already open. The handler is registered by the sessions page -// (the only surface with a preview panel). -export type OpenItemPreviewAction = { - id: string - type: 'open_item_preview' - label: string - previewKind: PreviewCardKind - path: string -} +// Re-exported: most consumers reach these through this module. +export { openItemPreviewAction, type PreviewCardKind, type OpenItemPreviewAction } export type ToolDisplayAction = CreatedResourceAction | NavigateAction | OpenItemPreviewAction -/** Build the action a preview card dispatches from its (kind, path). */ -export function openItemPreviewAction(kind: PreviewCardKind, path: string): OpenItemPreviewAction { - return { - id: `open-item-preview:${kind}:${path}`, - type: 'open_item_preview', - label: `Open ${kind === 'raw_app' ? 'app' : kind} preview`, - previewKind: kind, - path - } -} - export type UserQuestionDisplay = { question: string choices: string[] diff --git a/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts new file mode 100644 index 0000000000..e91d40bff0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts @@ -0,0 +1,10 @@ +import { createEnabledPathsPreference } from '../enabledPathsPreference' + +/** Which `ai_skill` resources the chat may follow, per workspace and per account. + * Every enabled skill spends context on every turn, so selecting one is a personal + * choice rather than a consequence of being able to read it. */ +const preference = createEnabledPathsPreference('wm_skills_enabled') + +export const enabledSkillPaths = preference.enabledPaths +export const isSkillEnabled = preference.isEnabled +export const setSkillEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/workspaceSettings/aiSkills.test.ts b/frontend/src/lib/components/copilot/chat/skills/skillMd.test.ts similarity index 99% rename from frontend/src/lib/components/workspaceSettings/aiSkills.test.ts rename to frontend/src/lib/components/copilot/chat/skills/skillMd.test.ts index 87f84d865a..b498cc2d24 100644 --- a/frontend/src/lib/components/workspaceSettings/aiSkills.test.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillMd.test.ts @@ -8,7 +8,7 @@ import { parseAndValidateSkill, parseSkillMd, validateSkill -} from './aiSkills' +} from './skillMd' describe('parseSkillMd', () => { it('splits frontmatter name/description from the body', () => { diff --git a/frontend/src/lib/components/workspaceSettings/aiSkills.ts b/frontend/src/lib/components/copilot/chat/skills/skillMd.ts similarity index 90% rename from frontend/src/lib/components/workspaceSettings/aiSkills.ts rename to frontend/src/lib/components/copilot/chat/skills/skillMd.ts index aeb148f7c1..93916c023f 100644 --- a/frontend/src/lib/components/workspaceSettings/aiSkills.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillMd.ts @@ -1,10 +1,13 @@ import YAML from 'yaml' import { z } from 'zod' +/** A SKILL.md split into the three parts a `skills` resource stores: `name` + * becomes the resource path's basename, `description` its description column, + * `instructions` its file body. */ export type SkillUpload = { name: string; description: string; instructions: string } -// `name` + `description` mirror the Claude SKILL.md spec (counted in characters); -// the body is a byte-bounded payload. Keep these in sync with backend `validate_skill`. +// `name` + `description` mirror the Claude SKILL.md spec (counted in characters), +// so a skill stays portable with Claude Code; the body is a byte-bounded payload. export const MAX_SKILL_NAME_LENGTH = 64 export const MAX_SKILL_DESCRIPTION_LENGTH = 1_024 export const MAX_SKILL_INSTRUCTIONS_LENGTH = 64 * 1024 @@ -12,8 +15,8 @@ export const MAX_SKILL_INSTRUCTIONS_LENGTH = 64 * 1024 const textEncoder = new TextEncoder() // Single source of truth for skill field validation, shared by the paste/edit -// modal and the folder importer. Lengths are code-point / byte bounded to match -// the backend, so `.refine` (not `.max`, which counts UTF-16 units) is used. +// modal and the folder importer. Lengths are code-point / byte bounded, so +// `.refine` (not `.max`, which counts UTF-16 units) is used. export const skillSchema = z.object({ name: z .string() diff --git a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts new file mode 100644 index 0000000000..20737caffa --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts @@ -0,0 +1,159 @@ +import { ResourceService } from '$lib/gen' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' + +/** + * Skills are resources of this type: a file resource (`format_extension = 'md'`) + * whose `value.content` is the SKILL.md body, whose description column is what the + * assistant reads when deciding the skill applies, and whose path names it. + */ +export const SKILLS_RESOURCE_TYPE = 'ai_skill' + +/** A skill as the picker and the system prompt see it — never the body, which + * `read_skill` fetches only once the model commits to using the skill. */ +export type SkillResource = { + path: string + /** Path basename: what the `/` command and the picker row show. */ + name: string + description: string + editedAt?: string + canWrite: boolean +} + +/** The `/`-command and display name for a skill. Paths are `[ufg]/x/y…`, so the + * last segment is always present. */ +export function skillNameFromPath(path: string): string { + return path.split('/').pop() ?? path +} + +/** Basenames carried by more than one of these skills. Two folders can each hold + * a `deploy`, and then the name alone no longer says which one — the picker shows + * the path for these, and the `/` command refuses to guess. */ +export function ambiguousSkillNames(skills: readonly { name: string }[]): Set { + const seen = new Map() + for (const s of skills) seen.set(s.name, (seen.get(s.name) ?? 0) + 1) + return new Set([...seen].filter(([, n]) => n > 1).map(([name]) => name)) +} + +const SKILLS_PAGE_SIZE = 100 +/** Pages to walk before giving up. Ordinary resources and repeated imports can + * make any number of skills, and a single page would drop the rest — including a + * selected one, which would then vanish from the prompt with nothing to explain + * it. The bound is a guard against a paging bug looping forever, not a product + * cap, so reaching it is reported rather than passed off as the whole set. */ +const MAX_SKILLS_PAGES = 100 + +/** The rows read, and whether the walk stopped at the bound rather than the end. + * Reported rather than thrown: a truncated read is still most of the skills, and + * dropping them all would take every selected skill out of the prompt at once. */ +export type SkillListing = { skills: SkillResource[]; truncated: boolean } + +/** Every skill resource readable in the workspace. + * + * `user` decides which rows the drawer offers to edit rather than only view; pass + * the account the workspace is being browsed as. Ownership is mostly implicit in + * the path (`u//…`, a folder the user owns), which is why this goes through + * the shared `canWrite` rather than reading `extra_perms` alone. */ +export async function listSkillResources( + workspace: string, + user?: UserExt +): Promise { + if (!workspace) return { skills: [], truncated: false } + const rows: SkillResource[] = [] + for (let page = 1; page <= MAX_SKILLS_PAGES; page++) { + const resources = await ResourceService.listResource({ + workspace, + resourceType: SKILLS_RESOURCE_TYPE, + page, + perPage: SKILLS_PAGE_SIZE + }) + rows.push( + ...resources.map((r) => ({ + path: r.path, + name: skillNameFromPath(r.path), + description: r.description ?? '', + editedAt: r.edited_at, + canWrite: canWrite(r.path, r.extra_perms ?? {}, user) + })) + ) + if (resources.length < SKILLS_PAGE_SIZE) return { skills: rows, truncated: false } + } + return { skills: rows, truncated: true } +} + +/** Cut `text` to `maxChars` code points. For the description, whose cap is stated + * in characters — cutting that one by bytes would reduce a legal 1,024-character + * CJK description to about a third of itself. */ +export function truncateChars(text: string, maxChars: number): string { + const points = [...text] + return points.length <= maxChars ? text : `${points.slice(0, maxChars).join('')}… [truncated]` +} + +/** Cut `text` to `maxBytes` of UTF-8, marking the cut so a reader (the model + * included) can tell truncation from a body that simply ends there. + * + * For the body, whose cap is a byte budget: 64k CJK characters are ~192 KiB, so a + * code-unit cut would let three times the intended payload through. */ +export function truncateForPrompt(text: string, maxBytes: number): string { + const encoded = new TextEncoder().encode(text) + if (encoded.byteLength <= maxBytes) return text + // `fatal: false` replaces the partial code point a byte-aligned cut can leave + // with U+FFFD; dropping it keeps the tail clean. + const cut = new TextDecoder('utf-8').decode(encoded.slice(0, maxBytes)).replace(/�$/, '') + return `${cut}… [truncated]` +} + +/** The SKILL.md body of one skill. Throws rather than returning `''` when the + * resource holds no readable body: an empty string reaches the model as a + * successful read of a skill with no instructions, which it would then act on. + * + * Deliberately unbounded — the editor loads through here and saves what it loaded, + * so truncating would rewrite an over-long skill the first time someone opened it. + * Bounding belongs at the prompt boundary, where the cost actually is. */ +export async function readSkillBody(workspace: string, path: string): Promise { + const value = (await ResourceService.getResourceValue({ workspace, path })) as + | { content?: unknown } + | undefined + if (typeof value?.content !== 'string') { + throw new Error(`resource ${path} has no string "content" — is it an ${SKILLS_RESOURCE_TYPE}?`) + } + return value.content +} + +export async function saveSkillResource( + workspace: string, + path: string, + description: string, + instructions: string, + { overwrite = false }: { overwrite?: boolean } = {} +): Promise { + await ResourceService.createResource({ + workspace, + updateIfExists: overwrite, + requestBody: { + path, + description, + value: { content: instructions }, + resource_type: SKILLS_RESOURCE_TYPE + } + }) +} + +/** Save an edit to an existing skill, moving it when the path changed. */ +export async function updateSkillResource( + workspace: string, + currentPath: string, + path: string, + description: string, + instructions: string +): Promise { + await ResourceService.updateResource({ + workspace, + path: currentPath, + requestBody: { path, description, value: { content: instructions } } + }) +} + +export async function deleteSkillResource(workspace: string, path: string): Promise { + await ResourceService.deleteResource({ workspace, path }) +} diff --git a/frontend/src/lib/components/copilot/chat/skills/skills.test.ts b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts new file mode 100644 index 0000000000..5b0d2196ca --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { session } = vi.hoisted(() => ({ + session: { email: 'first@windmill.dev' } as { email?: string } +})) + +vi.mock('$lib/stores', () => ({ + // Read at call time, so a test can switch accounts the way a logout does. + userStore: { subscribe: (run: (v: unknown) => void) => (run({ ...session }), () => {}) } +})) + +import { enabledSkillPaths, isSkillEnabled, setSkillEnabled } from './enabledSkills' +import { ambiguousSkillNames, truncateChars, truncateForPrompt } from './skillResources' + +describe('enabledSkills', () => { + beforeEach(() => { + localStorage.clear() + session.email = 'first@windmill.dev' + }) + + it('keeps the selection separate per workspace', () => { + setSkillEnabled('ws_a', 'u/me/deploy', true) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) + expect(isSkillEnabled('ws_b', 'u/me/deploy')).toBe(false) + }) + + it('does not hand the next account the previous one’s selection', () => { + setSkillEnabled('ws_a', 'u/me/deploy', true) + session.email = 'second@windmill.dev' + expect(enabledSkillPaths('ws_a')).toEqual([]) + }) + + it('reports failure when there is no account to record the choice against', () => { + session.email = undefined + expect(setSkillEnabled('ws_a', 'u/me/deploy', true)).toBe(false) + expect(enabledSkillPaths('ws_a')).toEqual([]) + }) +}) + +describe('skill names', () => { + it('flags a basename two folders both use, so /name is not resolved by chance', () => { + const ambiguous = ambiguousSkillNames([ + { name: 'deploy' }, + { name: 'deploy' }, + { name: 'release' } + ]) + expect([...ambiguous]).toEqual(['deploy']) + }) +}) + +describe('prompt truncation', () => { + // The two caps are stated in different units, and using one truncator for both + // either lets three times the payload through or cuts a legal value to a third. + it('bounds a skill body by utf-8 bytes, not code units', () => { + const body = '漢'.repeat(100) // 300 bytes + expect(truncateForPrompt(body, 3000)).toBe(body) + const cut = truncateForPrompt(body, 30) + expect(new TextEncoder().encode(cut.replace('… [truncated]', '')).byteLength).toBeLessThanOrEqual(30) + expect(cut).toContain('[truncated]') + // A byte-aligned cut must not leave a broken code point behind. + expect(cut).not.toContain('\ufffd') + }) + + it('bounds a description by code points, so a CJK one is not cut to a third', () => { + const description = '漢'.repeat(100) + expect(truncateChars(description, 100)).toBe(description) + expect([...truncateChars(description, 10)].slice(0, 10).join('')).toBe('漢'.repeat(10)) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts b/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts index b1a6f695d3..1e02573a32 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts @@ -22,6 +22,8 @@ import { findAndReplace } from 'mdast-util-find-and-replace' import { visit } from 'unist-util-visit' import type { Root, InlineCode, Link } from 'mdast' import type { CreatedResourceAction, ToolDisplayAction } from './shared' +// Leaf module, deliberately not './shared' — see the header of itemPreview.ts. +import { openItemPreviewAction } from './itemPreview' export type WindmillItemKind = | 'script' @@ -46,6 +48,9 @@ export interface WorkspaceItemEntry { kind: WindmillItemKind path: string targetKind?: WorkspaceItemTargetKind + /** Apps only. Both kinds share the `app` kind and the same `/apps/get/` route, + * so this flag is all that tells them apart downstream. */ + rawApp?: boolean } export type WorkspaceItemTargetKind = 'script' | 'flow' @@ -96,6 +101,7 @@ export function itemHref(entry: WorkspaceItemEntry, workspace?: string): string type WorkspaceItemListResult = Array<{ path: string is_flow?: boolean | null + raw_app?: boolean | null }> const workspaceItemLoaders: Array<{ @@ -165,7 +171,8 @@ class WorkspaceItemRegistry { kind, path: it.path, targetKind: - typeof it.is_flow === 'boolean' ? (it.is_flow ? 'flow' : 'script') : undefined + typeof it.is_flow === 'boolean' ? (it.is_flow ? 'flow' : 'script') : undefined, + rawApp: kind === 'app' ? it.raw_app === true : undefined }) } } @@ -213,13 +220,30 @@ export function extractCandidatePaths(text: string | undefined | null): string[] return [...seen] } +/** + * The in-app action a resolved path link runs instead of opening a new tab, or undefined + * when the kind has none (its link then stays outbound). Returning an action does not mean + * it can run on this surface — only the sessions page hosts a preview panel, so callers + * gate on `hasToolDisplayActionHandler(action.type)`. + */ export function workspaceItemAction( kind: WindmillItemKind | undefined, path: string | undefined, - targetKind?: WorkspaceItemTargetKind + targetKind?: WorkspaceItemTargetKind, + rawApp?: boolean ): ToolDisplayAction | undefined { if (!kind || !path) return undefined + if (kind === 'script' || kind === 'flow') { + return openItemPreviewAction(kind, path) + } + + // Raw apps only. A legacy drag-and-drop app has no editor the panel can host, and + // legacy items are not extended onto new surfaces — its link stays outbound. + if (kind === 'app') { + return rawApp ? openItemPreviewAction('raw_app', path) : undefined + } + const base = { id: `open_workspace_item:${kind}:${path}`, type: 'open_created_resource' as const, @@ -253,6 +277,9 @@ function buildPathLinkNode( if (entry.targetKind) { hProperties['data-wm-target-kind'] = entry.targetKind } + if (entry.rawApp) { + hProperties['data-wm-raw-app'] = 'true' + } return { type: 'link', diff --git a/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts b/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts index 9bb477fa83..70810dad9a 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts @@ -153,10 +153,27 @@ describe('workspaceItemAction', () => { }) }) - it('skips non-drawerable items and trigger items without target kind', () => { - expect(workspaceItemAction('script', 'f/a/b')).toBeUndefined() - expect(workspaceItemAction('flow', 'f/a/b')).toBeUndefined() + it('creates preview actions for scripts, flows and raw apps', () => { + expect(workspaceItemAction('script', 'f/a/b')).toMatchObject({ + type: 'open_item_preview', + previewKind: 'script', + path: 'f/a/b' + }) + expect(workspaceItemAction('flow', 'f/a/b')).toMatchObject({ + type: 'open_item_preview', + previewKind: 'flow' + }) + expect(workspaceItemAction('app', 'f/a/b', undefined, true)).toMatchObject({ + previewKind: 'raw_app' + }) + }) + + it('leaves legacy apps as plain links', () => { + expect(workspaceItemAction('app', 'f/a/b', undefined, false)).toBeUndefined() expect(workspaceItemAction('app', 'f/a/b')).toBeUndefined() + }) + + it('skips trigger items without target kind', () => { expect(workspaceItemAction('schedule', 'f/a/b')).toBeUndefined() expect(workspaceItemAction('http_trigger', 'f/a/b')).toBeUndefined() }) @@ -172,6 +189,7 @@ const SAMPLE_ENTRIES: Record = { path: 'u/admin/cleanup_old_jobs' }, 'f/ops/dashboard': { kind: 'app', path: 'f/ops/dashboard' }, + 'f/ops/live_board': { kind: 'app', path: 'f/ops/live_board', rawApp: true }, 'f/etl/daily': { kind: 'schedule', path: 'f/etl/daily', @@ -249,6 +267,23 @@ describe('remarkWindmillPaths (mdast)', () => { expect(props['data-wm-target-kind']).toBe('flow') }) + // Both app kinds reach the renderer as the same `app` kind and route, so this flag is + // the only thing that keeps a legacy app off the preview panel. + it('marks raw apps so the renderer can tell them from legacy apps', () => { + const processor = buildProcessor('admins') + const tree = processor.runSync( + processor.parse('Open f/ops/live_board and f/ops/dashboard.') + ) as MdastRoot + const byPath = Object.fromEntries( + findLinks(tree).map((l) => [ + (l.data?.hProperties as Record)['data-wm-path'], + l.data?.hProperties as Record + ]) + ) + expect(byPath['f/ops/live_board']['data-wm-raw-app']).toBe('true') + expect(byPath['f/ops/dashboard']['data-wm-raw-app']).toBeUndefined() + }) + it('leaves unknown paths as plain text', () => { const processor = buildProcessor() const tree = processor.runSync( diff --git a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts index ef2708e570..c665cc964e 100644 --- a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts +++ b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts @@ -134,6 +134,37 @@ describe('parseOpenAICompletion tool call arguments', () => { expect(addedMessages).toEqual(messages) }) + it('keeps braces inside string arguments out of the object-depth count', async () => { + const { parseOpenAICompletion } = await import('./lib') + const fn = vi.fn().mockResolvedValue('tool ok') + const messages: ChatCompletionMessageParam[] = [] + + await parseOpenAICompletion( + streamOf([ + toolCallChunk({ + id: 'call_1', + function: { name: 'patch_app_file', arguments: '{"new_string": ' } + }), + toolCallChunk({ function: { arguments: '"hello } goodbye \\" } "}' } }) + ]), + createCallbacks(), + messages, + [], + [createTool(fn)] as any, + {}, + undefined, + { workspace: 'test' } + ) + + expect(fn).toHaveBeenCalledWith( + expect.objectContaining({ args: { new_string: 'hello } goodbye " } ' } }) + ) + const assistant = messages.find((m) => m.role === 'assistant') as any + expect(assistant.tool_calls[0].function.arguments).toBe( + '{"new_string": "hello } goodbye \\" } "}' + ) + }) + it('marks only the executing tool call as loading when one message has several', async () => { const { parseOpenAICompletion } = await import('./lib') const statuses: Record = {} diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index fc8a00727f..19f350a665 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1177,9 +1177,21 @@ export async function getCompletion( function extractFirstJSON(str: string) { let depth = 0, i = 0 + // Braces inside string values are not depth changes, so the scan tracks + // quoting and escaping: otherwise an argument such as {"a": "} "} is cut short. + let inString = false, + escaped = false for (; i < str.length; i++) { - if (str[i] === '{') depth++ - else if (str[i] === '}' && --depth === 0) break + const ch = str[i] + if (inString) { + if (escaped) escaped = false + else if (ch === '\\') escaped = true + else if (ch === '"') inString = false + continue + } + if (ch === '"') inString = true + else if (ch === '{') depth++ + else if (ch === '}' && --depth === 0) break } return str.slice(0, i + 1) } diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index cf9d008a27..eb875973dc 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -15,12 +15,11 @@ workspaceStore } from '$lib/stores' import type { SupportedLanguage } from '$lib/common' - import { createEventDispatcher, getContext, onDestroy, onMount, untrack } from 'svelte' + import { createEventDispatcher, getContext, untrack } from 'svelte' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' import { type Script, type ScriptLang, type HubScriptKind } from '$lib/gen' import ListFiltersQuick from '$lib/components/home/ListFiltersQuick.svelte' import { ExternalLink, Folder, User, X } from 'lucide-svelte' - import type { FlowEditorContext } from '../../flows/types' import { fade } from 'svelte/transition' import { flip } from 'svelte/animate' import { Button } from '$lib/components/common' @@ -87,8 +86,6 @@ let hubCompletions: HubCompletion[] = $state([]) - const { insertButtonOpen } = getContext('FlowEditorContext') - let selected: { kind: 'owner' | 'integrations'; name: string | undefined } | undefined = $state(undefined) @@ -221,13 +218,6 @@ selectedByKeyboard = index } - onMount(() => { - $insertButtonOpen = true - }) - - onDestroy(() => { - $insertButtonOpen = false - }) let langs = $derived( processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) @@ -255,6 +245,7 @@ // on indices that render nothing. let showAiRows = $derived( !disableAi && + !$copilotInfo.workspaceDisabled && funcDesc?.length > 0 && kind != 'failure' && kind != 'preprocessor' && diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index d1e65de1ea..7c75d97fab 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -3,7 +3,7 @@ +
dispatch('close')} {disableAi} on:insert @@ -230,7 +307,10 @@ selected={selectedKind === 'aiagent'} onSelect={() => { selectedKind = 'aiagent' + selectedByKeyboard = 0 loadSavedAgents() + // Clicking leaves focus on this button, where Enter would only re-select it. + stepGen?.focus() }} /> {/if} @@ -240,6 +320,8 @@ selected={selectedKind === 'aisandbox'} onSelect={() => { selectedKind = 'aisandbox' + selectedByKeyboard = 0 + stepGen?.focus() }} /> {/if} @@ -248,19 +330,25 @@ {/if} {#if selectedKind === 'aiagent'} -
+
{#if savedAgentsLoading}
@@ -268,21 +356,23 @@
{:else if filteredAgents.length > 0}
Saved agents
- {#each filteredAgents as agent (agent.path)} + {#each filteredAgents as agent, i (agent.path)} {/each} {:else} @@ -297,17 +387,11 @@
{ - dispatch('close') - dispatch('new', { - kind: 'script', - inlineScript: { - language: 'bun', - kind: 'script', - subkind: 'claudesandbox' - } - }) - }} + neutral + returnIcon + selected={aiSelected === 0} + onSelect={newClaudeSandbox} + onHover={() => (selectedByKeyboard = 0)} />
{:else} diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte index d0e75538ca..441489db65 100644 --- a/frontend/src/lib/components/home/HomeAIChat.svelte +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -1,6 +1,6 @@ -
-
- {#if showComposer} -
-
-

Build with AI

- Beta +
+
+ {#if showComposer && !collapsed} + {#if !disabled} + +
+ setCollapsed(true)} />
- -
+ {/if} +
+

Build with AI

+ Beta +
+ +
+
+ {#if !value} + + + {/if}
+ {#if disabled} + +
+

+ {#if $aiUserDisabled} + Windmill AI is disabled in your account settings + {:else if freeTierExhausted} + You have used all of your free Windmill AI tokens + {:else} + No AI provider is configured + {/if} +

+
+ {#if $aiUserDisabled} + + + {:else} + + {/if} + +
+
+ {/if}
{/if} -
- {#if showComposer} -
+
+ {#if showComposer && !collapsed} +
{#each homeAIExamples as example (example.label)}
+ {:else if showComposer} + + {:else}
{/if} - -
+ +
- {#if showComposer && disabled} -
-

- {#if $aiUserDisabled} - Windmill AI is disabled in your account settings - {:else if freeTierExhausted} - You have used all of your free Windmill AI tokens - {:else} - No AI provider is configured - {/if} -

- {#if $aiUserDisabled} - - - {:else} - - {/if} -
- {/if}
diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 0f0cb6a6c0..84432112a3 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -24,7 +24,8 @@ ChevronsDownUp, ChevronsUpDown, Code2, - LayoutDashboard + LayoutDashboard, + Tag } from 'lucide-svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import CreateActionsMenu from './CreateActionsMenu.svelte' @@ -39,6 +40,7 @@ type FilterSchemaRec } from '$lib/components/FilterSearchbar.svelte' import NoItemFound from './NoItemFound.svelte' + import ListFilters from './ListFilters.svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import FlowIcon from './FlowIcon.svelte' @@ -74,10 +76,10 @@ ) // FilterSearchbar schema — `_default_` is the free-text search; the rest mirror the - // boolean/kind list filters. Owner and label scoping are offered as searchbar presets - // (searchPresets) and resolve server-side (path_start / label) rather than filtering - // client-side. `content` is a distinct mode: it swaps the list for the client-side - // content-match view below (usable on any instance, not EE-gated). + // boolean/kind list filters. Owner and label are also reachable as searchbar presets + // (searchPresets) and as on-page chip rows (the ListFilters markup below). `content` is + // a distinct mode: it swaps the list for the client-side content-match view below + // (usable on any instance, not EE-gated). let searchFilterSchema = $derived({ _default_: { type: 'string' as const, hidden: true }, content: { @@ -86,8 +88,8 @@ description: 'Search across item contents' }, // Owner (u/ or f/) and label are offered as presets built from what the - // list actually holds (see searchPresets); they drive the same server path-scope / label - // filter the old on-page chips did. + // list actually holds (see searchPresets); owner is a server path-scope, label a + // client-side filter over the loaded rows. owner: { type: 'string' as const, label: 'Owner' }, label: { type: 'string' as const, label: 'Label' }, kind: { @@ -717,11 +719,20 @@ return true // should not happen } - // Owner/label scope now live on the URL-synced searchbar filters (set via the presets), - // not standalone chip state — the whole data layer below still reads these two, so keep - // them as the single derived source. Empty string reads as "no filter". + // The whole data layer below reads these two derived views of the searchbar filters, so + // keep them the single source. Empty string reads as "no filter". let ownerFilter = $derived((filterValues.val.owner || undefined) as string | undefined) let labelFilter = $derived((filterValues.val.label || undefined) as string | undefined) + // Chip-row setters. Clearing deletes the key rather than writing null, which the + // searchbar would otherwise render as a `key: null` tag. + function setOwnerFilter(o: string | undefined) { + if (o == undefined) delete filterValues.val.owner + else filterValues.val.owner = o + } + function setLabelFilter(l: string | undefined) { + if (l == undefined) delete filterValues.val.label + else filterValues.val.label = l + } const cmp = new Intl.Collator('en').compare @@ -1155,11 +1166,26 @@ function itemLabels(x: { labels?: string[]; inherited_labels?: string[] }): string[] { return [...(x.labels ?? []), ...(x.inherited_labels ?? [])] } - let allLabels = $derived( - Array.from(new Set(combinedItems?.flatMap((x) => itemLabels(x)) ?? [])).sort() + // Labels ranked by how many loaded rows carry them (ties alphabetical). Unlike the owner + // chips there is no workspace-wide count endpoint, so the order is window-local and can + // shift as later pages load. A row carrying a label both directly and by inheritance + // counts once. + let allLabels = $derived.by(() => { + const counts = new Map() + for (const x of combinedItems ?? []) + for (const l of new Set(itemLabels(x))) counts.set(l, (counts.get(l) ?? 0) + 1) + return [...counts.keys()].sort( + (a, b) => (counts.get(b) ?? 0) - (counts.get(a) ?? 0) || cmp(a, b) + ) + }) + let hasChips = $derived( + owners.length > 0 || + allLabels.length > 0 || + ownerFilter != undefined || + labelFilter != undefined ) // FilterSearchbar presets: the owner prefixes and labels the list actually holds, so - // scoping to one is a click in the searchbar dropdown instead of a wall of on-page chips. + // scoping to one is a click in the searchbar dropdown. // Owner sets the `owner` filter (server path-scope), label sets `label` (client filter). // The `:\ ` separator and escaped spaces match the canonical `key:\ value` form parseToText // emits, so the "already applied" check finds them after a reparse and won't re-offer a @@ -1743,6 +1769,30 @@ {/if}
+ {#if !contentActive && hasChips} + +
+ ownerFilter, setOwnerFilter} + filters={owners} + queryName="owner" + maxDisplayed={10} + /> + labelFilter, setLabelFilter} + filters={allLabels} + queryName="label" + maxDisplayed={10} + color="blue" + icon={Tag} + /> +
+ {/if} {#if filteredItems?.length == 0}
{/if} diff --git a/frontend/src/lib/components/home/ListFilters.svelte b/frontend/src/lib/components/home/ListFilters.svelte index 38921a5ec3..ecddfec11a 100644 --- a/frontend/src/lib/components/home/ListFilters.svelte +++ b/frontend/src/lib/components/home/ListFilters.svelte @@ -1,6 +1,7 @@ {#if Array.isArray(filtersAndSelected) && filtersAndSelected.length > 0} -
+
{#each displayedFilters as filter (filter)}
- {#if resourceType} + {#if icon} + {@const Icon = icon} + + {:else if resourceType} {@const SvelteComponent = appIconComponent(filter)} {:else if filter.startsWith('u/')} @@ -123,7 +140,7 @@
(expanded = !expanded)} diff --git a/frontend/src/lib/components/home/NoItemFound.svelte b/frontend/src/lib/components/home/NoItemFound.svelte index ea90c8dd0d..94d6dfa1b8 100644 --- a/frontend/src/lib/components/home/NoItemFound.svelte +++ b/frontend/src/lib/components/home/NoItemFound.svelte @@ -30,7 +30,7 @@ {:else}
-
+
Get started by creating your first script, flow, or app
diff --git a/frontend/src/lib/components/home/TutorialBanner.svelte b/frontend/src/lib/components/home/TutorialBanner.svelte index 14d8b750c1..ef7b30c27e 100644 --- a/frontend/src/lib/components/home/TutorialBanner.svelte +++ b/frontend/src/lib/components/home/TutorialBanner.svelte @@ -1,6 +1,7 @@ {#if !isDismissed} -
-
- -
-
- {#if hasCompletedAny} - New tutorial available! - {:else} - Learn with interactive tutorials - {/if} -
-
- {#if hasCompletedAny} - Continue your learning journey and master new Windmill skills. - {:else} - Get started quickly with step-by-step guides on building flows, scripts, and more. - {/if} -
-
-
-
- - -
+ +
+ + {#if hasCompletedAny} + New tutorial available! + {:else} + First time? + {/if} + + +
{/if} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 24e7ac6438..8bbca5b4be 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -30,6 +30,10 @@ export interface Setting { placeholder?: string cloudonly?: boolean ee_only?: string + /** Ceiling a `seconds` field enforces on a build without a license, when CE genuinely caps + * the value. Not implied by `ee_only`: a setting can be EE-badged because the feature it + * configures is EE while the value itself has the same range on either edition. */ + ceMaxSeconds?: number tooltip?: string key: string // If value is not specified for first element, it will automatcally use undefined @@ -283,6 +287,8 @@ export const settings: Record = { placeholder: '30', storage: 'setting', ee_only: 'You can only adjust this setting to above 30 days in the EE version', + // Mirrors CE_MAX_RETENTION_PERIOD_SECS, which the backend clamps to on write. + ceMaxSeconds: 60 * 60 * 24 * 30, cloudonly: false }, { @@ -656,6 +662,16 @@ export const settings: Record = { fieldType: 'text', placeholder: 'okta', storage: 'setting' + }, + { + label: 'SSO groups claim', + description: + 'Name of the SAML attribute or OIDC userinfo claim carrying the user\'s IdP groups ("http://schemas.microsoft.com/ws/2008/06/identity/claims/groups" on Entra SAML, "groups" for most OIDC providers). Its values must be the same group ids that SCIM stored as the instance groups\' external id (Entra emits object ids in both), since matching is by external id only. When set, every SSO login reconciles the user\'s membership in those SCIM-provisioned instance groups against the claim, so IdP group changes take effect at the next login instead of waiting for the SCIM push. Instance groups without an external id are never touched, and a login whose claim is absent or empty changes nothing. Leave empty to disable.', + key: 'sso_groups_claim', + fieldType: 'text', + placeholder: 'groups', + storage: 'setting', + ee_only: '' } ], 'DB Health': [], @@ -971,6 +987,24 @@ export const settings: Record = { triggersRestart: true, defaultValue: () => ({ enabled: false, enabled_languages: [...OTEL_TRACING_PROXY_LANGUAGES] }) }, + { + label: 'HTTP Request Tracing retention in secs', + key: 'otel_traces_retention_secs', + description: + 'How long a captured HTTP request span is kept in the database, and therefore how far back the job details view can show a job its requests. Independent of the job retention period, so a span may outlive its job or be swept while the job remains. Defaults to 7 days. Leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + // Badged EE because only the EE proxy captures spans, but deliberately no + // `ceMaxSeconds`: a CE build still sweeps rows an EE-era instance left behind, and + // the backend accepts the same range on either edition. + ee_only: 'HTTP Request Tracing is an EE feature', + error: + 'HTTP Request Tracing retention must be between 1 second and 100 years, leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + }, { label: 'Prometheus', description: diff --git a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte index da63e8d64a..acd18130da 100644 --- a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte @@ -283,7 +283,17 @@ placeholder: '12345', disabled: fieldsDisabled }} - bind:value={$values['github_enterprise_app'].app_id} + bind:value={ + () => $values['github_enterprise_app'].app_id, + (v) => { + // The backend expects app_id as a positive integer (i64). Reject + // fractional/out-of-range values instead of truncating them, and store + // undefined (never a string or 0) so the config omits the key when unset. + const n = typeof v === 'string' ? Number(v.trim() || NaN) : (v ?? NaN) + $values['github_enterprise_app'].app_id = + Number.isSafeInteger(n) && n > 0 ? n : undefined + } + } />
diff --git a/frontend/src/lib/components/mcp/enabledServers.ts b/frontend/src/lib/components/mcp/enabledServers.ts index 4a7bb7bae2..b49b788c67 100644 --- a/frontend/src/lib/components/mcp/enabledServers.ts +++ b/frontend/src/lib/components/mcp/enabledServers.ts @@ -1,66 +1,11 @@ -import { get } from 'svelte/store' -import { userStore } from '$lib/stores' +import { createEnabledPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' -/** - * Which MCP servers the chat may use, per workspace and per account. - * - * Being able to read an `mcp` resource is not the same as wanting the chat to - * act through it: a resource in a shared folder is readable by a whole team, and - * each server's tools both reach an external system and put their descriptions - * in the model's context. So a server is off until it is turned on here, and - * connecting one through the chat turns it on for the person who connected it. - * - * Stored per browser, like the chat's other per-user preferences, but keyed by - * email as well as workspace: browser storage outlives a logout, and inheriting - * the previous account's enabled servers would hand the next person tools they - * never turned on. - */ -const KEY = 'wm_mcp_enabled' +/** Which MCP servers the chat may act through, per workspace and per account. A + * server's tools both reach an external system and put their descriptions in the + * model's context, so one is off until it is turned on; connecting one through the + * chat turns it on for the person who connected it. */ +const preference = createEnabledPathsPreference('wm_mcp_enabled') -function scope(workspace: string): string | undefined { - const email = get(userStore)?.email - return email ? `${workspace}:${email}` : undefined -} - -function read(): Record { - if (typeof localStorage === 'undefined') return {} - try { - return JSON.parse(localStorage.getItem(KEY) ?? '{}') - } catch { - return {} - } -} - -function write(all: Record) { - try { - localStorage.setItem(KEY, JSON.stringify(all)) - } catch (e) { - console.error('Failed to persist enabled MCP servers', e) - } -} - -export function enabledMcpPaths(workspace: string): string[] { - const key = scope(workspace) - return key ? (read()[key] ?? []) : [] -} - -export function isMcpEnabled(workspace: string, path: string): boolean { - return enabledMcpPaths(workspace).includes(path) -} - -/** Returns false when there is no account to record the preference against, so a - * caller that just connected a server can say it did not stay on. */ -export function setMcpEnabled(workspace: string, path: string, enabled: boolean): boolean { - const key = scope(workspace) - if (!key) return false - const all = read() - const current = new Set(all[key] ?? []) - if (enabled) { - current.add(path) - } else { - current.delete(path) - } - all[key] = [...current] - write(all) - return true -} +export const enabledMcpPaths = preference.enabledPaths +export const isMcpEnabled = preference.isEnabled +export const setMcpEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte b/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte index bafa276697..12ea1557b0 100644 --- a/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte @@ -404,70 +404,73 @@ {/if}
-
-

- - Start with AI - (optional) -

+ {#if !$copilotInfo.workspaceDisabled} +
+

+ + Start with AI + (optional) +

- {#if !aiConfigLoaded} -
- - Loading AI settings... -
- {:else if !isAiEnabled} - - You can still create an app manually but using AI is highly recommended. -
- {#if $userStore?.is_admin} - Configure AI in - workspace settings - - {#if $superadmin} - or + {#if !aiConfigLoaded} +
+ + Loading AI settings... +
+ {:else if !isAiEnabled} + + You can still create an app manually but using AI is highly recommended. +
+ {#if $userStore?.is_admin} + Configure AI in + workspace settings + + {#if $superadmin} + or + instance settings + + {/if} to enable this feature. + {:else if $superadmin} + Configure AI in instance settings - - {/if} to enable this feature. - {:else if $superadmin} - Configure AI in - instance settings - to enable this feature. - {:else} - Ask your workspace admin to configure AI in workspace settings to enable this feature. - {/if} - - {:else} -
- -

- {handsOffToSession - ? 'Leave empty to start with a blank template, or describe your app to open an AI session that builds it.' - : 'Leave empty to start with a blank template, or describe your app to get AI assistance right away.'} -

-
- {/if} -
+ to enable this feature. + {:else} + Ask your workspace admin to configure AI in workspace settings to enable this + feature. + {/if} + + {:else} +
+ +

+ {handsOffToSession + ? 'Leave empty to start with a blank template, or describe your app to open an AI session that builds it.' + : 'Leave empty to start with a blank template, or describe your app to get AI assistance right away.'} +

+
+ {/if} +
+ {/if}
{#if isAiEnabled}
- {#if (itemMap[tab] ?? []).length === 0 && searchTerm.length > 0} + {#if (itemMap[tab] ?? []).length === 0 && searchTerm.length > 0 && !$copilotInfo.workspaceDisabled} - { - askAiButton?.onClick() - }} - id={'ai:no-results-ask-ai'} - hovered={true} - label={`Try asking \`${searchTerm}\` to AI`} - icon={WandSparkles} - bind:mouseMoved - /> + {#if !$copilotInfo.workspaceDisabled} + { + askAiButton?.onClick() + }} + id={'ai:no-results-ask-ai'} + hovered={true} + label={`Try asking \`${searchTerm}\` to AI`} + icon={WandSparkles} + bind:mouseMoved + /> + {/if}
Tip: press `esc` to quickly clear the search bar
diff --git a/frontend/src/lib/components/sessions/OpenInSessionButton.svelte b/frontend/src/lib/components/sessions/OpenInSessionButton.svelte index 00bf1c829e..03ce21589e 100644 --- a/frontend/src/lib/components/sessions/OpenInSessionButton.svelte +++ b/frontend/src/lib/components/sessions/OpenInSessionButton.svelte @@ -40,6 +40,7 @@ import AIButton from '$lib/components/copilot/chat/AIButton.svelte' import { AIBtnClasses } from '$lib/components/copilot/chat/AIButtonStyle' import { prefersSessionHandoff } from '$lib/components/copilot/chat/global/gate' + import { copilotInfo } from '$lib/aiStore' import { userStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { openSourceInSession } from './sessionSwitch.svelte' @@ -80,7 +81,9 @@ // them, so an entry point on a page they can reach (Runs, the trigger lists) // would only route them into that refusal. const show = $derived( - !inSessionPanel && !!(source?.target || source?.page) && prefersSessionHandoff($userStore?.operator) + !inSessionPanel && + !!(source?.target || source?.page) && + prefersSessionHandoff($userStore?.operator) ) // Not $state: only read inside open() as a re-entrancy latch, never rendered. @@ -102,7 +105,10 @@ } -{#if show} +{#if $copilotInfo.workspaceDisabled} + +{:else if show} void } = $props() + // The group's highlighted side. Melt moves it on click, before the navigation + // that would change `mode`, so it is derived from the route (which wins once + // a switch navigates) and pushed back when one does not, or the rail would + // read "AI Sessions" on an editor page with the clicked side inert until + // "Workspace" was pressed first. + let selected: string | string[] | null | undefined = $derived(mode) + function onSelected(next: 'nav' | 'session') { if (next === mode) return onToggle?.() - if (next === 'session') void enterSessionMode() - else void exitSessionMode() + if (next === 'session') { + // An editor whose draft could not be persisted keeps the user on the + // page, as its own "Open in AI session" button does, rather than open a + // session on an older draft than the one on screen. + void enterSessionModeFromNav().catch((e) => { + selected = mode + sendUserToast(e instanceof Error ? e.message : String(e), true) + }) + } else void exitSessionMode() } // Pressing the already-active "Workspace" side goes home, so the toggle doubles @@ -41,7 +56,7 @@ child of the group's track — so the buttons fill the rail width only if those wrappers grow. `[&>*]:flex-1` makes every direct child split the track evenly. --> *]:w-full' : 'w-full [&>*]:flex-1'} > diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index 87462e301b..e5b62da0eb 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -1,5 +1,6 @@ - - - -{#snippet pasteZone()} - - {#if editingOriginalName || pasteContent.trim()} -
- {isDirty && pasteError ? pasteError : ''} - -
- {/if} -{/snippet} - - - {#snippet headerAction()} -
- {#if manageMode} - - - {:else} - {#if skills.length > 1} - - {/if} - - {/if} -
- {#if $addMenuOpen} -
- - -
- {/if} - {/snippet} - -
- {#if skills.length === 0} -
- No custom skills yet -
- {:else} -
- {#if manageMode} -
- 0 && !allSelected} - onChange={toggleSelectAll} - /> - - {selectedCount ? `${selectedCount} selected` : 'Select all'} - -
- {/if} - {#each skills as skill (skill.name)} -
- {#if manageMode} - - {:else} -
-
{skill.name}
-
{skill.description}
- -
- openSkill(skill.name, 'edit') - }, - { - displayName: 'Delete', - icon: Trash2, - type: 'delete', - action: () => (toDelete = skill.name) - } - ]} - /> - {/if} -
- {/each} -
- {/if} -
- - - - - - - {#snippet headerRight()} - {#if editingOriginalName} - - {#snippet children({ item })} - - - {/snippet} - - {/if} - {/snippet} -
- {#if detailMode === 'view'} -
- {#if viewParsed.description} -

{viewParsed.description}

- {/if} -
- -
-
- {:else} - {@render pasteZone()} - {/if} -
- - - { - const toImport = [...pendingNew, ...pendingConflicts.filter((s) => overwriteChoices[s.name])] - const skipped = pendingSkipped - pendingImport = undefined - pendingSkipped = [] - overwriteChoices = {} - if (toImport.length) await uploadSkills(toImport, skipped) - else sendUserToast('No skills imported.') - }} - onCanceled={() => { - pendingImport = undefined - pendingSkipped = [] - overwriteChoices = {} - }} -> -
- {#if pendingNew.length} -
- Add {pendingNew.length} new skill(s): - {pendingNew.map((s) => s.name).join(', ')} -
- {/if} - {#if pendingConflicts.length} -
- - {pendingConflicts.length} skill(s) already exist — choose which to overwrite: - -
- {#each pendingConflicts as conflict (conflict.name)} -
- {conflict.name} - -
- {/each} -
-
- {/if} - {#if pendingSkipped.length} - {pendingSkipped.length} file(s) will be skipped. - {/if} -
- - - { - const name = toDelete - toDelete = undefined - if (name) await deleteSkill(name) - }} - onCanceled={() => (toDelete = undefined)} -> - - Delete the skill {toDelete}? The AI chat will no longer be able to use it. - - - - { - confirmBatchDelete = false - await deleteSelected() - }} - onCanceled={() => (confirmBatchDelete = false)} -> - - Delete {selectedCount} selected skill(s)? The AI chat will no longer be able to use them. - - diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts index 669c0a43de..1833a33b33 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -16,6 +16,8 @@ import { collectExportVarPaths, extractTriggerConfigResourceRefs, extractVarRefsFromValue, + projectReferencesResource, + textHoldsBarePath, type ProjectExport, type FetchedItem, type ItemRef @@ -867,3 +869,46 @@ describe('flow_env and preprocessor_module', () => { expect(out.flow_env.PLAIN).toBe('not-a-ref') }) }) + +describe('projectReferencesResource', () => { + /** + * A project declares one resource per `resource-` input schema as well as one per + * `$res:` reference, so an app pinning `f/proj/google_calendar` for a script whose schema + * says `resource-gcal` ships both it and an unreferenced `f/proj/gcal`. Only the pinned + * one has to hold a credential for the project to work. + */ + const bundle = { + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [{ path: 'f/proj/send', content: 'const c = "$res:f/proj/google_calendar"' }], + flows: [], + apps: [], + resources: [ + { path: 'f/proj/gcal', resource_type: 'gcal' }, + { path: 'f/proj/google_calendar', resource_type: 'gcal' }, + { path: 'f/proj/db', resource_type: 'postgresql' } + ], + triggers: [{ path: 'f/proj/ingest', config: { postgres_resource_path: 'f/proj/db' } }] + } as unknown as ProjectExport + + it("sees a $res: token and a trigger's bare path, and not a stub nothing points at", () => { + expect(projectReferencesResource(bundle, 'f/proj/google_calendar')).toBe(true) + expect(projectReferencesResource(bundle, 'f/proj/db')).toBe(true) + // Declared only because a script's input schema names the type. + expect(projectReferencesResource(bundle, 'f/proj/gcal')).toBe(false) + }) +}) + +describe('textHoldsBarePath', () => { + // Gates three deletion decisions, and its two directions cost differently: a false yes + // keeps a stub nobody needed, a false no deletes one something still reads. + const P = 'f/proj/db' + it.each([ + ['a token only', `const c = "$res:${P}"`, false], + ['a path written in code', `await getResource("${P}")`, true], + ['both spellings', `"$res:${P}"; getResource("${P}")`, true], + ['a longer path that starts the same', `await getResource("${P}_prod")`, false], + ['the same path inside a longer token', `"$res:${P}_prod"`, false] + ])('%s', (_label, text, expected) => { + expect(textHoldsBarePath(text, P)).toBe(expected) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts index 1fd62498d5..8adedeb559 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -238,6 +238,89 @@ export function extractTriggerConfigResourceRefs(config: any): string[] { return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) } +/** + * Whether a value reaches a resource, in either spelling a rewrite has to handle: a + * `$res:`/`res://` token embedded in content, or the bare path standing alone as a string + * the way trigger configs hold it (`kafka_resource_path: "f/slug/db"`). + * + * Matching the parsed structure rather than its serialization is what keeps + * `f/slug/db` out of `$res:f/slug/db_prod`. + */ +export function referencesResourcePath(value: unknown, path: string): boolean { + const walk = (v: any): boolean => { + if (typeof v === 'string') { + if (v === path) return true + RES_TOKEN_RE.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = RES_TOKEN_RE.exec(v)) !== null) if (m[1] === path) return true + return false + } + if (Array.isArray(v)) return v.some(walk) + if (v && typeof v === 'object') return Object.values(v).some(walk) + return false + } + return walk(value) +} + +/** + * Whether a `$res:`/`res://` token for this path survives anywhere in the value — the one + * spelling the rewriters relocate, and so the only one whose survival means a rewrite did + * not take. A bare path is deliberately not matched: nothing here moves one, so its presence + * says nothing about whether the rewrite worked. + */ +export function holdsResourceToken(value: unknown, path: string): boolean { + const walk = (v: any): boolean => { + if (typeof v === 'string') { + RES_TOKEN_RE.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = RES_TOKEN_RE.exec(v)) !== null) if (m[1] === path) return true + return false + } + if (Array.isArray(v)) return v.some(walk) + if (v && typeof v === 'object') return Object.values(v).some(walk) + return false + } + return walk(value) +} + +/** + * Whether the text names the resource somewhere no rewriter reaches — a path written on its + * own rather than inside a `$res:` token, the way `getResource("f/proj/db")` does. The + * tokens are stripped first so the ones a rewrite would move do not count, and the match is + * bounded so `f/proj/db` is not found inside `f/proj/db_prod`. + */ +export function textHoldsBarePath(text: string, path: string): boolean { + const withoutTokens = text.replace(RES_TOKEN_RE, '') + const boundary = /[\w\-./]/ + for (let i = withoutTokens.indexOf(path); i !== -1; i = withoutTokens.indexOf(path, i + 1)) { + const before = withoutTokens[i - 1] ?? '' + const after = withoutTokens[i + path.length] ?? '' + if (!boundary.test(before) && !boundary.test(after)) return true + } + return false +} + +/** + * Whether anything the project ships points at one of its own resources. + * + * A project declares two kinds of resource. One is referenced — an app pins `$res:` for a + * script argument, a trigger names it — and the project does not work until it holds a + * credential. The other is minted from a `resource-` input schema: it names a type + * a script accepts, nothing points at it, and a standalone run picks a resource in the + * argument picker instead. Only the first kind is worth asking anyone to fill in. + * + * The `resources` list is excluded from the walk because a stub's own declaration carries + * its path, which would make every stub look referenced. + */ +export function projectReferencesResource(bundle: ProjectExport, path: string): boolean { + const { resources: _resources, ...rest } = bundle as any + if (referencesResourcePath(rest, path)) return true + // A script that reads the resource by name rather than through a `$res:` token still needs + // it filled in. Asked about is the safe side of this answer: the cost of a wrong yes is a + // row nobody had to act on, and of a wrong no a credential nobody was told to set up. + return textHoldsBarePath(JSON.stringify(rest ?? {}), path) +} + /** * Trigger configs reference resources as plain path strings (e.g. * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting diff --git a/frontend/src/lib/folderDraft.test.ts b/frontend/src/lib/folderDraft.test.ts new file mode 100644 index 0000000000..75b8fa6dd0 --- /dev/null +++ b/frontend/src/lib/folderDraft.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect } from 'vitest' +import { + folderPermissionDiff, + isFolderDraftDirty, + type FolderDraft, + type FolderMember, + type FolderRole +} from './folderDraft' + +function member(role: FolderRole): FolderMember { + return { owner_name: 'u/alice', role } +} + +function baseline(): FolderDraft { + return { + summary: 'Reporting jobs', + labels: ['prod'], + defaultPermissionedAs: [{ path_glob: '**', permissioned_as: 'u/admin' }], + perms: [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'viewer' } + ] + } +} + +describe('folderPermissionDiff', () => { + // The whole transition matrix: which endpoint each role change maps to. `admin` lives in + // `owners` and the other two in `extra_perms`, so leaving admin is the one transition that + // cannot go through the ACL endpoint. + const transitions: Array<[from: FolderRole | 'absent', to: FolderRole, expected: unknown]> = [ + ['absent', 'viewer', { kind: 'setAcl', owner: 'u/alice', write: false }], + ['absent', 'writer', { kind: 'setAcl', owner: 'u/alice', write: true }], + ['absent', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['viewer', 'writer', { kind: 'setAcl', owner: 'u/alice', write: true }], + ['viewer', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['writer', 'viewer', { kind: 'setAcl', owner: 'u/alice', write: false }], + ['writer', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['admin', 'viewer', { kind: 'demoteAdmin', owner: 'u/alice', write: false }], + ['admin', 'writer', { kind: 'demoteAdmin', owner: 'u/alice', write: true }] + ] + + it.each(transitions)('%s → %s', (from, to, expected) => { + const prev = from === 'absent' ? [] : [member(from)] + expect(folderPermissionDiff(prev, [member(to)])).toEqual([expected]) + }) + + it.each(['viewer', 'writer', 'admin'] as const)('%s → removed drops owner and acl', (role) => { + expect(folderPermissionDiff([member(role)], [])).toEqual([{ kind: 'remove', owner: 'u/alice' }]) + }) + + it.each(['viewer', 'writer', 'admin'] as const)('%s unchanged calls nothing', (role) => { + expect(folderPermissionDiff([member(role)], [member(role)])).toEqual([]) + }) + + // The caller is a folder admin only through `g/ops`, so that demotion is the one the write + // policy refuses. Sent first it takes the rest of the save down with it. + it('gives up the caller own admin last', () => { + const prev: FolderMember[] = [ + { owner_name: 'g/ops', role: 'admin' }, + { owner_name: 'u/bob', role: 'viewer' } + ] + const next: FolderMember[] = [ + { owner_name: 'g/ops', role: 'viewer' }, + { owner_name: 'u/bob', role: 'admin' } + ] + expect(folderPermissionDiff(prev, next, ['u/alice', 'g/ops'])).toEqual([ + { kind: 'grantAdmin', owner: 'u/bob' }, + { kind: 'demoteAdmin', owner: 'g/ops', write: false } + ]) + }) + + // `g/z` is a group the caller belongs to but holds no admin through, so removing it is an + // ordinary call — queued behind the refused one it would never run. + it('defers only the rows the caller is an admin through', () => { + const prev: FolderMember[] = [ + { owner_name: 'g/a', role: 'admin' }, + { owner_name: 'g/z', role: 'viewer' } + ] + expect(folderPermissionDiff(prev, [], ['u/alice', 'g/a', 'g/z'])).toEqual([ + { kind: 'remove', owner: 'g/z' }, + { kind: 'remove', owner: 'g/a' } + ]) + }) + + it('touches only the members that changed', () => { + const prev: FolderMember[] = [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'viewer' }, + { owner_name: 'g/ops', role: 'writer' } + ] + const next: FolderMember[] = [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'writer' } + ] + expect(folderPermissionDiff(prev, next)).toEqual([ + { kind: 'setAcl', owner: 'g/all', write: true }, + { kind: 'remove', owner: 'g/ops' } + ]) + }) +}) + +describe('isFolderDraftDirty', () => { + it('is clean against its own baseline', () => { + expect(isFolderDraftDirty(baseline(), baseline())).toBe(false) + }) + + it('is clean before anything has loaded', () => { + expect(isFolderDraftDirty(baseline(), undefined)).toBe(false) + }) + + // A reload rebuilds the members in the server's order, which is not the order they were + // added in. Order-sensitive, an applied change would keep Save lit with nothing to send. + it('ignores the order the members are held in', () => { + const reordered = baseline() + reordered.perms = [...reordered.perms].reverse() + expect(isFolderDraftDirty(reordered, baseline())).toBe(false) + }) + + // Enumerated from the value itself rather than a hand-written list: a field added to + // `FolderDraft` and to `baseline()` is covered here without anyone remembering to add a + // case. An edit this misses is one the drawer discards without asking. + it.each(Object.keys(baseline()) as Array)('notices a change to %s', (key) => { + const edited = baseline() + if (key === 'summary') edited.summary = 'Something else' + else if (key === 'labels') edited.labels = [...edited.labels, 'staging'] + else if (key === 'defaultPermissionedAs') edited.defaultPermissionedAs = [] + else if (key === 'perms') edited.perms[1].role = 'writer' + else throw new Error(`no edit defined for ${key} — add one so the field stays covered`) + + expect(isFolderDraftDirty(edited, baseline())).toBe(true) + }) + + it('notices a member added and a member removed', () => { + const added = baseline() + added.perms.push({ owner_name: 'g/ops', role: 'writer' }) + expect(isFolderDraftDirty(added, baseline())).toBe(true) + + const removed = baseline() + removed.perms.pop() + expect(isFolderDraftDirty(removed, baseline())).toBe(true) + }) + + it('is clean again once the baseline catches up', () => { + const saved = baseline() + saved.summary = 'Renamed' + expect(isFolderDraftDirty(saved, structuredClone(saved))).toBe(false) + }) +}) diff --git a/frontend/src/lib/folderDraft.ts b/frontend/src/lib/folderDraft.ts new file mode 100644 index 0000000000..ade1b43d71 --- /dev/null +++ b/frontend/src/lib/folderDraft.ts @@ -0,0 +1,96 @@ +import { deepEqual } from 'fast-equals' +import type { FolderDefaultPermissionedAs } from '$lib/gen' + +/** What a member may hold on a folder. `admin` is the `owners` array server-side; `writer` + * and `viewer` are the `true`/`false` entries of `extra_perms`. */ +export type FolderRole = 'viewer' | 'writer' | 'admin' + +export type FolderMember = { owner_name: string; role: FolderRole } + +/** Everything the folder editor can change, held as one value so the whole edit is one + * comparison against the loaded folder and one Save. */ +export type FolderDraft = { + summary: string + labels: string[] + defaultPermissionedAs: FolderDefaultPermissionedAs + perms: FolderMember[] +} + +/** Whether the draft still matches the folder it was loaded from. Every field of + * `FolderDraft` participates, so a field added to the type is covered by construction — + * which is what the discard guard depends on: an edit this misses is an edit the drawer + * throws away without asking. No baseline means nothing has loaded yet, so nothing to lose. */ +export function isFolderDraftDirty(draft: FolderDraft, baseline: FolderDraft | undefined): boolean { + return baseline != undefined && !deepEqual(sortedMembers(draft), sortedMembers(baseline)) +} + +/** Members are a set, but a reload rebuilds them in the server's `extra_perms` key order while + * the draft keeps the order they were added in. Compared as-is, a change that has already been + * applied still reads as dirty. Labels and rules keep their order, which is meaningful. */ +function sortedMembers(value: FolderDraft): FolderDraft { + return { + ...value, + perms: [...value.perms].sort((a, b) => a.owner_name.localeCompare(b.owner_name)) + } +} + +/** One backend call the folder's members need. Kept as data so the mapping from role + * transitions to endpoints can be read — and tested — without a server. */ +export type FolderPermissionCall = + /** `addowner`: appends to `owners` and sets `extra_perms[owner] = true`. */ + | { kind: 'grantAdmin'; owner: string } + /** `removeowner` with a write flag: takes the member out of `owners` and sets their + * level. The only way down from admin. */ + | { kind: 'demoteAdmin'; owner: string; write: boolean } + /** `acls/add`: sets `extra_perms[owner]`, for a member who is not an admin. */ + | { kind: 'setAcl'; owner: string; write: boolean } + /** Both removals. `removeowner` without a write only drops the member from `owners`, + * leaving their `extra_perms` entry — alone it demotes an admin rather than removing + * them, so the ACL delete is not optional. */ + | { kind: 'remove'; owner: string } + +/** The calls that turn `prev` into `next`. Members whose role is unchanged produce none. + * + * `callerOwners` is the caller's own `u/name` plus every group they belong to. Giving up the + * last of those that is in `owners` goes last: the write policy checks the row the update + * would produce, so that call is refused for anyone but a workspace admin, and sent early it + * takes the rest of the save with it. */ +export function folderPermissionDiff( + prev: FolderMember[], + next: FolderMember[], + callerOwners?: string[] +): FolderPermissionCall[] { + const previousRole = new Map(prev.map((p) => [p.owner_name, p.role])) + const calls: FolderPermissionCall[] = [] + + for (const member of next) { + const before = previousRole.get(member.owner_name) + if (before === member.role) continue + if (member.role === 'admin') { + calls.push({ kind: 'grantAdmin', owner: member.owner_name }) + } else if (before === 'admin') { + calls.push({ + kind: 'demoteAdmin', + owner: member.owner_name, + write: member.role === 'writer' + }) + } else { + calls.push({ kind: 'setAcl', owner: member.owner_name, write: member.role === 'writer' }) + } + } + + const kept = new Set(next.map((n) => n.owner_name)) + for (const member of prev) { + if (kept.has(member.owner_name)) continue + calls.push({ kind: 'remove', owner: member.owner_name }) + } + + // `previousRole === 'admin'` is what makes it a handle: `callerOwners` lists every group + // the caller belongs to, and one holding only a viewer or writer row is not in `owners`, + // so removing it is an ordinary call that should not queue behind the fatal one. + const revokesCaller = (call: FolderPermissionCall) => + (call.kind === 'demoteAdmin' || call.kind === 'remove') && + (callerOwners?.includes(call.owner) ?? false) && + previousRole.get(call.owner) === 'admin' + return [...calls.filter((c) => !revokesCaller(c)), ...calls.filter(revokesCaller)] +} diff --git a/frontend/src/lib/groupDraft.test.ts b/frontend/src/lib/groupDraft.test.ts new file mode 100644 index 0000000000..2b8ef8c180 --- /dev/null +++ b/frontend/src/lib/groupDraft.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest' +import { groupMemberDiff, isGroupDraftDirty, type GroupDraft, type GroupRole } from './groupDraft' + +function baseline(): GroupDraft { + return { + summary: 'On-call engineers', + members: [ + { member_name: 'admin', role: 'admin' }, + { member_name: 'alice', role: 'member' } + ] + } +} + +describe('groupMemberDiff', () => { + // The whole transition matrix: which endpoints each role change maps to. A role is a + // membership row plus an ACL entry, so only the halves that actually change are sent — + // an extra call would log a permission-history row for something that did not move. + const transitions: Array< + [from: GroupRole | 'absent', to: GroupRole | 'absent', expected: unknown[]] + > = [ + ['absent', 'member', [{ kind: 'addUser', username: 'bob' }]], + ['absent', 'manager', [{ kind: 'setAcl', username: 'bob' }]], + [ + 'absent', + 'admin', + [ + { kind: 'addUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' } + ] + ], + ['member', 'admin', [{ kind: 'setAcl', username: 'bob' }]], + [ + 'member', + 'manager', + [ + { kind: 'removeUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' } + ] + ], + ['manager', 'admin', [{ kind: 'addUser', username: 'bob' }]], + [ + 'manager', + 'member', + [ + { kind: 'addUser', username: 'bob' }, + { kind: 'removeAcl', username: 'bob' } + ] + ], + ['admin', 'member', [{ kind: 'removeAcl', username: 'bob' }]], + ['admin', 'manager', [{ kind: 'removeUser', username: 'bob' }]], + ['member', 'absent', [{ kind: 'removeUser', username: 'bob' }]], + ['manager', 'absent', [{ kind: 'removeAcl', username: 'bob' }]], + [ + 'admin', + 'absent', + [ + { kind: 'removeUser', username: 'bob' }, + { kind: 'removeAcl', username: 'bob' } + ] + ] + ] + + for (const [from, to, expected] of transitions) { + it(`${from} to ${to}`, () => { + const prev = from === 'absent' ? [] : [{ member_name: 'bob', role: from }] + const next = to === 'absent' ? [] : [{ member_name: 'bob', role: to }] + expect(groupMemberDiff(prev, next)).toEqual(expected) + }) + } + + it('sends nothing for an unchanged member', () => { + expect(groupMemberDiff(baseline().members, baseline().members)).toEqual([]) + }) + + it('revokes the caller last so the rest of the save stays authorized', () => { + const prev = [{ member_name: 'admin', role: 'admin' as GroupRole }] + const next = [ + { member_name: 'admin', role: 'member' as GroupRole }, + { member_name: 'bob', role: 'admin' as GroupRole } + ] + expect(groupMemberDiff(prev, next, 'admin')).toEqual([ + { kind: 'addUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' }, + { kind: 'removeAcl', username: 'admin' } + ]) + }) +}) + +describe('isGroupDraftDirty', () => { + it('is clean against an equal baseline and dirty on any field', () => { + expect(isGroupDraftDirty(baseline(), baseline())).toBe(false) + expect(isGroupDraftDirty({ ...baseline(), summary: 'Other' }, baseline())).toBe(true) + expect( + isGroupDraftDirty( + { ...baseline(), members: [{ member_name: 'admin', role: 'member' }] }, + baseline() + ) + ).toBe(true) + }) + + it('is clean while nothing has loaded', () => { + expect(isGroupDraftDirty(baseline(), undefined)).toBe(false) + }) + + // A reload rebuilds the members in the server's order, which is not the order they were + // added in. Order-sensitive, an applied change would keep Save lit with nothing to send. + it('ignores the order the members are held in', () => { + const reordered = baseline() + reordered.members = [...reordered.members].reverse() + expect(isGroupDraftDirty(reordered, baseline())).toBe(false) + }) +}) diff --git a/frontend/src/lib/groupDraft.ts b/frontend/src/lib/groupDraft.ts new file mode 100644 index 0000000000..dfd1767c30 --- /dev/null +++ b/frontend/src/lib/groupDraft.ts @@ -0,0 +1,99 @@ +import { deepEqual } from 'fast-equals' + +/** What a member may hold on a group. `member` is the `usr_to_group` row server-side and + * `manager` is the `true` entry in `extra_perms`; `admin` is both at once. */ +export type GroupRole = 'member' | 'manager' | 'admin' + +export type GroupMember = { member_name: string; role: GroupRole } + +/** Everything the group editor can change, held as one value so the whole edit is one + * comparison against the loaded group and one Save. */ +export type GroupDraft = { + summary: string + members: GroupMember[] +} + +/** Whether the draft still matches the group it was loaded from. Every field of `GroupDraft` + * participates, so a field added to the type is covered by construction — which is what the + * discard guard depends on: an edit this misses is an edit the drawer throws away without + * asking. No baseline means nothing has loaded yet, so nothing to lose. */ +export function isGroupDraftDirty(draft: GroupDraft, baseline: GroupDraft | undefined): boolean { + return baseline != undefined && !deepEqual(sortedMembers(draft), sortedMembers(baseline)) +} + +/** Members are a set, but a reload rebuilds them in the server's order while the draft keeps + * the order they were added in. Compared as-is, a change that has already been applied still + * reads as dirty. */ +function sortedMembers(value: GroupDraft): GroupDraft { + return { + ...value, + members: [...value.members].sort((a, b) => a.member_name.localeCompare(b.member_name)) + } +} + +/** One backend call a group's members need. Kept as data so the mapping from role + * transitions to endpoints can be read — and tested — without a server. */ +export type GroupMemberCall = + /** `addUserToGroup` / `removeUserToGroup`: the `usr_to_group` row. */ + | { kind: 'addUser'; username: string } + | { kind: 'removeUser'; username: string } + /** `acls/add` / `acls/remove` on kind `group_`: the write entry that lets someone + * manage the group. */ + | { kind: 'setAcl'; username: string } + | { kind: 'removeAcl'; username: string } + +/** The two independent things a role is made of: belonging to the group, and holding the + * write entry that lets you manage it. Every role is one combination of the two, which is + * why a transition needs at most one call per flag. */ +function flagsOf(role: GroupRole | undefined): { belongs: boolean; manages: boolean } { + return { + belongs: role === 'member' || role === 'admin', + manages: role === 'manager' || role === 'admin' + } +} + +/** The calls that turn `prev` into `next`. Members whose role is unchanged produce none, and + * a member dropped from `next` is treated as holding neither flag — which is what removing + * one means. + * + * `require_is_owner` authorizes each of these against `extra_perms['u/']`, so the + * caller's own revocation goes last: in row order it lands first and the rest 403s. */ +export function groupMemberDiff( + prev: GroupMember[], + next: GroupMember[], + caller?: string +): GroupMemberCall[] { + const previousRole = new Map(prev.map((p) => [p.member_name, p.role])) + const calls: GroupMemberCall[] = [] + + const transition = ( + username: string, + before: GroupRole | undefined, + after: GroupRole | undefined + ) => { + const from = flagsOf(before) + const to = flagsOf(after) + if (to.belongs !== from.belongs) { + calls.push({ kind: to.belongs ? 'addUser' : 'removeUser', username }) + } + if (to.manages !== from.manages) { + calls.push({ kind: to.manages ? 'setAcl' : 'removeAcl', username }) + } + } + + for (const member of next) { + const before = previousRole.get(member.member_name) + if (before === member.role) continue + transition(member.member_name, before, member.role) + } + + const kept = new Set(next.map((n) => n.member_name)) + for (const member of prev) { + if (kept.has(member.member_name)) continue + transition(member.member_name, member.role, undefined) + } + + const revokesCaller = (call: GroupMemberCall) => + call.kind === 'removeAcl' && call.username === caller + return [...calls.filter((c) => !revokesCaller(c)), ...calls.filter(revokesCaller)] +} diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 2862fc7768..ee60298db4 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28184/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28910/git-sync-init-repository-windmill", + "gitInitRepo": "hub/28930/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", diff --git a/frontend/src/lib/importWizard/execution.svelte.ts b/frontend/src/lib/importWizard/execution.svelte.ts index 06c33f3222..6a2c39b0b6 100644 --- a/frontend/src/lib/importWizard/execution.svelte.ts +++ b/frontend/src/lib/importWizard/execution.svelte.ts @@ -8,9 +8,10 @@ import { installProject, type InstallResult } from '$lib/components/workspaceSettings/projectInstall' -import type { - ProjectExport, - ProjectMigration +import { + projectReferencesResource, + type ProjectExport, + type ProjectMigration } from '$lib/components/workspaceSettings/projectBundle' import { planWorkspaceId, type ImportPlan } from './plan' import { probeImportedPaths, probeWorkspace } from './probe' @@ -110,12 +111,16 @@ export class ImportExecution { } /** - * How many resources the project shipped. Every one arrives as an empty stub — - * the hub never publishes resource values — so a non-zero count means the setup - * step has something to offer. + * How many of the project's resources the setup step will ask about. Every resource + * arrives as an empty stub — the hub never publishes resource values — but only the ones + * something in the project points at have to hold a credential for it to work, and those + * are the ones the step lists. Counting all of them here would offer a fourth step that + * then has nothing on it. */ get resourceCount(): number { - return this.#export?.resources?.length ?? 0 + const e = this.#export + if (!e) return 0 + return (e.resources ?? []).filter((r) => projectReferencesResource(e, String(r.path))).length } get extraCounts(): { triggers: number; migrations: number } | undefined { @@ -453,9 +458,7 @@ export class ImportExecution { const problems: string[] = [] if (failed > 0) problems.push(`${failed} item${failed === 1 ? '' : 's'} failed to import`) if (badMigrations > 0) { - problems.push( - `${badMigrations} data table migration${badMigrations === 1 ? '' : 's'} failed` - ) + problems.push(`${badMigrations} data table migration${badMigrations === 1 ? '' : 's'} failed`) } if (problems.length) this.error = `${problems.join(', ')}.` } diff --git a/frontend/src/lib/importWizard/retargetDeployed.test.ts b/frontend/src/lib/importWizard/retargetDeployed.test.ts new file mode 100644 index 0000000000..adf7a4c795 --- /dev/null +++ b/frontend/src/lib/importWizard/retargetDeployed.test.ts @@ -0,0 +1,393 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * The safety property: the stub survives anything the scan cannot account for. Rewriting an + * item onto the chosen resource is safe on its own, so it always happens; deleting the stub + * while an item still reads it is what breaks the imported project, and that item is exactly + * the one nobody looks at afterwards. + */ + +const state = vi.hoisted(() => ({ + apps: [] as any[], + scripts: [] as any[], + triggers: [] as any[], + scheduleListError: undefined as any, + failingTriggerPath: undefined as string | undefined, + deployedJs: 'COMPILED', + deletedResources: [] as string[], + updatedRawApps: [] as any[], + updatedApps: [] as any[], + updatedFlows: [] as any[], + flows: [] as any[], + updatedTriggers: [] as any[] +})) + +vi.mock('$lib/gen', () => ({ + ScriptService: { + listSearchScript: vi.fn(async () => state.scripts), + getScriptByPath: vi.fn(), + createScript: vi.fn() + }, + FlowService: { + listSearchFlow: vi.fn(async () => state.flows), + getFlowByPath: vi.fn(async ({ path }: any) => state.flows.find((f: any) => f.path === path)), + updateFlow: vi.fn(async (p: any) => state.updatedFlows.push(p)) + }, + AppService: { + listSearchApp: vi.fn(async () => state.apps), + getAppByPath: vi.fn(async ({ path }: any) => state.apps.find((a) => a.path === path)), + getPublicSecretOfLatestVersionOfApp: vi.fn(async () => 'secret'), + updateApp: vi.fn(async (p: any) => state.updatedApps.push(p)), + updateAppRaw: vi.fn(async (p: any) => state.updatedRawApps.push(p)) + }, + ScheduleService: { updateSchedule: vi.fn() }, + ResourceService: { + deleteResource: vi.fn(async ({ path }: any) => state.deletedResources.push(path)) + } +})) + +vi.mock('$lib/components/triggers/workspaceTriggersList', () => ({ + TRIGGER_KINDS: { + schedule: { + badge: 'Schedule', + list: vi.fn(async () => { + if (state.scheduleListError) throw state.scheduleListError + return [] + }) + }, + postgres: { + badge: 'Postgres', + list: vi.fn(async () => state.triggers), + update: vi.fn(async (_w: string, path: string, body: any) => { + if (path === state.failingTriggerPath) throw new Error('the update was rejected') + state.updatedTriggers.push({ path, body }) + }) + } + }, + WORKSPACE_TRIGGER_KINDS: ['schedule', 'postgres'], + createWorkspaceTriggerDisabled: vi.fn(), + triggerHandlerRefs: () => [] +})) + +vi.mock('$lib/components/apps/editor/appPolicy', () => ({ updatePolicy: vi.fn(async () => ({})) })) +vi.mock('$lib/sharedUtils', () => ({ updateRawAppPolicy: vi.fn(async () => ({})) })) + +// The deployed bundle is served by secret, not through the generated client. +vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => ({ + ok: true, + status: 200, + text: async () => (url.endsWith('.js') ? state.deployedJs : 'STYLES') + })) +) + +import { applyRetarget, seesWholeWorkspace } from './retargetDeployed' + +const FROM = 'f/proj/smtp' +const TO = 'f/shared/company_smtp' + +/** A deployed raw app: sources plus runnables, with the bundle stored out of the value. */ +const rawApp = { + path: 'f/proj/dash', + raw_app: true, + value: { files: { '/App.tsx': 'v1' }, runnables: { send: { fields: { smtp: `$res:${FROM}` } } } } +} + +async function run() { + return applyRetarget({ + workspace: 'w', + folder: 'proj', + from: FROM, + to: TO, + seesWholeWorkspace: true + }) +} + +describe('applyRetarget', () => { + beforeEach(() => { + state.apps = [rawApp] + state.scripts = [] + state.triggers = [] + state.deployedJs = 'COMPILED' + state.scheduleListError = undefined + state.failingTriggerPath = undefined + state.deletedResources = [] + state.updatedRawApps = [] + state.updatedApps = [] + state.flows = [] + state.updatedFlows = [] + state.updatedTriggers = [] + }) + + // `updateAppRaw` refuses without a bundle, and the browser cannot rebuild one. The bundle + // that goes back is the deployed one, read back by secret. + it('sends the deployed bundle back with the rewritten value, then drops the stub', async () => { + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.gaps).toEqual([]) + expect(outcome.stubDeleted).toBe(true) + expect(state.deletedResources).toEqual([FROM]) + const sent = state.updatedRawApps[0] + expect(sent.formData.js).toBe('COMPILED') + expect(sent.formData.css).toBe('STYLES') + // Rewritten, and the bundle entries stay out of the value the way the import leaves them. + expect(JSON.stringify(sent.formData.app.value)).toContain(`$res:${TO}`) + expect(Object.keys(sent.formData.app.value.files)).toEqual(['/App.tsx']) + }) + + // The bundle is compiled from the sources, so a `$res:` a source spells out is baked into + // it. `retargetProjectExport` rewrites that copy on import, while /bundle.js is still one + // of `files` — sending the deployed bundle back untouched would undo exactly that. + it("rewrites the deployed bundle's own tokens before sending it back", async () => { + state.deployedJs = `const cfg = "$res:${FROM}"; export default cfg` + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(true) + const sent = state.updatedRawApps[0] + expect(sent.formData.js).toContain(`$res:${TO}`) + expect(sent.formData.js).not.toContain(`$res:${FROM}`) + }) + + // A path the bundle names any other way is one nothing here can move, so the app is left + // alone and the stub it still reads has to survive. + it('keeps the stub when the bundle names the resource outside a $res: token', async () => { + state.deployedJs = `const cfg = await getResource("${FROM}")` + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(state.updatedRawApps).toEqual([]) + }) + + // The path written inside a source file is a reference too, and no rewriter reaches it. + // + it('keeps the stub when a source file names the resource in code', async () => { + state.apps = [ + { + path: 'f/proj/dash', + raw_app: true, + value: { + files: { '/App.tsx': `const c = await getResource("${FROM}")` }, + runnables: {} + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(state.updatedRawApps).toEqual([]) + }) + + // A trigger holds its resource as a bare path in its own column, not as a `$res:` token. + it('finds a trigger that holds the resource as a bare path', async () => { + state.apps = [] + state.triggers = [ + { + path: 'f/proj/ingest', + script_path: 'f/proj/run', + postgres_resource_path: FROM, + permissioned_as: 'u/service_account', + enabled: true + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + // The trigger keeps its own path even though it was the string being remapped. + expect(state.updatedTriggers[0].path).toBe('f/proj/ingest') + // Pointing a trigger at a credential must not also start it: `enabled` is left out so + // the backend keeps whatever the trigger is set to. + expect(state.updatedTriggers[0].body).not.toHaveProperty('enabled') + // Nor run it as whoever picked the credential: a trigger states its identity as + // `permissioned_as`, which the backend keeps only when told to preserve it. + expect(state.updatedTriggers[0].body.permissioned_as).toBe('u/service_account') + expect(state.updatedTriggers[0].body.preserve_permissioned_as).toBe(true) + expect(state.deletedResources).toEqual([FROM]) + }) + + // The stub is what a reference this run did not move still resolves through, so a write + // that fails partway must not take it: the moved items and the rest both keep working. + it('keeps the stub when a write fails, and reports what had already moved', async () => { + state.apps = [] + state.triggers = [ + { path: 'f/proj/first', postgres_resource_path: FROM }, + { path: 'f/proj/second', postgres_resource_path: FROM } + ] + state.failingTriggerPath = 'f/proj/second' + const outcome = await run() + expect(outcome.error).toContain('the update was rejected') + expect(outcome.rewritten.map((r) => r.path)).toEqual(['f/proj/first']) + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + }) + + // Most trigger kinds are cargo features an instance may not compile in, and their routes + // then 404. Reading that as a failed listing keeps the stub on every stock build. + it('drops the stub when a trigger kind is not compiled in, keeps it when one truly fails', async () => { + state.scheduleListError = { status: 404 } + expect((await run()).gaps).toEqual([]) + expect(state.deletedResources).toEqual([FROM]) + + state.deletedResources = [] + state.scheduleListError = { status: 500 } + expect((await run()).gaps.map((g) => g.path)).toContain('Schedule triggers') + expect(state.deletedResources).toEqual([]) + }) + + // The referrer scan matches a bare path anywhere in an app's value, while the rewriter only + // relocates `$res:` tokens and runnable paths. Rewriting such an app would claim a move that + // did not happen, and the stub it still reads would go. + it('leaves an app holding the resource path as a bare string alone, and keeps the stub', async () => { + state.apps = [ + { + path: 'f/proj/page', + value: { grid: [{ data: { input: { type: 'static', value: FROM } } }] } + } + ] + const outcome = await run() + expect(outcome.rewritten).toEqual([]) + expect(outcome.gaps.map((g) => g.path)).toEqual(['f/proj/page']) + expect(state.deletedResources).toEqual([]) + }) + + // Both spellings at once. The token is moved so the item stops depending on the stub for + // what it could, and the mention it spells out still keeps the stub alive. + it('rewrites the token of an app that also names the resource in code, and keeps the stub', async () => { + state.apps = [ + { + path: 'f/proj/dash', + raw_app: true, + value: { + files: { '/App.tsx': `// the credential lives at ${FROM}` }, + runnables: { send: { fields: { smtp: `$res:${FROM}` } } } + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.rewritten.map((r) => r.path)).toEqual(['f/proj/dash']) + expect(outcome.gaps.map((g) => g.path)).toEqual(['f/proj/dash']) + expect(outcome.stubDeleted).toBe(false) + expect(JSON.stringify(state.updatedRawApps[0].formData.app.value)).toContain(`$res:${TO}`) + }) + + // Scripts, flows and resources share a path namespace, and the map holds a resource path. + // The import's rewriters remap a runnable's own `path` on an exact match, which here would + // repoint the step at the credential. + it("moves an app's tokens without repointing a runnable that shares the path", async () => { + state.apps = [ + { + path: 'f/proj/page', + value: { + grid: [{ data: { type: 'runnableByPath', runType: 'script', path: FROM } }], + inline: `$res:${FROM}` + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + const sent = state.updatedApps[0] + expect(sent.requestBody.value.inline).toBe(`$res:${TO}`) + expect(sent.requestBody.value.grid[0].data.path).toBe(FROM) + }) + + // A token moves wherever it sits in the value, and a step's own path is left alone. + it('moves a flow token outside the fields the import rewriter reached', async () => { + state.flows = [ + { + path: 'f/proj/pipeline', + value: { + modules: [ + { id: 'a', summary: `reads $res:${FROM}`, value: { type: 'script', path: FROM } } + ] + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + const sent = state.updatedFlows[0] + expect(sent.requestBody.value.modules[0].summary).toBe(`reads $res:${TO}`) + expect(sent.requestBody.value.modules[0].value.path).toBe(FROM) + }) + + // The listings run as the caller and row-level security filters them inside the query, so + // an item a member cannot read is invisible rather than counted. The stub has to outlive + // a scan that cannot see the whole workspace. + it('keeps the stub when the caller is not shown the whole workspace', async () => { + const outcome = await applyRetarget({ + workspace: 'w', + folder: 'proj', + from: FROM, + to: TO, + seesWholeWorkspace: false + }) + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(outcome.gaps.map((g) => g.path)).toContain('This workspace') + }) + + // A handler field names a runnable, and the map holds a resource path. A script sharing + // that path is not the reference being moved. + it("moves a trigger's resource field without repointing its error handler", async () => { + state.apps = [] + state.triggers = [ + { + path: 'f/proj/ingest', + script_path: 'f/proj/run', + postgres_resource_path: FROM, + on_failure: `script/${FROM}`, + error_handler_path: FROM, + permissioned_as: 'u/service_account' + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + expect(state.updatedTriggers[0].body.on_failure).toBe(`script/${FROM}`) + // The bare spelling too: a handler path is a path and never a `$res:` token. + expect(state.updatedTriggers[0].body.error_handler_path).toBe(FROM) + }) + + // `listSearchApp` caps at 1000 rows server-side, unordered and unpaginated, so a full page + // may not hold the project's own app — and the stub would go anyway. + it('keeps the stub when the app listing comes back at its server-side cap', async () => { + state.apps = Array.from({ length: 1000 }, (_, i) => ({ path: `f/other/a${i}`, value: {} })) + const outcome = await run() + expect(outcome.gaps.map((g) => g.path)).toContain('Apps') + expect(state.deletedResources).toEqual([]) + }) + + // The listings are workspace-wide. An item outside the project's folder is the user's own, + // so it is not rewritten — and it is exactly why the stub it reads has to stay. + it('rewrites what it owns and keeps the stub for a reference outside the project', async () => { + state.apps = [] + state.scripts = [{ path: 'u/alice/report', content: `$res:${FROM}` }] + state.triggers = [ + { path: 'f/proj/ingest', script_path: 'f/proj/run', postgres_resource_path: FROM } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + expect(outcome.gaps.map((g) => g.path)).toEqual(['u/alice/report']) + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + }) +}) + +describe('seesWholeWorkspace', () => { + // The user record is per-workspace and survives a workspace change, so reading `is_admin` + // without checking which workspace it describes answers for the wrong one — and a wrong + // yes here is what lets an RLS-filtered scan clear the stub for deletion. + it.each([ + ['admin of this workspace', { workspace_id: 'w', is_admin: true }, false, true], + ['admin of another workspace', { workspace_id: 'other', is_admin: true }, false, false], + ['member of this workspace', { workspace_id: 'w', is_admin: false }, false, false], + ['superadmin, record stale', { workspace_id: 'other', is_admin: false }, true, true], + ['no user record', undefined, false, false] + ])('%s', (_label, user, isSuperadmin, expected) => { + expect(seesWholeWorkspace(user as any, isSuperadmin, 'w')).toBe(expected) + }) +}) diff --git a/frontend/src/lib/importWizard/retargetDeployed.ts b/frontend/src/lib/importWizard/retargetDeployed.ts new file mode 100644 index 0000000000..b33a69a667 --- /dev/null +++ b/frontend/src/lib/importWizard/retargetDeployed.ts @@ -0,0 +1,650 @@ +/** + * Point an already-imported project at a resource the workspace already has. + * + * The import writes `$res:f//` into every item that uses the project's + * resource — except a trigger, which holds the bare path in its own `*_resource_path` + * field. This rewrites those references to an existing resource, so the project reads the + * workspace's own credential — the same end state the import would have produced, reached + * after the fact. + * + * Two rules make that safe to run over deployed items: + * + * - Only items inside the project's folder are rewritten. The import wrote nothing outside + * it, so a reference from elsewhere is the user's own and not ours to move. + * - The stub is deleted only when the scan can prove it saw every reference to it. + * Listings come back capped, a trigger kind can fail to list, a reference can sit where no + * rewriter reaches, and the listings themselves are row-level-security filtered so a + * caller who is not a workspace admin is not shown every item — each of those is a gap, + * and any gap keeps the stub. + * + * Rewriting is separable from deleting, and only the delete is destructive. An item moved + * onto the chosen resource resolves whether or not the stub survives; an item the scan never + * saw resolves only while the stub is there. So an incomplete scan downgrades the run to + * "rewritten, stub kept" rather than refusing it — the outcome names the gaps so the caller + * can say the placeholder is still around. + */ + +import { AppService, FlowService, ResourceService, ScheduleService, ScriptService } from '$lib/gen' +import { + rewriteContent, + rewriteTriggerConfig, + referencesResourcePath, + holdsResourceToken, + textHoldsBarePath +} from '$lib/components/workspaceSettings/projectBundle' +import { + TRIGGER_KINDS, + WORKSPACE_TRIGGER_KINDS, + type WorkspaceTriggerKind +} from '$lib/components/triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { apiErrorMessage as errorMessage } from '$lib/utils' + +export type ReferrerKind = 'script' | 'flow' | 'app' | 'trigger' + +export interface Referrer { + kind: ReferrerKind + path: string + /** Present for triggers: which kind's table it lives in. */ + triggerKind?: WorkspaceTriggerKind + /** + * The plan already recorded this item as naming the path somewhere no rewrite reaches, so + * the stub survives whatever the write does. Its tokens are still worth moving, and the + * write's own staleness check has nothing left to add. + */ + gapped?: true + /** + * Present for triggers: the row the scan read, which is also what the write sends back. + * Re-reading it costs another listing of the whole kind per trigger — and for schedules a + * listing plus a detail fetch per row, since `list` resolves each one. + */ + row?: Record +} + +/** Something the scan could not account for. `path` names an item, or a listing standing in + * for every item it failed to return. */ +export interface Gap { + path: string + reason: string +} + +export interface RetargetPlan { + /** Items whose reference this run will move. */ + referrers: Referrer[] + /** + * Why the scan cannot claim it saw every reference to the stub. Empty is the only state + * in which deleting the stub is provably safe. + */ + gaps: Gap[] +} + +export interface RetargetOutcome { + /** Items now reading the chosen resource. */ + rewritten: Referrer[] + /** Why the stub was kept, when it was. */ + gaps: Gap[] + stubDeleted: boolean + /** Set when a write failed. The run stopped there and the stub stays, so what was + * rewritten before it and what was not both resolve. */ + error?: string +} + +/** + * Whether this caller's listings are the whole workspace, and so whether a clean scan proves + * anything. Row-level security filters the listings inside the query for everyone else. + * + * `UserExt` is per-workspace and outlives a workspace change, so the role is only this + * workspace's role when the record says it is. On the setup step's reload path nothing + * re-fetches it, and it still describes the workspace the user came from — reading + * `is_admin` alone there answers for the wrong workspace. An instance superadmin bypasses + * the policies everywhere, which is why it is asked separately. + */ +export function seesWholeWorkspace( + user: { workspace_id?: string; is_admin?: boolean } | undefined, + isSuperadmin: boolean, + workspace: string +): boolean { + if (isSuperadmin) return true + return !!user?.is_admin && user.workspace_id === workspace +} + +/** Everything under the project's folder, which is all this rewrites. */ +function inFolder(path: unknown, folder: string): boolean { + return typeof path === 'string' && path.startsWith(`f/${folder}/`) +} + +/** + * Whether the item names the resource path anywhere a rewriter would not reach it. + * + * A script, flow or app spells a reference the rewriters move as a `$res:` token and nothing + * else. The path appearing any other way is either a reference beyond them — a static string + * value in a component, an argument inline code assembles itself — or not the resource at + * all, such as a step running a script that happens to share the path. Neither is rewritable: + * moving the first is beyond the rewriters, and moving the second would repoint a runnable at + * a credential. Both are reasons the stub has to outlive the run. + * + * The whole serialized item is searched, because inline code is a string inside it and a path + * written in code is no less a reference for being surrounded by other characters. Triggers + * are the exception and hold the bare path by design, so they never come here. + */ +function namesPathUnreachably(value: unknown, path: string): boolean { + return textHoldsBarePath(JSON.stringify(value ?? null), path) +} + +/** + * What each `listSearch*` endpoint caps its answer at, server-side. The queries carry no + * `ORDER BY` and the routes take no pagination, so a full page is an arbitrary subset with no + * page two to ask for. + * + * A full page is a sound truncation test only for an unscoped caller, which a wizard session + * is. The server applies its scope-path predicate to the rows the `LIMIT` already returned, + * so a scoped token can be handed a short page cut from a truncated query — reuse this scan + * under one and the cap goes undetected. + */ +const SEARCH_LIMITS = { script: 10000, flow: 1000, app: 1000 } + +/** + * What a trigger listing caps at. The kinds' list routes take pagination this table does not + * pass, so each answers with the server's `DEFAULT_PER_PAGE`. A full page is read the same + * way as a full `listSearch*` page: as a listing that cannot account for the rest. + */ +const TRIGGER_LIST_LIMIT = 1000 + +/** + * Where a trigger names a runnable, taken from what `triggerHandlerRefs` reads. + * + * `rewriteTriggerConfig` remaps a runnable reference on an exact path match, which is right + * for the folder-wide map the import hands it and wrong for a map holding one resource path: + * a script sharing that path is not the reference being moved, and remapping it makes the + * trigger run a resource. So these are put back from the row after the rewrite. + * + * Split by spelling, because only one of the two can also hold a `$res:` token. A prefixed + * field is restored only when it holds the runnable spelling, so a token in it still moves; + * a bare field is a path and nothing else, so it is always restored. + */ +const PREFIXED_RUNNABLE_FIELDS = ['on_failure', 'on_recovery', 'on_success', 'url'] +const PREFIXED_RUNNABLE_RE = /^(?:\$(?:script|flow):|(?:script|flow)\/)/ +const BARE_RUNNABLE_FIELDS = ['dynamic_skip', 'error_handler_path', 'script_path'] + +/** The rewritten config with every runnable reference put back as the row holds it. */ +function restoreRunnableRefs(rewritten: any, row: any): any { + const out = { ...rewritten } + for (const k of PREFIXED_RUNNABLE_FIELDS) { + if (typeof row?.[k] === 'string' && PREFIXED_RUNNABLE_RE.test(row[k])) out[k] = row[k] + } + for (const k of BARE_RUNNABLE_FIELDS) { + if (typeof row?.[k] === 'string') out[k] = row[k] + } + // A websocket's initial messages can each carry a runnable result, whose `path` is bare. + // Only that field is put back; the rest of the message is rewritten like any other value. + if (Array.isArray(out.initial_messages)) { + out.initial_messages = out.initial_messages.map((m: any, i: number) => { + const was = row?.initial_messages?.[i]?.runnable_result?.path + return m?.runnable_result && typeof was === 'string' + ? { ...m, runnable_result: { ...m.runnable_result, path: was } } + : m + }) + } + return out +} + +/** A reference this run will not move, recorded so the stub outlives it. */ +const OUTSIDE_PROJECT = 'reads this resource from outside the project' +const UNSEEN_BY_CALLER = 'holds items this account is not shown' +const UNREACHABLE_REFERENCE = 'names the resource path outside a $res: reference' + +/** + * Which deployed items reference the stub, which of them this run can move, and what it + * could not account for. + * + * The `listSearch*` endpoints return each item's content in one call per kind, so this is + * three calls plus one per trigger kind rather than one per item. + */ +export async function planRetarget( + workspace: string, + folder: string, + from: string, + opts: { seesWholeWorkspace: boolean } +): Promise { + const referrers: Referrer[] = [] + const gaps: Gap[] = [] + + // The listings run as the caller. Row-level security filters the rows out inside the + // query, so an item this account cannot read is not merely absent from the answer — it + // does not count towards the full-page test either, and nothing downstream can notice it. + // A colleague's private script reading this stub is exactly that shape. + if (!opts.seesWholeWorkspace) gaps.push({ path: 'This workspace', reason: UNSEEN_BY_CALLER }) + + const [scripts, flows, apps] = await Promise.all([ + ScriptService.listSearchScript({ workspace }), + FlowService.listSearchFlow({ workspace }), + AppService.listSearchApp({ workspace }) + ]) + + for (const [label, rows, limit] of [ + ['Scripts', scripts, SEARCH_LIMITS.script], + ['Flows', flows, SEARCH_LIMITS.flow], + ['Apps', apps, SEARCH_LIMITS.app] + ] as const) { + if ((rows?.length ?? 0) >= limit) gaps.push({ path: label, reason: 'could not all be listed' }) + } + + // The listings are workspace-wide, so an item outside the folder is seen for free. It is + // the user's own and stays on the stub, which is the whole reason the stub stays too. + for (const s of scripts ?? []) { + const content = String(s.content ?? '') + const reads = referencesResourcePath(content, from) + // `rewriteContent` moves the `$res:` tokens and nothing else, so a path the code spells + // out is a reference this run leaves behind. + const bare = textHoldsBarePath(content, from) + if (!reads && !bare) continue + if (!inFolder(s.path, folder)) { + gaps.push({ path: s.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: s.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + referrers.push({ kind: 'script', path: s.path!, ...(bare ? { gapped: true as const } : {}) }) + } + for (const f of flows ?? []) { + const value: any = f.value ?? {} + // A token is the only spelling a rewriter moves. `referencesResourcePath` would also + // count a whole string equal to the path, which is the unreachable case `bare` covers. + const reads = holdsResourceToken(value, from) + const bare = namesPathUnreachably(value, from) + if (!reads && !bare) continue + if (!inFolder(f.path, folder)) { + gaps.push({ path: f.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: f.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + const gapped = bare ? ({ gapped: true } as const) : undefined + referrers.push({ kind: 'flow', path: f.path!, ...gapped }) + } + for (const a of apps ?? []) { + const value: any = a.value ?? {} + // A token is the only spelling a rewriter moves. `referencesResourcePath` would also + // count a whole string equal to the path, which is the unreachable case `bare` covers. + const reads = holdsResourceToken(value, from) + const bare = namesPathUnreachably(value, from) + if (!reads && !bare) continue + if (!inFolder(a.path, folder)) { + gaps.push({ path: a.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: a.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + const gapped = bare ? ({ gapped: true } as const) : undefined + referrers.push({ kind: 'app', path: a.path!, ...gapped }) + } + + for (const kind of WORKSPACE_TRIGGER_KINDS) { + const def = TRIGGER_KINDS[kind] + // No `eeOnly` skip. That reads a client-side store, which is empty on an EE instance + // whose licence is unset or whose fetch failed — while the rows are still in the + // database and the routes still answer. A kind skipped that way leaves no gap, so the + // stub would go while an EE trigger still points at it. On CE the routes are not + // registered and the 404 below says so, from the server. + let rows: Array> = [] + // A 404 is not an incomplete listing — the instance has that trigger feature compiled + // out, so there is no trigger of the kind to have missed. Anything else means triggers + // of this kind may reference the stub without this ever seeing them. + let incomplete = false + try { + rows = await def.list(workspace, () => (incomplete = true)) + } catch (e: any) { + if (e?.status === 404) continue + incomplete = true + } + if (incomplete) { + gaps.push({ path: `${def.badge} triggers`, reason: 'could not be listed' }) + continue + } + if (rows.length >= TRIGGER_LIST_LIMIT) { + gaps.push({ path: `${def.badge} triggers`, reason: 'could not all be listed' }) + continue + } + for (const t of rows) { + if (!referencesResourcePath(t, from)) continue + if (!inFolder(t.path, folder)) { + gaps.push({ path: String(t.path), reason: OUTSIDE_PROJECT }) + continue + } + // `schedule` has no `update` in the table because its service takes a different body + // shape; `rewriteTrigger` handles it directly, the way the import's create does. + if (kind !== 'schedule' && !def.update) { + gaps.push({ + path: String(t.path), + reason: `${def.badge} triggers cannot be updated from here` + }) + continue + } + referrers.push({ kind: 'trigger', path: String(t.path), triggerKind: kind, row: t }) + } + } + + return { referrers, gaps } +} + +/** + * Rewrite every referrer the plan found, then delete the stub if the plan came back clean. + * + * A gap never stops the rewriting — moving an item onto the chosen resource is safe on its + * own. It stops only the delete, which is the one step that can strand a reference nobody + * looked at. + */ +export async function applyRetarget(args: { + workspace: string + folder: string + from: string + to: string + /** Whether the listings the scan reads are the whole workspace. See `planRetarget`. */ + seesWholeWorkspace: boolean +}): Promise { + const { workspace, folder, from, to, seesWholeWorkspace } = args + const map = new Map([[from, to]]) + const rewritten: Referrer[] = [] + + const plan = await planRetarget(workspace, folder, from, { seesWholeWorkspace }) + const gaps = [...plan.gaps] + + for (const r of plan.referrers) { + let moved: true | string + try { + if (r.kind === 'script') moved = await rewriteScript(workspace, r, map) + else if (r.kind === 'flow') moved = await rewriteFlow(workspace, r, map) + else if (r.kind === 'app') moved = await rewriteApp(workspace, r, map) + else moved = await rewriteTrigger(workspace, r, map) + } catch (e: any) { + return { rewritten, gaps, stubDeleted: false, error: errorMessage(e) } + } + if (moved === true) rewritten.push(r) + else gaps.push({ path: r.path, reason: moved }) + } + + if (gaps.length > 0) return { rewritten, gaps, stubDeleted: false } + + try { + await ResourceService.deleteResource({ workspace, path: from }) + } catch (e: any) { + return { rewritten, gaps, stubDeleted: false, error: errorMessage(e) } + } + return { rewritten, gaps, stubDeleted: true } +} + +/** + * The rewrite this module needs: `$res:` tokens and nothing else. + * + * `rewriteFlowValue` and `rewriteAppValue` also remap a runnable's own `path` on an exact + * match. That is right for the folder-wide map the import hands them, where every path is + * moving. Here the map holds one entry, a resource path — and scripts, flows and resources + * share a namespace, so a project shipping both a script and a resource named `smtp` would + * have the step calling `f//smtp` repointed at the credential. Rewriting the + * serialized value moves the tokens and leaves every path alone. + */ +function rewriteTokens(value: T, map: Map): T { + return JSON.parse(rewriteContent(JSON.stringify(value ?? null), map)) +} + +/** + * Whether the item as just read names a path where no rewrite reaches it. + * + * The plan classifies items from the `listSearch*` rows; every rewriter then re-reads its + * item by path. A value that has gained an unreachable mention in between is one the plan + * cleared and the write must not: rewriting its tokens is fine, but the stub it still names + * has to survive. Checking the read the write is about to send is the only place that can be + * seen. + */ +function readsPathUnreachably(value: unknown, map: Map): boolean { + for (const from of map.keys()) if (namesPathUnreachably(value, from)) return true + return false +} + +/** + * Every write here is an in-place edit of a deployed item, not a redeployment by whoever + * opened the wizard, so each one has to say so. + * + * `preserve_on_behalf_of` is the flag that says it. Without it the backend replaces the + * item's stored run identity with the caller's — `resolve_on_behalf_of` for scripts and + * flows, the `should_preserve` branch of `update_app` for apps — and an imported item that + * ran as a service account would silently start running as the person who picked a + * credential. The backend still gates it on `wm_deployers` membership, so a caller who + * cannot preserve gets what they would have got anyway. + * + * For apps the policy is the other half: it carries the run identity, the execution mode and + * the sandbox rules, so it is read from the deployed app and handed back to the recompute + * rather than rebuilt from nothing. + */ +const PRESERVE_DEPLOYED_IDENTITY = { preserve_on_behalf_of: true } + +/** + * A new script version, the way the editor saves one. Spread rather than field-by-field: + * `Script` and `NewScript` share their names, and listing them here would silently drop + * whichever field someone adds next. + */ +async function rewriteScript( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const s: any = await ScriptService.getScriptByPath({ workspace, path }) + if (!r.gapped) + for (const stub of map.keys()) + if (textHoldsBarePath(String(s.content ?? ''), stub)) return UNREACHABLE_REFERENCE + const content = rewriteContent(s.content ?? '', map) + if (content === s.content) return true + await ScriptService.createScript({ + workspace, + requestBody: { + ...s, + ...PRESERVE_DEPLOYED_IDENTITY, + content, + parent_hash: s.hash, + deployment_message: undefined + } + }) + return true +} + +async function rewriteFlow( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const f: any = await FlowService.getFlowByPath({ workspace, path }) + if (!r.gapped && readsPathUnreachably(f.value ?? {}, map)) return UNREACHABLE_REFERENCE + const value = rewriteTokens(f.value ?? {}, map) + // The tokens the plan saw are gone from the deployed item, so there is nothing to write. + if (JSON.stringify(value) === JSON.stringify(f.value ?? {})) return true + await FlowService.updateFlow({ + workspace, + path, + requestBody: { ...f, ...PRESERVE_DEPLOYED_IDENTITY, path, value } + }) + return true +} + +/** + * The deployed policy is recomputed, not rebuilt. Recomputing is required: `triggerables_v2` + * is keyed by `:rawscript/`, and rewriting an inline + * runnable's content changes that key, so a policy copied verbatim would leave the component + * "forbidden by policy". Handing the deployed policy to the recompute is what keeps the run + * identity, the sandbox rules and everything else it does not touch. + * + * No execution mode is defaulted. The backend keeps the deployed mode when the submitted + * policy states none, and stating one here would put a `viewer` app on the publisher's + * identity. + */ +async function rewriteApp( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const a: any = await AppService.getAppByPath({ workspace, path }) + // The deployed record says which kind this is. `list_search_apps` returns only the path and + // the value, so the scan could only have guessed from the value's shape — and a guess wrong + // in either direction is a deploy the backend refuses for changing an app's kind. + if (a.raw_app) return rewriteRawApp(workspace, r, map, a) + if (!r.gapped && readsPathUnreachably(a.value ?? {}, map)) return UNREACHABLE_REFERENCE + const next = rewriteTokens(a.value ?? {}, map) + // The tokens the plan saw are gone from the deployed item, so there is nothing to write. + if (JSON.stringify(next) === JSON.stringify(a.value ?? {})) return true + const policy = (await updatePolicy(next as App, a.policy)) as any + await AppService.updateApp({ + workspace, + path, + requestBody: { ...PRESERVE_DEPLOYED_IDENTITY, path, value: next, policy } + }) + return true +} + +/** + * One half of a deployed raw app's compiled bundle, read back the way the Hub publish reads + * it: `/apps/get_data/v/{secret}.{ext}` serves it to anyone holding the secret, and the + * secret is minted for the caller against a plain `apps:read:` check. + * + * A missing `.css` is an app that ships no styles. A missing `.js` is a broken deployment, + * and uploading an empty one in its place would break it further. + */ +async function fetchBundlePart( + workspace: string, + secret: string, + ext: 'js' | 'css' +): Promise { + const res = await fetch( + `/api/w/${encodeURIComponent(workspace)}/apps/get_data/v/${secret}.${ext}`, + { credentials: 'include' } + ) + if (res.ok) return await res.text() + if (ext === 'css' && res.status === 404) return '' + throw new Error(`the compiled bundle could not be read (${res.status})`) +} + +/** + * A raw app: its deployed value carries the sources and the runnables, and `updateAppRaw` + * refuses without a bundle. The bundle is the deployed one, read back, rewritten and sent + * again — rebuilding it is not possible from the browser and is not needed, but re-uploading + * it untouched is not an option either. + * + * The bundle is compiled from the sources, so a `$res:` a source file spells out is baked + * into it. The import rewrites that copy — `retargetProjectExport` runs while `/bundle.js` + * is still one of `files`, and only `installProject` splits it out afterwards. Sending the + * deployed bundle back unrewritten would undo on reuse what the import got right, and the + * app would keep reading a resource this run is about to delete. + */ +async function rewriteRawApp( + workspace: string, + r: Referrer, + map: Map, + a: any +): Promise { + const path = r.path + const value: any = a.value ?? {} + // One walk over the whole value: `$res:` tokens live in the runnables and can appear in + // the sources too, and both are plain text inside this JSON. + if (!r.gapped && readsPathUnreachably(value, map)) return UNREACHABLE_REFERENCE + const next = rewriteTokens(value, map) + const runnables = next.runnables ?? {} + const policy = (await updateRawAppPolicy(runnables, a.policy)) as any + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ workspace, path }) + const [deployedJs, deployedCss] = await Promise.all([ + fetchBundlePart(workspace, secret, 'js'), + fetchBundlePart(workspace, secret, 'css') + ]) + const js = rewriteContent(deployedJs, map) + const css = rewriteContent(deployedCss, map) + // `rewriteContent` moves the `$res:` tokens. A path the bundle spells out any other way + // is one nothing here can move, and uploading it would leave the app reading a resource + // about to be deleted. + for (const stub of map.keys()) if (textHoldsBarePath(js, stub)) return UNREACHABLE_REFERENCE + // The tokens the plan saw are gone from both the value and the bundle, so there is nothing + // to write, and uploading would cut a version differing from the last in nothing. + if (js === deployedJs && css === deployedCss && JSON.stringify(next) === JSON.stringify(value)) + return true + const files = { ...(next.files ?? {}) } + delete files['/bundle.js'] + delete files['/bundle.css'] + await AppService.updateAppRaw({ + workspace, + path, + formData: { + app: { + ...PRESERVE_DEPLOYED_IDENTITY, + path, + summary: a.summary ?? '', + value: { + files, + runnables, + ...(next.data !== undefined ? { data: next.data } : {}), + ...(next.datatables !== undefined ? { datatables: next.datatables } : {}) + }, + policy + }, + js, + css + } + }) + return true +} + +/** + * The trigger's own row, rewritten and written back. `enabled` is deliberately not sent: + * imported triggers are created disabled and re-enabling one is the user's decision, not a + * side effect of pointing it at a credential. + * + * `path` is put back from the row afterwards, and so is every runnable reference — see + * `restoreRunnableRefs`. The rewrite remaps any string equal to the stub's path, so a trigger + * sitting at the path the resource used to hold would otherwise be renamed along with the + * reference. + * + * A trigger states its run identity as `permissioned_as`, not the `on_behalf_of` the other + * kinds use, and `resolve_permissioned_as` keeps the row's value only when + * `preserve_permissioned_as` says so. Without the pair a trigger created under a folder's + * `default_permissioned_as` would start running as whoever picked the credential. + */ +async function rewriteTrigger( + workspace: string, + r: Referrer, + map: Map +): Promise { + const def = TRIGGER_KINDS[r.triggerKind!] + const row: any = r.row ?? {} + const { enabled: _enabled, ...rest } = { + ...restoreRunnableRefs(rewriteTriggerConfig(row, map), row), + path: r.path, + ...(typeof row.permissioned_as === 'string' + ? { permissioned_as: row.permissioned_as, preserve_permissioned_as: true } + : {}) + } + // No unreachable-mention check: a trigger holds the bare path by design, in its own + // resource field, and `rewriteTriggerConfig` remaps every bare match at every depth. What + // can still read `from` afterwards is a restored identity field, not a reference. + if (r.triggerKind === 'schedule') { + // `EditSchedule` needs these three; everything else on the row carries over by name. + await ScheduleService.updateSchedule({ + workspace, + path: r.path, + requestBody: { + ...rest, + schedule: rest.schedule ?? '0 0 * * * *', + timezone: rest.timezone ?? 'UTC', + args: rest.args ?? {} + } + }) + return true + } + await def.update!(workspace, r.path, rest) + return true +} diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 3e8ae4cba0..ce4adf82be 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -885,7 +885,7 @@ export const mcpEndpointTools: EndpointTool[] = [ { name: "runScriptByPath", description: "run script by path", - instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected.", + instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`.", path: "/w/{workspace}/jobs/run/p/{path}", method: "POST", pathParamsSchema: { @@ -1426,7 +1426,7 @@ export const mcpEndpointTools: EndpointTool[] = [ { name: "runFlowByPath", description: "run flow by path", - instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected.", + instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`.", path: "/w/{workspace}/jobs/run/f/{path}", method: "POST", pathParamsSchema: { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 257af0fbbd..a20c61f2ff 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -83,6 +83,7 @@ import SessionPicker from '$lib/components/sessions/SessionPicker.svelte' import SessionModeSwitch from '$lib/components/sessions/SessionModeSwitch.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + import { copilotInfo } from '$lib/aiStore' import { parsePreviewItemRoute } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' @@ -250,6 +251,10 @@ // so it follows the gate; opted-out users get the legacy Ask-AI pane instead. // The /sessions page has its own gate for direct navigation. const globalAiEnabled = isGlobalAiEnabled() + // A workspace that hid the assistant (`ai_config.copilot_disabled`) loses both entry + // points: the Workspace ⇄ Sessions switch and the legacy Ask-AI button. + const sessionsSwitchShown = $derived(globalAiEnabled && !$copilotInfo.workspaceDisabled) + const askAiShown = $derived(!globalAiEnabled && !$copilotInfo.workspaceDisabled) if (page.status == 404) { goto('/user/login') @@ -999,7 +1004,7 @@
- {#if !embedded && globalAiEnabled} + {#if !embedded && sessionsSwitchShown}
-
+
{/if} @@ -1047,7 +1052,7 @@ class="!text-xs" shortcut={`${getModifierKey()}k`} /> - {#if !globalAiEnabled} + {#if askAiShown}
- {#if !embedded && globalAiEnabled} + {#if !embedded && sessionsSwitchShown}
@@ -1145,7 +1150,7 @@ -
+
{/if} @@ -1184,7 +1189,7 @@ class="!text-xs" shortcut={`${getModifierKey()}k`} /> - {#if !globalAiEnabled} + {#if askAiShown} - aiChatManager.toggleOpen()} - {isCollapsed} - icon={WandSparkles} - iconProps={{ - forceDarkMode: true - }} - label="Ask AI" - class="!text-xs" - iconClasses="!text-ai" - shortcut={`${getModifierKey()}L`} - /> + {#if !$copilotInfo.workspaceDisabled} + aiChatManager.toggleOpen()} + {isCollapsed} + icon={WandSparkles} + iconProps={{ + forceDarkMode: true + }} + label="Ask AI" + class="!text-xs" + iconClasses="!text-ai" + shortcut={`${getModifierKey()}L`} + /> + {/if}
+ + + status and on the workspace inside the component, which owns its own vertical spacing + because the hero and the bare connect row want different amounts of it. --> {#if isGlobalAiEnabled()} -
- -
+ {/if} {#if $workspaceStore == 'admins'} @@ -280,8 +281,6 @@
{/if} - - (showCreateButtons = v)} /> {#if tab == 'hub'} diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index e35c4a7381..b184e4aa4a 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -4,11 +4,10 @@ import CenteredPage from '$lib/components/CenteredPage.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import FolderEditor from '$lib/components/FolderEditor.svelte' + import FolderEditorDrawer from '$lib/components/FolderEditorDrawer.svelte' import PageHeader from '$lib/components/PageHeader.svelte' import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Button, Drawer, DrawerContent, EmptyState, Skeleton } from '$lib/components/common' - import Popover from '$lib/components/meltComponents/Popover.svelte' import FolderInfo from '$lib/components/FolderInfo.svelte' import FolderUsageInfo from '$lib/components/FolderUsageInfo.svelte' import { sendUserToast } from '$lib/utils' @@ -28,9 +27,8 @@ isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) ) - let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) - let folderDrawer: Drawer | undefined = $state() + let folderEditorDrawer: FolderEditorDrawer | undefined = $state() let hubDrawer: Drawer | undefined = $state() let publishFolderName: string = $state('') @@ -47,23 +45,11 @@ }) } - function handleKeyUp(event: KeyboardEvent, close: () => void) { - const key = event.key - if (key === 'Enter') { - event.preventDefault() - addFolder() - close() - } - } - async function addFolder() { - await FolderService.createFolder({ - workspace: $workspaceStore ?? '', - requestBody: { name: newFolderName } - }) - $userStore?.folders.push(newFolderName) - loadFolders() - editFolderName = newFolderName - folderDrawer?.openDrawer() + function onFolderSaved(name: string, created: boolean) { + if (created) $userStore?.folders.push(name) + // Returned, not fired: the drawer reports a failed reload, and it can only see one + // through the promise this hands back. + return loadFolders() } $effect(() => { @@ -74,8 +60,6 @@ } }) - let editFolderName: string = $state('') - function computeMembers(owners: string[], extra_perms: Record) { const members = new Set(owners) for (const [user, _] of Object.entries(extra_perms)) { @@ -85,48 +69,18 @@ } -{#snippet newFolderPopover( - label: string, - placement: 'bottom' | 'bottom-end', - variant: 'accent' | 'default' -)} - folderEditorDrawer?.initNew()} > - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} - handleKeyUp(e, () => close())} - placeholder="New folder name" - bind:value={newFolderName} - /> - -
- -
- {/snippet} - + {label} + {/snippet} - - - - - + {:else} - {@render newFolderPopover('New folder', 'bottom-end', 'accent')} + {@render newFolderButton('New folder', 'accent')} {/if}
@@ -181,7 +135,7 @@ description="Folders are how you grant permissions: make a user or group viewer, writer or admin on a folder and that access applies to every script, flow, app, resource and schedule inside it." > {#if !restricted} - {@render newFolderPopover('Add a folder', 'bottom', 'default')} + {@render newFolderButton('Add a folder', 'default')} {/if} {:else} @@ -196,8 +150,8 @@ Schedules Variables Resources - Participants - + Members + Actions @@ -211,13 +165,7 @@ {/each} {:else} {#each folders as { name, extra_perms, owners, canWrite, summary, labels } (name)} - { - editFolderName = name - folderDrawer?.openDrawer() - }} - > + folderEditorDrawer?.initEdit(name)}> {name} {#if summary} @@ -250,17 +198,14 @@ - + { - editFolderName = name - folderDrawer?.openDrawer() - } + action: () => folderEditorDrawer?.initEdit(name) }, { displayName: 'Publish to Hub', diff --git a/frontend/src/routes/(root)/(logged)/groups/+page.svelte b/frontend/src/routes/(root)/(logged)/groups/+page.svelte index 85b4b536fe..ce8c6a8db0 100644 --- a/frontend/src/routes/(root)/(logged)/groups/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/groups/+page.svelte @@ -5,9 +5,8 @@ import CenteredPage from '$lib/components/CenteredPage.svelte' import { Button, Drawer, DrawerContent, Skeleton } from '$lib/components/common' - import Popover from '$lib/components/meltComponents/Popover.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import GroupEditor from '$lib/components/GroupEditor.svelte' + import GroupEditorDrawer from '$lib/components/GroupEditorDrawer.svelte' import InstanceGroupEditor from '$lib/components/InstanceGroupEditor.svelte' import GroupInfo from '$lib/components/GroupInfo.svelte' import PageHeader from '$lib/components/PageHeader.svelte' @@ -20,7 +19,6 @@ import Cell from '$lib/components/table/Cell.svelte' import Row from '$lib/components/table/Row.svelte' import { untrack } from 'svelte' - import TextInput from '$lib/components/text_input/TextInput.svelte' import { Tooltip } from '$lib/components/meltComponents' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' @@ -30,10 +28,9 @@ isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) ) - let newGroupName: string = $state('') let groups: GroupW[] | undefined = $state(undefined) let instanceGroups: InstanceGroupWithWorkspaces[] | undefined = $state(undefined) - let groupDrawer: Drawer | undefined = $state() + let groupEditorDrawer: GroupEditorDrawer | undefined = $state() async function loadGroups(): Promise { groups = (await GroupService.listGroups({ workspace: $workspaceStore! })).map((x) => { @@ -49,24 +46,6 @@ } } - function handleKeyUp(event: KeyboardEvent, close: () => void) { - const key = event.key - if (key === 'Enter') { - event.preventDefault() - addGroup() - close() - } - } - async function addGroup() { - await GroupService.createGroup({ - workspace: $workspaceStore ?? '', - requestBody: { name: newGroupName } - }) - loadGroups() - editGroupName = newGroupName - groupDrawer?.openDrawer() - } - $effect(() => { untrack(() => loadInstanceGroups()) if ($workspaceStore && $userStore) { @@ -74,16 +53,11 @@ } }) - let editGroupName: string = $state('') let instanceGroupDrawer: Drawer | undefined = $state() let editInstanceGroupName: string = $state('') - - - - - + {:else} - - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} -
- handleKeyUp(e, close) - }} - bind:value={newGroupName} - /> - -
- {/snippet} - + {/if}
@@ -161,7 +112,7 @@ Name Members - + Actions @@ -175,13 +126,7 @@ {/each} {:else} {#each groups as { name, summary, extra_perms, canWrite } (name)} - { - editGroupName = name - groupDrawer?.openDrawer() - }} - > + groupEditorDrawer?.initEdit(name)}>
@@ -197,7 +142,7 @@ - + { e?.stopPropagation() - editGroupName = name - groupDrawer?.openDrawer() + groupEditorDrawer?.initEdit(name) } }, { diff --git a/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte index a1bc2ab970..a6f4632526 100644 --- a/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte @@ -261,9 +261,10 @@ setupUndecided = false return } - // Every resource the project ships arrives as an empty stub, so any project with - // resources has something to fill in. The step itself re-checks and shows only - // what is genuinely outstanding, which is what makes a re-import quiet. + // `resourceCount` is the referenced subset — the resources something in the project + // points at — and each one arrives as an empty stub, so any project that has them has + // something to fill in. The step itself re-checks and shows only what is genuinely + // outstanding, which is what makes a re-import quiet. if (execution.resourceCount > 0) { setupNeeded = true setupUndecided = false diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 6f3fd1a6dc..4932355654 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -1086,8 +1086,8 @@ Path Resource type Description - - + Status + Actions @@ -1254,8 +1254,8 @@ {/if}
- -
+ +
{#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} Name Description - + Actions @@ -1418,7 +1418,7 @@
- + {#if !canWrite} diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index fbf045fe19..da8fc31c65 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -21,6 +21,8 @@ type Scope } from '$lib/components/sessions/PreviewRouterPicker.svelte' import { goto } from '$lib/navigation' + import { resource } from 'runed' + import { WorkspaceService } from '$lib/gen' import SessionWrapper from '$lib/components/sessions/SessionWrapper.svelte' import PreviewTabHost from '$lib/components/sessions/PreviewTabHost.svelte' import { useIsDarkMode } from '$lib/components/DarkModeObserver.svelte' @@ -278,6 +280,36 @@ (activeSession ? getEffectiveWorkspaceId(activeSession) : undefined) ?? $workspaceStore ) + // Whether that workspace hid the AI assistant (`ai_config.copilot_disabled`). Read + // directly rather than from `copilotInfo`: that store holds whichever workspace loaded + // last, and the gate below unmounts the session wrapper that would refresh it, so a + // hidden verdict would stick across session and workspace switches. Tagged with its + // workspace and guarded against a superseded response like the protection-rules + // resource: runed keeps the previous `current` while a new source loads, so a switch + // would otherwise be judged on the previous workspace's verdict. A failed read leaves + // the page usable, as an unloaded config does everywhere else. + const workspaceAiHidden = resource( + () => previewWorkspace, + async (workspace, _prev, { signal }) => { + if (!workspace) return { workspace, hidden: false } + let hidden = false + try { + hidden = (await WorkspaceService.getCopilotInfo({ workspace })).copilot_disabled === true + } catch (e) { + console.error(`Failed to read the AI config of workspace ${workspace}:`, e) + } + // The generated client can't take an abort signal, so drop a superseded response here. + if (signal.aborted) throw new DOMException('superseded', 'AbortError') + return { workspace, hidden } + } + ) + // Only a verdict for the workspace currently judged counts; `undefined` means it has not + // landed yet. + const aiHiddenVerdict = $derived.by(() => { + const current = workspaceAiHidden.current + return current && current.workspace === previewWorkspace ? current.hidden : undefined + }) + // Lazy-mount gate: a tab's content only renders once its key lands here (on // first activation) — so restoring a session with N saved tabs boots just // the active one instead of N full Windmill apps at once. Pure "has this @@ -559,9 +591,7 @@ } }) - // Preview cards on create/update tool calls dispatch here. Open - // (or focus, if already shown) the item's preview in the active session's panel — - // the visible chat is always the active session, so `owner` is its panel. Read + // The visible chat is always the active session, so `owner` is its panel. Read // `owner` lazily inside the handler (not in the effect body) so this registers // once, not on every session switch. $effect(() => { @@ -836,284 +866,309 @@
{:else} -
- +
+
- {#if !fullscreen} - - -
- {#each warmSessions as s (s.id)} -
- -
- {/each} -
- - {/if} + +
+ {#each warmSessions as s (s.id)} +
+ +
+ {/each} +
+ + {/if} - - -
-
- {#if !fullscreen} - - - {/if} + + {/if} - -
- {#if !activeTabIsArtifact} - + {#if !activeTabIsArtifact} + + + + {/if} + -
+ {#if fullscreen} + + {:else} + + {/if} + +
- - (activeTabPickerOpen = !activeTabPickerOpen)} - onClose={closeTab} - onReorder={reorderTabs} - class="session-preview-tab-strip h-8 border-b border-light bg-surface-secondary/50 {fullscreen - ? 'pl-1.5' - : 'pl-9'} pr-16" - > - {#snippet tabAccessory(_tab, isActive)} - {#if isActive} - - - (e.currentTarget as HTMLElement) - .closest('[role="tab"]') - ?.focus() - }} - > - {#snippet content()} - - {#key activePickerScope?.dir ?? ''} - { - activeTabPickerOpen = false - navigatePreviewTo(t) - }} - /> - {/key} - {/snippet} - - - {/if} - {/snippet} - {#snippet afterTabs()} - - {#snippet trigger()} - - {/snippet} - {#snippet content()} - { - newTabOpen = false - openInNewTab(t) - }} + {#key activePickerScope?.dir ?? ''} + { + activeTabPickerOpen = false + navigatePreviewTo(t) + }} + /> + {/key} + {/snippet} + + - {/snippet} - - {/snippet} - - - -
- {#each warmSessions as s (s.id)} - {@const rt = getRuntime(s.id)} - {@const tabs = rt?.previewTabs} - {#each tabs?.tabs ?? [] as tab (tab.id)} - - - tabs && onTabLoad(tabs, tab, frame)} - /> - {/each} - {/each} - {#if (owner?.tabs.length ?? 0) === 0} - -
- -
- No preview open - Open a page, flow, script or app to preview it alongside the chat. -
+ {/if} + {/snippet} + {#snippet afterTabs()} {#snippet trigger()} - - Open a preview - + {/snippet} {#snippet content()} { - emptyStateNewTabOpen = false + newTabOpen = false openInNewTab(t) }} /> {/snippet} -
- {/if} + {/snippet} + + + +
+ {#each warmSessions as s (s.id)} + {@const rt = getRuntime(s.id)} + {@const tabs = rt?.previewTabs} + {#each tabs?.tabs ?? [] as tab (tab.id)} + + + tabs && onTabLoad(tabs, tab, frame)} + /> + {/each} + {/each} + {#if (owner?.tabs.length ?? 0) === 0} + +
+ +
+ No preview open + Open a page, flow, script or app to preview it alongside the chat. +
+ + {#snippet trigger()} + + Open a preview + + {/snippet} + {#snippet content()} + { + emptyStateNewTabOpen = false + openInNewTab(t) + }} + /> + {/snippet} + +
+ {/if} +
-
- - - {#if previewCollapsed && !fullscreen} - -
- +
+ +
+ {/if} +
+ {#if aiHiddenVerdict === undefined} +
+ +
+ {:else if aiHiddenVerdict} + +
+

AI Sessions are hidden in this workspace

+

A workspace admin hid AI sessions in the workspace settings.

+
{/if}
diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 0a23656514..016841f1ae 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -391,8 +391,8 @@ Path Value Description - - + Status + Actions @@ -494,7 +494,7 @@ {#if refresh_error} + so it can't paint over anything that scrolls past it -->
@@ -546,7 +546,7 @@ {/if}
- + { let owner = isOwner(path, $userStore, $workspaceStore) diff --git a/frontend/src/routes/kitchen_sink/+page.svelte b/frontend/src/routes/kitchen_sink/+page.svelte index 8681fb78da..a001da222f 100644 --- a/frontend/src/routes/kitchen_sink/+page.svelte +++ b/frontend/src/routes/kitchen_sink/+page.svelte @@ -9,6 +9,7 @@ import type { DisplayMessage } from '$lib/components/copilot/chat/shared' import DraggableTabs, { type TabItem } from '$lib/components/common/tabs/DraggableTabs.svelte' import { Globe } from 'lucide-svelte' + import { workspaceStore } from '$lib/stores' let tab = $state('button') @@ -195,7 +196,7 @@ That's the full round-trip.` CodeDisplay → HighlightCode), constrained to the chat panel width.
- +
diff --git a/integration_tests/test/git_sync_test.py b/integration_tests/test/git_sync_test.py index 3cc54f2796..47ac02351e 100644 --- a/integration_tests/test/git_sync_test.py +++ b/integration_tests/test/git_sync_test.py @@ -1,9 +1,12 @@ +import json import os +import re import shutil import tempfile import time import unittest import uuid +from pathlib import Path import git as gitpython @@ -19,6 +22,21 @@ def unique_name(prefix: str = "git-sync-test") -> str: return f"{prefix}-{uuid.uuid4().hex[:8]}" +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def ui_pull_script_path() -> str: + """The hub script the git-sync UI runs for a pull (git → workspace).""" + with open(REPO_ROOT / "frontend/src/lib/hubPaths.json") as f: + return json.load(f)["gitInitRepo"] + + +def backend_pull_script_path() -> str: + """The hub script the backend runs for an auto-pull: `GIT_SYNC_PULL_SCRIPT_PATH`.""" + source = (REPO_ROOT / "backend/windmill-common/src/workspaces.rs").read_text() + return re.search(r'GIT_SYNC_PULL_SCRIPT_PATH: &str = "([^"]+)"', source).group(1) + + class GitSyncTestBase(unittest.TestCase): """Shared fixture + helpers for git sync e2e tests (no tests of its own). @@ -221,6 +239,84 @@ class GitSyncTestBase(unittest.TestCase): ) return matching[0] + def _seed_wmill_yaml( + self, repo_name: str, branch: str = "main", include_schedules: bool = False + ): + """Commit a minimal wmill.yaml: the pull CLI requires one in the repo. + Real setups get it from the init/settings-push flow; pushes alone + don't write it.""" + self._gitea.create_file( + repo_name, + "wmill.yaml", + "defaultTs: bun\n" + "includes:\n" + ' - "**"\n' + "excludes: []\n" + "codebases: []\n" + "skipVariables: true\n" + "skipResources: true\n" + "skipResourceTypes: true\n" + "skipSecrets: true\n" + f"includeSchedules: {'true' if include_schedules else 'false'}\n" + "includeTriggers: false\n", + branch=branch, + ) + + def _create_fork(self, client: WindmillClient) -> tuple: + """Fork `client`'s workspace the way the UI does (branch job first, then + the workspace). Returns (fork_id, fork_branch).""" + fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" + self._fork_workspaces_to_cleanup.append(fork_id) + job_ids = client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") + if job_ids: + client.wait_for_jobs_by_ids(job_ids, timeout=90) + time.sleep(3) + client.create_workspace_fork(fork_id, f"Fork {fork_id}") + return fork_id, f"wm-fork/main/{fork_id[len('wm-fork-'):]}" + + def _run_ui_pull( + self, + client: WindmillClient, + resource_path: str, + include_type: list, + clone_ref: str = None, + dry_run: bool = False, + timeout: int = 180, + ) -> dict: + """Run the pull the git-sync UI runs (git → `client`'s workspace) and + return the job's result. `dry_run` is the UI's preview.""" + payload = { + "workspace_id": client._workspace, + "repo_url_resource_path": resource_path, + "dry_run": dry_run, + "pull": True, + "only_wmill_yaml": False, + "settings_json": json.dumps({ + "include_path": ["**"], + "exclude_path": [], + "extra_include_path": [], + "include_type": include_type, + }), + "use_promotion_overrides": False, + **({"clone_ref": clone_ref} if clone_ref else {}), + } + response = client._client.post( + f"/api/w/{client._workspace}/jobs/run/p/{ui_pull_script_path()}", + params={"skip_preprocessor": "true"}, + json=payload, + ) + self.assertEqual( + response.status_code // 100, 2, f"UI pull failed to start: {response.content.decode()}" + ) + job_id = response.content.decode() + client.wait_for_jobs_by_ids([job_id], timeout=timeout) + job = client._client.get(f"/api/w/{client._workspace}/jobs_u/get/{job_id}").json() + self.assertTrue( + job.get("success"), + f"UI pull job {job_id} failed: {job.get('result')}\n{job.get('logs')}", + ) + return job.get("result") or {} + class TestGitSync(GitSyncTestBase): # ────────────────────────────────────────────────── @@ -802,27 +898,6 @@ class TestGitSyncAutoPull(GitSyncTestBase): # Two poll cycles + job execution, with slack for a loaded CI runner. PULL_TIMEOUT = 240 - def _seed_wmill_yaml(self, repo_name: str, branch: str = "main"): - """Commit a minimal wmill.yaml: the pull CLI requires one in the repo. - Real setups get it from the init/settings-push flow; pushes alone - don't write it.""" - self._gitea.create_file( - repo_name, - "wmill.yaml", - "defaultTs: bun\n" - "includes:\n" - ' - "**"\n' - "excludes: []\n" - "codebases: []\n" - "skipVariables: true\n" - "skipResources: true\n" - "skipResourceTypes: true\n" - "skipSecrets: true\n" - "includeSchedules: false\n" - "includeTriggers: false\n", - branch=branch, - ) - def _configure_auto_pull(self, resource_path: str, sync_forks: bool = False): """Single sync repo with auto-pull enabled in polling mode.""" auto_pull = {"enabled": True, "mode": "polling"} @@ -914,16 +989,7 @@ class TestGitSyncAutoPull(GitSyncTestBase): self._configure_auto_pull(resource_path, sync_forks=True) - # Create the fork (branch first, then workspace), like the UI does. - fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" - self._fork_workspaces_to_cleanup.append(fork_id) - job_ids = self._client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") - if job_ids: - self._client.wait_for_jobs_by_ids(job_ids, timeout=90) - time.sleep(3) - self._client.create_workspace_fork(fork_id, f"Fork {fork_id}") - - fork_branch = f"wm-fork/main/{fork_id[len('wm-fork-'):]}" + fork_id, fork_branch = self._create_fork(self._client) self._gitea.create_file( repo_name, script_file, ts_script("return 'fork only'"), branch=fork_branch, @@ -1012,19 +1078,12 @@ class TestGitSyncAutoPull(GitSyncTestBase): f"attach_dev_workspace failed: {attach.content.decode()}", ) - # Fork the dev workspace (branch first, then workspace) — its parent is - # the dev, so this is a fork OF a dev workspace. - fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" - self._fork_workspaces_to_cleanup.append(fork_id) - job_ids = dev_client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") - if job_ids: - dev_client.wait_for_jobs_by_ids(job_ids, timeout=90) - time.sleep(3) - dev_client.create_workspace_fork(fork_id, f"Fork {fork_id}") + # Fork the dev workspace — its parent is the dev, so this is a fork OF + # a dev workspace. + fork_id, fork_branch = self._create_fork(dev_client) # The fork branch is named after the tracked branch, not the dev label. fork_suffix = fork_id[len("wm-fork-"):] - fork_branch = f"wm-fork/main/{fork_suffix}" branches = self._get_branches(self._clone_repo_all_branches(repo_name)) self.assertTrue( any(fork_branch in b for b in branches), @@ -1195,3 +1254,73 @@ class TestGitSyncAutoPull(GitSyncTestBase): initial_count, "Unknown webhook delivery enqueued a job", ) + + +class TestGitSyncUiPull(GitSyncTestBase): + """The pull the git-sync UI runs (hub init script, git → workspace).""" + + def test_ui_pull_script_is_the_backend_pull_script(self): + """The UI's pull and the backend's auto-pull are the same hub script, + so the two pins must move together.""" + self.assertEqual(ui_pull_script_path(), backend_pull_script_path()) + + def test_fork_pull_does_not_report_parent_owned_schedule_enabled(self): + """In a fork, a schedule the parent also has takes its `enabled` from + the parent, so a fork-branch file that disagrees on that flag can never + be made to agree: a pull that treated it as a change would list the + same row on every run. Pull into the fork, then preview: the schedule + must not be reported, while an ordinary fork-branch edit does land.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + include_type = ["script", "schedule"] + self._configure_single_repo_sync(resource_path, include_type=include_type) + + script_path = self._deploy_seed_script("forkuipull") + schedule_path = f"u/admin/{unique_name('forkuipull_sched')}" + initial_count = self._client.count_deployment_callback_jobs() + self._client.create_schedule(schedule_path, script_path, schedule="0 0 0 1 1 *") + self.addCleanup(self._client.delete_schedule, schedule_path) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + self._seed_wmill_yaml(repo_name, include_schedules=True) + + # Fork after the schedule reached git so the fork branch inherits it + # with the parent's `enabled: true`; the fork's own copy lands disabled. + fork_id, fork_branch = self._create_fork(self._client) + + schedule_file = f"{schedule_path}.schedule.yaml" + fork_dir = self._clone_repo(repo_name, branch=fork_branch) + content = self._read_file_content(fork_dir, schedule_file) + self.assertIn( + "enabled: true", content, f"expected the parent's enabled schedule in git:\n{content}" + ) + self._gitea.create_file( + repo_name, + schedule_file, + content.replace("enabled: true", "enabled: false"), + branch=fork_branch, + ) + # A real change alongside it proves the pull ran against the fork branch. + script_file = self._repo_script_file(repo_name, script_path, branch=fork_branch) + self._gitea.create_file( + repo_name, script_file, ts_script("return 'fork ui pull'"), branch=fork_branch + ) + + fork_client = WindmillClient(workspace=fork_id) + self._run_ui_pull(fork_client, resource_path, include_type, clone_ref=fork_branch) + self.assertIn( + "fork ui pull", + fork_client.get_script_content(script_path), + "the fork-branch script edit was not applied by the pull", + ) + preview = self._run_ui_pull( + fork_client, resource_path, include_type, clone_ref=fork_branch, dry_run=True + ) + self.assertIn("changes", preview, f"preview result has no changes list: {preview}") + changes = preview["changes"] + self.assertIsInstance(changes, list, f"preview changes is not a list: {preview}") + self.assertEqual( + [c for c in changes if c.get("path", "").endswith(schedule_file)], + [], + f"a pull into the fork keeps reporting the parent-owned schedule flag: {changes}", + ) diff --git a/lsp/Pipfile b/lsp/Pipfile index 05fe3b7141..0bc31bcb41 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.800.0" +wmill = ">=1.803.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index bed3ee67c5..bdf8ecce08 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.800.0 + version: 1.803.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 272bacc934..089f7ff05a 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.800.0' + ModuleVersion = '1.803.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 705d51d673..35a2b8d857 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.800.0" +version = "1.803.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index e51c8371dc..acf6622dc9 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -160,11 +160,11 @@ flow related commands - `flow push ` - push a local flow spec. This overrides any remote versions. - `--message ` - Deployment message - `flow run ` - run a flow by path. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files. - `--step ` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -560,11 +560,11 @@ script related commands - `--json` - Output as JSON (for piping to jq) - `script show ` - show a script's content (alias for get) - `script run ` - run a script by path - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script preview ` - preview a local script without deploying it. Supports both regular and codebase scripts. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other than the final output. Useful for scripting. - `--tag ` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script new ` - create a new script diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 89f517d3c8..262ecda2be 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -498,6 +498,41 @@ Reference other resources: } \`\`\` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema \`format: resource-\`) is passed as +the **bare string** \`$res:\` — the whole argument value. Same for a variable, with +\`$var:\`. This applies everywhere job arguments are supplied: \`wmill script run/preview\`, +\`wmill flow run/preview\`, the \`runScriptByPath\` / \`runFlowByPath\` API, a schedule's \`args\`, a +trigger's configured static args. + +\`\`\`json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +\`\`\` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with \`$res:\` / \`$var:\`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +\`\`\`json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +\`\`\` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(\`{ "gh_auth": { "token": "$var:g/all/gh_token" } }\`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL @@ -3244,11 +3279,11 @@ flow related commands - \`flow push \` - push a local flow spec. This overrides any remote versions. - \`--message \` - Deployment message - \`flow run \` - run a flow by path. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`--step \` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -3644,11 +3679,11 @@ script related commands - \`--json\` - Output as JSON (for piping to jq) - \`script show \` - show a script's content (alias for get) - \`script run \` - run a script by path - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script preview \` - preview a local script without deploying it. Supports both regular and codebase scripts. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other than the final output. Useful for scripting. - \`--tag \` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script new \` - create a new script diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 1f3c54ea26..fef40d3341 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -165,11 +165,11 @@ flow related commands - `flow push ` - push a local flow spec. This overrides any remote versions. - `--message ` - Deployment message - `flow run ` - run a flow by path. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files. - `--step ` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -565,11 +565,11 @@ script related commands - `--json` - Output as JSON (for piping to jq) - `script show ` - show a script's content (alias for get) - `script run ` - run a script by path - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script preview ` - preview a local script without deploying it. Supports both regular and codebase scripts. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other than the final output. Useful for scripting. - `--tag ` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script new ` - create a new script diff --git a/system_prompts/auto-generated/skills/resources/SKILL.md b/system_prompts/auto-generated/skills/resources/SKILL.md index 19b8c9f420..396b3b9d8e 100644 --- a/system_prompts/auto-generated/skills/resources/SKILL.md +++ b/system_prompts/auto-generated/skills/resources/SKILL.md @@ -64,6 +64,41 @@ Reference other resources: } ``` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema `format: resource-`) is passed as +the **bare string** `$res:` — the whole argument value. Same for a variable, with +`$var:`. This applies everywhere job arguments are supplied: `wmill script run/preview`, +`wmill flow run/preview`, the `runScriptByPath` / `runFlowByPath` API, a schedule's `args`, a +trigger's configured static args. + +```json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +``` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with `$res:` / `$var:`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +```json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +``` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(`{ "gh_auth": { "token": "$var:g/all/gh_token" } }`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 6a5f36e9b4..925ffbe6ec 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -77,6 +77,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute `wmill flow preview -d ''` directly — pick plausible args from the flow's input schema. +An input typed as a resource (`format: resource-` in the schema) takes the bare string `"$res:"` as its whole value — `-d '{"db": "$res:f/databases/postgres_prod"}'`, not `{"db": {"$res": "..."}}` and not a plain path. Same for a variable, with `"$var:"`. See the `resources` skill. + `wmill flow preview` is safe to run yourself (it does not deploy). `wmill generate-metadata` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's `AGENTS.md` opts into automatic metadata. After running it, check the regenerated `.lock` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only `wmill sync push` deploys; run it only when the user explicitly asks. ### Visual preview diff --git a/system_prompts/base/flow-cli.md b/system_prompts/base/flow-cli.md index d8a2b41b13..fa4ec9cfcc 100644 --- a/system_prompts/base/flow-cli.md +++ b/system_prompts/base/flow-cli.md @@ -72,6 +72,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute `wmill flow preview -d ''` directly — pick plausible args from the flow's input schema. +An input typed as a resource (`format: resource-` in the schema) takes the bare string `"$res:"` as its whole value — `-d '{"db": "$res:f/databases/postgres_prod"}'`, not `{"db": {"$res": "..."}}` and not a plain path. Same for a variable, with `"$var:"`. See the `resources` skill. + `wmill flow preview` is safe to run yourself (it does not deploy). `wmill generate-metadata` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's `AGENTS.md` opts into automatic metadata. After running it, check the regenerated `.lock` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only `wmill sync push` deploys; run it only when the user explicitly asks. ### Visual preview diff --git a/system_prompts/base/resources.md b/system_prompts/base/resources.md index 1c548a2484..5252683a81 100644 --- a/system_prompts/base/resources.md +++ b/system_prompts/base/resources.md @@ -59,6 +59,41 @@ Reference other resources: } ``` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema `format: resource-`) is passed as +the **bare string** `$res:` — the whole argument value. Same for a variable, with +`$var:`. This applies everywhere job arguments are supplied: `wmill script run/preview`, +`wmill flow run/preview`, the `runScriptByPath` / `runFlowByPath` API, a schedule's `args`, a +trigger's configured static args. + +```json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +``` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with `$res:` / `$var:`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +```json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +``` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(`{ "gh_auth": { "token": "$var:g/all/gh_token" } }`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index a972a6bcc5..f148f1f03c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.800.0", + "version": "1.803.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 7efdb21bc3..4970364ccd 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.800.0", + "version": "1.803.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 51a576cb15..e5952c62ae 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.800.0 +1.803.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 09a4ec629c..d8c4b1e0fd 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.800.0", + "version": "1.803.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.800.0", + "version": "1.803.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index edb2469364..8245569685 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.800.0", + "version": "1.803.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",
diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 4a0bd58715..b321e76693 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1070,8 +1070,10 @@
  • feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions - are started for, and the plan tier and quota shown when the execution meter is - opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1123,8 +1125,10 @@
  • feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions - are started for, and the plan tier and quota shown when the execution meter is - opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1152,20 +1156,19 @@
    Instance object storage is not configured, so every server and worker keeps its log - files on its own disk. This page lists what each host wrote, but can only open the - files belonging to the replica serving the request — another host's are listed and - not readable — and a host's files go with it when it is replaced. Retention below - still governs the entries in the database and the files on disk. + files on its own disk. This page lists what each host wrote, but can only open the files + belonging to the replica serving the request — another host's are listed and not + readable — and a host's files go with it when it is replaced. Retention below still + governs the entries in the database and the files on disk.
    {:else if !$enterpriseLicense}
    Log files are uploaded to instance object storage, and the indexer that would ingest - them into the columnar store and delete each one afterwards is an enterprise - feature. Retention below expires the database entries and the local files; the - uploaded copies are only removed when Delete logs from s3 periodically is on - under Object Storage. + them into the columnar store and delete each one afterwards is an enterprise feature. + Retention below expires the database entries and the local files; the uploaded copies + are only removed when Delete logs from s3 periodically is on under Object Storage.
    {/if} diff --git a/frontend/src/lib/components/LabelsInput.svelte b/frontend/src/lib/components/LabelsInput.svelte index 933fe38ea1..c7eda30d66 100644 --- a/frontend/src/lib/components/LabelsInput.svelte +++ b/frontend/src/lib/components/LabelsInput.svelte @@ -8,9 +8,22 @@ labels: string[] | undefined onchange?: () => void class?: string + /** Suggest the labels of this workspace rather than the active one, for an editor + * aimed elsewhere (the folder drawer opened from a cross-workspace picker). */ + workspace?: string + /** Text typed into the input but not yet added to `labels`. An editor with a Save + * button needs it: without it that text is invisible to the editor's dirty state, + * so it is silently dropped on close and cannot even enable Save on its own. */ + onPendingChange?: (pending: string) => void } - let { labels = $bindable(), onchange, class: clazz = '' }: Props = $props() + let { + labels = $bindable(), + onchange, + class: clazz = '', + workspace, + onPendingChange + }: Props = $props() let adding = $state(false) let inputValue = $state('') @@ -34,9 +47,13 @@ !(labels ?? []).includes(trimmedInput) ) + $effect(() => { + onPendingChange?.(adding ? trimmedInput : '') + }) + async function loadExistingLabels() { try { - const resp = await fetch(`/api/w/${$workspaceStore}/labels/list`) + const resp = await fetch(`/api/w/${workspace ?? $workspaceStore}/labels/list`) if (resp.ok) existingLabels = await resp.json() } catch {} } @@ -82,8 +99,15 @@ addLabel() // either "Create new" selected or free text } } else if (e.key === 'Escape') { + // Escape cancels the label, and nothing else. Left to bubble it also reaches + // whatever encloses us — a drawer or dialog closes on it, and one guarding on + // unsaved changes reads `pending` before this clears it, so it prompts to + // discard work this key just discarded. + e.preventDefault() + e.stopPropagation() inputValue = '' adding = false + onPendingChange?.('') } else if (e.key === 'ArrowDown') { e.preventDefault() const maxIdx = suggestions.length + (showCreateNew ? 1 : 0) - 1 @@ -100,6 +124,14 @@ if (adding) addLabel() }, 150) } + + /** Add whatever is typed but not yet committed, right now. Blur commits on a 150ms + * grace period, so a caller that reads `labels` in the same tick as the blur — a Save + * button, which blurs this input by being clicked — would miss the last label. + * `adding` is cleared here, so the pending timer then finds nothing to do. */ + export function flushPendingLabel(): void { + if (adding) addLabel() + }
    diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index c7f7065565..14658a4228 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -305,6 +305,7 @@ resourceType="s3_bucket" workspaceOverride="admins" buttonTextOverride="Test from a worker" + viaWorker />
    diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 29da96cf98..471ed74099 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -252,7 +252,7 @@ {/if}
    Extra permissions ({acls?.length ?? 0})Extra members ({acls?.length ?? 0}) {#if linkedVarPaths.length > 0}
    @@ -299,7 +299,7 @@ size="lg" variant="accent" disabled={!newOwner} - on:click={() => addAcl(newOwner, write)}>Add permission addAcl(newOwner, write)}>Add member
    {/if} @@ -307,7 +307,7 @@ {#snippet headerRow()} - owner + member diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 37e3e152db..6a58af8031 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -1,5 +1,13 @@ @@ -22,6 +24,7 @@ on:click={() => (dispatch('close'), onClick?.())} on:pointerdown={(e) => e.stopPropagation()} {id} + {title} startIcon={{ icon: Icon ?? X }} iconOnly unifiedSize="sm" diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index b9d393035b..f85d34f225 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -23,6 +23,12 @@ /** Tailwind z-index class for the modal root. Override to stack this modal * above another modal that's already open (both default to `z-[9999]`). */ zIndexClass?: string + /** Render into `body` instead of where this component sits. Needed when an ancestor + * creates a stacking context the dialog has to escape — a drawer paints over the page + * whatever the dialog's z-index, and a `transform`, `filter` or `overflow` on the way + * up confines it. Off by default: it moves the dialog out of its DOM position, so opt + * in per call site rather than assuming every caller wants it. */ + alwaysPortal?: boolean children?: Snippet onConfirmed?: () => void | Promise onCanceled?: () => void @@ -40,6 +46,7 @@ id, trashbin = false, zIndexClass = 'z-[9999]', + alwaysPortal = false, children, onConfirmed, onCanceled @@ -141,7 +148,11 @@ - + {#if open}
    -
    -
    - -

    AI can help with these inputs

    - - {#snippet fallback()} - - {/snippet} - +

    AI can help with these inputs

    + + {#snippet fallback()} + + {/snippet} + +
    +
    +

    + {instructions + ? 'Instructions: ' + instructions + : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} +

    +
    -
    -

    - {instructions - ? 'Instructions: ' + instructions - : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} -

    -
    -
    +{/if} diff --git a/frontend/src/lib/components/copilot/AIFormSettings.svelte b/frontend/src/lib/components/copilot/AIFormSettings.svelte index 44151a01b3..ef627ace4b 100644 --- a/frontend/src/lib/components/copilot/AIFormSettings.svelte +++ b/frontend/src/lib/components/copilot/AIFormSettings.svelte @@ -3,6 +3,7 @@ import Label from '../Label.svelte' import Toggle from '../Toggle.svelte' import Tooltip from '../Tooltip.svelte' + import { copilotInfo } from '$lib/aiStore' interface Props { prompt?: string | undefined @@ -12,35 +13,37 @@ let { prompt = $bindable(undefined), type = 'script' }: Props = $props() -
    - { - if (prompt !== undefined) { - prompt = undefined - } else { - prompt = '' - } - }} - options={{ right: `Enable filling ${type} inputs with AI` }} - /> - {#if prompt !== undefined} -
    - -
    - {/if} -
    +{#if !$copilotInfo.workspaceDisabled} +
    + { + if (prompt !== undefined) { + prompt = undefined + } else { + prompt = '' + } + }} + options={{ right: `Enable filling ${type} inputs with AI` }} + /> + {#if prompt !== undefined} +
    + +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index 6a8dc6892e..94715a0f18 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -79,66 +79,68 @@ }) - - {#snippet trigger()} -
    - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} -