diff --git a/.claude/settings.json b/.claude/settings.json index a464ca3719..37aaeeb0f5 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -55,22 +55,18 @@ "mcp__claude_ai_Gmail__list_drafts" ], "deny": [ - "Read(.env)", - "Read(.env.*)", - "Read(**/.env)", - "Read(**/.env.*)", - "Read(**/secrets/**)", - "Read(**/*.pem)", - "Read(**/*.key)", - "Read(**/credentials.json)", - "Read(**/.secret*)", - "Read(**/.secrets*)", - "Read(**/*.secret)", - "Read(**/*.secrets)", "Edit(.env)", "Edit(.env.*)", "Edit(**/.env)", - "Edit(**/.env.*)" + "Edit(**/.env.*)", + "Edit(**/secrets/**)", + "Edit(**/*.pem)", + "Edit(**/*.key)", + "Edit(**/credentials.json)", + "Edit(**/.secret*)", + "Edit(**/.secrets*)", + "Edit(**/*.secret)", + "Edit(**/*.secrets)" ], "ask": [ "Bash(rmdir:*)", diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cdd0e2a212..9f65c2ae47 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -61,7 +61,7 @@ jobs: bun-version: 1.4.0 - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "24" - uses: astral-sh/setup-uv@v6.2.1 with: version: "0.11.24" diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index ee732569dd..bd15ec8876 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -9,6 +9,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -22,6 +23,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -59,7 +61,7 @@ jobs: echo "$CHANGED_FILES" # Direct git sync file changes — always relevant. - if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|frontend/src/lib/hubPaths\.json|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Relevant: direct git sync file changes" exit 0 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ab80d05f6b..249f8350ae 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.800.1" + ".": "1.803.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc8ecb481..33e24c0c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,58 @@ # Changelog +## [1.803.0](https://github.com/windmill-labs/windmill/compare/v1.802.0...v1.803.0) (2026-09-03) + + +### Features + +* expose request headers to scripts invoked via MCP ([#10903](https://github.com/windmill-labs/windmill/issues/10903)) ([e474e88](https://github.com/windmill-labs/windmill/commit/e474e8803ce2ff5c2df09a58dab51d45f5c922ca)) +* reuse an existing workspace resource in the project import wizard ([#10935](https://github.com/windmill-labs/windmill/issues/10935)) ([582761e](https://github.com/windmill-labs/windmill/commit/582761e37c776e92dc1c6ebfee8c4efe7c35d822)) + + +### Bug Fixes + +* bump git sync hub scripts to cli 1.802.1, test the fork ui pull ([#10955](https://github.com/windmill-labs/windmill/issues/10955)) ([ca88009](https://github.com/windmill-labs/windmill/commit/ca8800959aa6a0017cc29bad187c9f49e0d13cc4)) +* **cli:** make a sync push into a fork converge on schedules and inline names ([#10951](https://github.com/windmill-labs/windmill/issues/10951)) ([0f5a1db](https://github.com/windmill-labs/windmill/commit/0f5a1db2abba8df30a2f975f4498e269f13cf93d)) +* fade the home Build with AI placeholder every 10s instead of typing it ([#10953](https://github.com/windmill-labs/windmill/issues/10953)) ([3d089b5](https://github.com/windmill-labs/windmill/commit/3d089b57344f5814086e6176301c5031dc519674)) +* let operators use wmill.datatable() from within running jobs ([#10931](https://github.com/windmill-labs/windmill/issues/10931)) ([9b64a89](https://github.com/windmill-labs/windmill/commit/9b64a89cd46ae718d6c58fa12f925fa041fb1032)) + +## [1.802.0](https://github.com/windmill-labs/windmill/compare/v1.801.0...v1.802.0) (2026-09-02) + + +### Features + +* add retention cleanup for the otel_traces table ([#10949](https://github.com/windmill-labs/windmill/issues/10949)) ([d472193](https://github.com/windmill-labs/windmill/commit/d472193e5bf5f6428e0096a402eb2c9299634fb2)) +* open path links from chat messages in the session preview panel ([#10881](https://github.com/windmill-labs/windmill/issues/10881)) ([f10ac6c](https://github.com/windmill-labs/windmill/commit/f10ac6c2b3644fb16697e650efbc4f7cd3c6944c)) +* restore owner and label filter chips on the homepage ([#10942](https://github.com/windmill-labs/windmill/issues/10942)) ([ccf8476](https://github.com/windmill-labs/windmill/commit/ccf84761dd664b9228dfe2f65867e8c32cd20c21)) +* **sessions:** offer the item you came from when starting a new session ([#10940](https://github.com/windmill-labs/windmill/issues/10940)) ([d3747d6](https://github.com/windmill-labs/windmill/commit/d3747d62555ebcb09c78cfabcaa3b6177758d6ea)) +* workspace setting to hide the AI assistant, agent steps unaffected ([#10941](https://github.com/windmill-labs/windmill/issues/10941)) ([fdd3b36](https://github.com/windmill-labs/windmill/commit/fdd3b36423344a2e1a464674179406581074e926)) + + +### Bug Fixes + +* apply object-storage test SSRF validation to all non-super-admins ([#10933](https://github.com/windmill-labs/windmill/issues/10933)) ([4fef119](https://github.com/windmill-labs/windmill/commit/4fef1195adaa9fa036a219884bd6c996460ca37f)) +* connect to dev server instead of localhost ([#10912](https://github.com/windmill-labs/windmill/issues/10912)) ([337154b](https://github.com/windmill-labs/windmill/commit/337154b8304a5969f35216add627b5c1153c0f6c)) +* preselect first row of AI agent and AI sandbox insert panes ([#10937](https://github.com/windmill-labs/windmill/issues/10937)) ([95b6bbd](https://github.com/windmill-labs/windmill/commit/95b6bbd46ada11d96a914ae5b0e92aba4dd02530)) +* record supplied script lock hashes so importers can skip relocking ([#10915](https://github.com/windmill-labs/windmill/issues/10915)) ([17ba521](https://github.com/windmill-labs/windmill/commit/17ba521c352aec65a8270893752bbadd7f3d6eaa)) +* sandbox script-controlled content types in result_to_response ([#10932](https://github.com/windmill-labs/windmill/issues/10932)) ([419741e](https://github.com/windmill-labs/windmill/commit/419741e5d226c67c51429094fb6ded9474afed99)) + +## [1.801.0](https://github.com/windmill-labs/windmill/compare/v1.800.1...v1.801.0) (2026-09-01) + + +### Features + +* **ai-chat:** make reusable skills ai_skill resources you select per workspace ([#10914](https://github.com/windmill-labs/windmill/issues/10914)) ([cfcfe29](https://github.com/windmill-labs/windmill/commit/cfcfe298dd9ab50196bd64926ef78c4563f58c2c)) +* **ai-sessions:** show a running session across tabs and reload finished turns ([#10916](https://github.com/windmill-labs/windmill/issues/10916)) ([816dc9d](https://github.com/windmill-labs/windmill/commit/816dc9dcd2c310e499d2d210a0abcd403469f29c)) +* edit folders and groups in a drawer that saves once ([#10873](https://github.com/windmill-labs/windmill/issues/10873)) ([5d5ad4e](https://github.com/windmill-labs/windmill/commit/5d5ad4e8974e076ef53a26a5584e4209255a2248)) +* make the home Build with AI composer dismissible, quiet the rest of the home page ([#10930](https://github.com/windmill-labs/windmill/issues/10930)) ([772fafe](https://github.com/windmill-labs/windmill/commit/772fafec8316a1e0c0e76b9a0737cc41d40a9a8c)) + + +### Bug Fixes + +* let a principal without a login account own a draft ([#10925](https://github.com/windmill-labs/windmill/issues/10925)) ([94af8d0](https://github.com/windmill-labs/windmill/commit/94af8d0fb5aceebe83936fd6761c6c1c02c75323)) +* resolve chat path links against the session's operating workspace ([#10924](https://github.com/windmill-labs/windmill/issues/10924)) ([9074de2](https://github.com/windmill-labs/windmill/commit/9074de25ea730ca02653c9a2e2b8b99eda6f3137)) +* tolerate string app_id in GHES app config deserialization ([#10923](https://github.com/windmill-labs/windmill/issues/10923)) ([af8ff38](https://github.com/windmill-labs/windmill/commit/af8ff3868748412cb658c803ebc8a71edc3cd8fb)) + ## [1.800.1](https://github.com/windmill-labs/windmill/compare/v1.800.0...v1.800.1) (2026-09-01) diff --git a/CONTEXT.md b/CONTEXT.md index 6efb92b669..64aa1b93d8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -36,3 +36,22 @@ _Avoid_: argument field, param **Expression input**: Any other place a property can be picked into: the loop iterator, skip and early-stop predicates, the retry condition, a branch predicate, timeout. Its prop picker opens in a popover from the connect button rather than taking a pane. _Avoid_: JS field, code input + +### Permissions + +**Member**: +A user or group granted a role on a folder, a group, or an item's extra ACL. The list of them is +"Members (n)" everywhere it is shown, and one is added with "Add member". +_Avoid_: participant, collaborator, owner, ACL entry, permission (that names the concept, not the people) + +**Role**: +The access level a member holds: viewer, writer or admin on a folder; member or admin on a group. +Viewers read, writers also edit, admins also manage the members. A group role of **manager** — +manages the group without belonging to it — is a legacy state the UI shows and can leave, but +offers no way to enter. +_Avoid_: permission level, access level, rank + +**Owner**: +Reserved for the path prefix that says where an item lives — `u/alice` or `f/team`. A folder's +`owners` column in the database is its admin members; call those admins, never owners, in the UI. +_Avoid_: using "owner" for a folder admin diff --git a/Dockerfile b/Dockerfile index 8ecee96725..6a290c3553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,9 +141,9 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 ARG KUBECTL_VERSION=1.36.2 -ARG HELM_VERSION=3.21.2 +ARG HELM_VERSION=3.21.4 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte -ARG GO_VERSION=1.26.0 +ARG GO_VERSION=1.26.8 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true @@ -250,8 +250,8 @@ RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_r RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ +RUN curl -sL https://deb.nodesource.com/setup_24.x | bash - +RUN apt-get -y update && apt-get install -y --no-install-recommends curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* # go build is slower the first time it is ran, so we prewarm it in the build @@ -299,7 +299,7 @@ RUN bun install -g windmill-cli \ RUN curl -fsSL https://claude.ai/install.sh | bash \ && cp /root/.local/share/claude/versions/* /usr/bin/claude -COPY --from=php:8.3.30-cli-trixie /usr/local/bin/php /usr/bin/php +COPY --from=php:8.3.33-cli-trixie /usr/local/bin/php /usr/bin/php COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 5c682be5eb..6d80f19197 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -7,6 +7,7 @@ import type { ListableApp, ListableResource, ListableVariable, + Resource, Script } from '../../../frontend/src/lib/gen' import type { @@ -81,6 +82,15 @@ export interface BenchmarkWorkspaceAiProvider { isDefault?: boolean } +/** A plain (non-AI) resource of the benchmark workspace, for cases about referencing a + * credential — passing one as a run argument, say. `value` is what `get_resource` returns. */ +export interface BenchmarkWorkspaceResource { + path: string + resource_type: string + value?: Record + description?: string +} + export interface BenchmarkWorkspaceJob { /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ id?: string @@ -98,6 +108,7 @@ export interface BenchmarkWorkspaceRunnables { apps?: BenchmarkWorkspaceApp[] variables?: BenchmarkWorkspaceVariable[] aiProviders?: BenchmarkWorkspaceAiProvider[] + resources?: BenchmarkWorkspaceResource[] datatables?: BenchmarkDatatableSeed[] jobs?: BenchmarkWorkspaceJob[] } @@ -284,15 +295,71 @@ export function listBenchmarkAiProviderResources(workspace: string): ListableRes })) } -/** The value of a seeded AI provider resource. Only the endpoint fields are modelled — a key is - * never needed, because no eval run calls the provider through this resource. */ +/** Plain seeded resources of a benchmark workspace, shaped like `ResourceService.listResource` + * rows. Null when the workspace is not a benchmark one. */ +export function listBenchmarkPlainResources(workspace: string): ListableResource[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + return null + } + return (runnables.resources ?? []).map((seed) => ({ + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: null, + is_oauth: false, + is_linked: false, + is_refreshed: false, + extra_perms: {}, + edited_at: BENCHMARK_TIMESTAMP + })) +} + +/** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both + * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI + * providers too, and a case that lists resources and then reads one by path would otherwise get + * a row it cannot fetch. */ +export function getBenchmarkResource(workspace: string, path: string): Resource | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const seed = runnables?.resources?.find((entry) => entry.path === path) + if (seed) { + return { + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: seed.value ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource + } + const provider = runnables?.aiProviders?.find((entry) => entry.path === path) + if (!provider) { + return null + } + return { + workspace_id: workspace, + path: provider.path, + resource_type: provider.kind, + value: getBenchmarkResourceValue(workspace, path) ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource +} + +/** The value of a seeded resource. For an AI provider only the endpoint fields are modelled — a + * key is never needed, because no eval run calls the provider through this resource. */ export function getBenchmarkResourceValue( workspace: string, path: string ): Record | null { - const seed = benchmarkWorkspaceRunnables - .get(workspace) - ?.aiProviders?.find((entry) => entry.path === path) + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const plain = runnables?.resources?.find((entry) => entry.path === path) + if (plain) { + return plain.value ?? {} + } + const seed = runnables?.aiProviders?.find((entry) => entry.path === path) if (!seed) { return null } diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index e59ec11ad9..338ed8504c 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -70,7 +70,9 @@ vi.mock('$lib/gen', async () => { getBenchmarkResourceValue, getBenchmarkVariableByPath, hasBenchmarkWorkspace, + getBenchmarkResource, listBenchmarkAiProviderResources, + listBenchmarkPlainResources, listBenchmarkApps, listBenchmarkDatatables, listBenchmarkDrafts, @@ -359,18 +361,24 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkResourceValue(data.workspace, data.path)) : actual.ResourceService.existsResource(data), - // Only AI provider resources are modelled: they are what an AI agent step references. listResource: async (data: { workspace: string; resourceType?: string }) => { if (!hasBenchmarkWorkspace(data.workspace)) { return actual.ResourceService.listResource(data) } - const seeded = listBenchmarkAiProviderResources(data.workspace) ?? [] + const seeded = [ + ...(listBenchmarkAiProviderResources(data.workspace) ?? []), + ...(listBenchmarkPlainResources(data.workspace) ?? []) + ] const wanted = data.resourceType?.split(',') return wanted ? seeded.filter((r) => wanted.includes(r.resource_type)) : seeded }, getResource: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + const resource = getBenchmarkResource(data.workspace, data.path) + if (!resource) { + throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + } + return resource } return actual.ResourceService.getResource(data) }, diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 21f80bef7e..94ecb5c029 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -2365,3 +2365,37 @@ - the step uses the workspace's anthropic resource f/evals/global/anthropic_main - the model is the Opus one the user asked for, taken from the models that resource serves - the diff flow input reaches the agent + +# The failure this pins: passing a resource as `{"$res": ""}` (or as a bare path), which +# reaches the script unresolved because the backend only substitutes a string value that itself +# starts with `$res:`. The mock preview echoes args back and reports success, so nothing in the +# loop corrects a wrong shape — the arg form is the whole test. +- id: global-run-arg-resource-reference + prompt: |- + Run `f/evals/global/github_repo_stats` against the `windmill-labs/windmill` repo, passing our + GitHub credentials at `f/evals/global/github_main` as its `gh_auth` input, and tell me whether + it went through. + initial: ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - test_run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + toolCallArgs: + # Exact: the mock never resolves the reference, so a near-miss path like + # `$res:f/evals/global/github_main_backup` would otherwise pass. + - tool: test_run_script + field: args.gh_auth + stringEqualsAnyOf: + - "$res:f/evals/global/github_main" + # The judge only sees drafts, and this case makes none — the deliverable is the shape of the + # run argument, checked deterministically above. + skipJudge: true + judgeChecklist: + - runs the existing script rather than rewriting it + - passes the GitHub resource as the bare string $res:f/evals/global/github_main diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index c9a4f7830e..4107dc53da 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -160,6 +160,13 @@ export interface ToolCallArgumentRule { field: string; stringStartsWithAnyOf?: string[]; stringMustNotStartWithAnyOf?: string[]; + /** + * Universal over calls: every recorded call to `tool` must carry `field` as + * exactly one of these strings. Use when a near-miss would still satisfy a + * prefix — a resource reference like `$res:f/a/b` shares its prefix with the + * wrong `$res:f/a/b_backup`, and the mock never resolves it to catch that. + */ + stringEqualsAnyOf?: string[]; /** * Case-insensitive "contains", existential over calls: at least one recorded * call to `tool` must have `field` containing one of these substrings. Other diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 9b606c1c85..4b7891f67e 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -228,6 +228,43 @@ describe("validateToolExpectations", () => { }); }); + // A resource reference shares its prefix with a wrong sibling path, and the mock + // never resolves it, so only exact matching separates the two. + it("rejects a resource reference whose path merely shares the prefix", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["test_run_script"], + toolCallDetails: [ + { + name: "test_run_script", + arguments: { args: { gh_auth: "$res:f/evals/global/github_main_backup" } }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [ + { + tool: "test_run_script", + field: "args.gh_auth", + stringEqualsAnyOf: ["$res:f/evals/global/github_main"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "test_run_script.args.gh_auth matches an accepted value", + passed: false, + details: + 'accepted values: $res:f/evals/global/github_main; values: "$res:f/evals/global/github_main_backup"', + }); + }); + // The whole point of the same-call rule: the per-field rules are existential over // calls, so two single-filter pages would satisfy them while never opening the // combined view the case asks for. diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 7150dab0c3..132a9ff3d0 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -278,6 +278,20 @@ export function validateToolExpectations(input: { ); } + if (rule.stringEqualsAnyOf && rule.stringEqualsAnyOf.length > 0) { + const invalidValues = values.filter( + (value) => + typeof value !== "string" || !rule.stringEqualsAnyOf!.includes(value) + ); + checks.push( + check( + `${rule.tool}.${rule.field} matches an accepted value`, + invalidValues.length === 0, + `accepted values: ${rule.stringEqualsAnyOf.join(", ")}; values: ${summarizeToolValues(values)}` + ) + ); + } + if (rule.stringMustNotStartWithAnyOf && rule.stringMustNotStartWithAnyOf.length > 0) { const invalidValues = values.filter( (value) => diff --git a/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json new file mode 100644 index 0000000000..80e1b960c5 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "resources": [ + { + "path": "f/evals/global/github_main", + "resource_type": "github", + "description": "GitHub credentials", + "value": { "token": "$var:f/evals/global/github_token" } + } + ], + "scripts": [ + { + "path": "f/evals/global/github_repo_stats", + "summary": "Count open issues on a GitHub repository", + "description": "Reads the open issue count for a repository using GitHub credentials.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "gh_auth": { + "type": "object", + "format": "resource-github", + "description": "GitHub credentials" + }, + "repo": { + "type": "string", + "description": "Repository in owner/name form" + } + }, + "required": ["gh_auth", "repo"] + }, + "content": "type Github = { token: string }\n\nexport async function main(gh_auth: Github, repo: string) {\n const res = await fetch(`https://api.github.com/repos/${repo}/issues?state=open`, {\n headers: { Authorization: `Bearer ${gh_auth.token}` }\n })\n const issues = await res.json()\n return { repo, open_issues: issues.length }\n}\n" + } + ] + } +} diff --git a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json b/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json deleted file mode 100644 index abae0ba456..0000000000 --- a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COUNT(*)::bigint AS \"total!\",\n COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS \"replacing!\"\n FROM ai_skill\n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "total!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "replacing!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7" -} diff --git a/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json new file mode 100644 index 0000000000..304efc22d2 --- /dev/null +++ b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n LEFT JOIN password p\n ON p.email = d.email\n AND p.super_admin = true\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)\n ORDER BY d.email NULLS LAST", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username?", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "draft_saved_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f" +} diff --git a/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json new file mode 100644 index 0000000000..8275796e90 --- /dev/null +++ b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88" +} diff --git a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json similarity index 50% rename from backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json rename to backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json index 29ce0c6f2c..03f11ef137 100644 --- a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json +++ b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", + "query": "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "name", - "type_info": "Varchar" + "name": "lockfile_hash", + "type_info": "Int8" } ], "parameters": { @@ -19,5 +19,5 @@ false ] }, - "hash": "e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61" + "hash": "1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a" } diff --git a/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json new file mode 100644 index 0000000000..693d84173a --- /dev/null +++ b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trace_id FROM otel_traces ORDER BY trace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trace_id", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3" +} diff --git a/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json new file mode 100644 index 0000000000..73d5000f2d --- /dev/null +++ b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.path = dest.path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef" +} diff --git a/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json new file mode 100644 index 0000000000..f35dbc23e4 --- /dev/null +++ b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM otel_traces WHERE ctid IN (\n SELECT ctid FROM otel_traces\n WHERE start_time_unix_nano < EXTRACT(\n EPOCH FROM now() - ($1::bigint::text || ' s')::interval\n )::bigint * 1000000000\n LIMIT $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e" +} diff --git a/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json new file mode 100644 index 0000000000..f7c39c21fc --- /dev/null +++ b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET email = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8" +} diff --git a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json b/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json deleted file mode 100644 index 8d86ff3db6..0000000000 --- a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Text", - "Varchar", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57" -} diff --git a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json b/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json deleted file mode 100644 index 63283a830a..0000000000 --- a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at)\n VALUES ('admins', $1, $2, $3, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb" -} diff --git a/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json new file mode 100644 index 0000000000..681206a381 --- /dev/null +++ b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668" +} diff --git a/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json new file mode 100644 index 0000000000..67b672228f --- /dev/null +++ b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano)\n VALUES ($1, $2, 'GET /', 3, $3, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc" +} diff --git a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json b/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json deleted file mode 100644 index d55722ed24..0000000000 --- a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by)\n VALUES ($1, $2, $3, $4, now(), $5)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET description = EXCLUDED.description,\n instructions = EXCLUDED.instructions,\n edited_at = now(),\n edited_by = EXCLUDED.edited_by", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8" -} diff --git a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json b/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json similarity index 63% rename from backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json rename to backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json index d453eb430d..3b9a2e2f34 100644 --- a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json +++ b/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", "describe": { "columns": [ { @@ -13,12 +13,14 @@ "Left": [ "Text", "Jsonb", - "Text" + "Text", + "Text", + "Bool" ] }, "nullable": [ null ] }, - "hash": "1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae" + "hash": "8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a" } diff --git a/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json new file mode 100644 index 0000000000..610030cffc --- /dev/null +++ b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, lock FROM script\n WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "lock", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455" +} diff --git a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json b/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json deleted file mode 100644 index 733478e0e9..0000000000 --- a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7" -} diff --git a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json new file mode 100644 index 0000000000..3624a32893 --- /dev/null +++ b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at)\n VALUES ('admins', $1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9" +} diff --git a/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json new file mode 100644 index 0000000000..c7db67f4d6 --- /dev/null +++ b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e" +} diff --git a/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json new file mode 100644 index 0000000000..0a0f8b4732 --- /dev/null +++ b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + null + ] + }, + "hash": "9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032" +} diff --git a/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json new file mode 100644 index 0000000000..f4c4fd31de --- /dev/null +++ b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text", + "Varchar", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4" +} diff --git a/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json new file mode 100644 index 0000000000..c3ef22f973 --- /dev/null +++ b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db" +} diff --git a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json b/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json deleted file mode 100644 index ca39442f59..0000000000 --- a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "description", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b" -} diff --git a/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json new file mode 100644 index 0000000000..19dc4781a6 --- /dev/null +++ b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n SELECT $1, * FROM UNNEST($2::text[], $3::bigint[])\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash\n WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a" +} diff --git a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json similarity index 72% rename from backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json rename to backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json index b31e532fc2..2947dd8413 100644 --- a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json +++ b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4" + "hash": "cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f" } diff --git a/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json new file mode 100644 index 0000000000..eba1b0da99 --- /dev/null +++ b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a" +} diff --git a/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json new file mode 100644 index 0000000000..6ffd197e90 --- /dev/null +++ b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_fileset, format_extension FROM resource_type\n WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fileset", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "format_extension", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a" +} diff --git a/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json new file mode 100644 index 0000000000..fc42c451f8 --- /dev/null +++ b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a" +} diff --git a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json b/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json deleted file mode 100644 index 121d7fe04a..0000000000 --- a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "instructions", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d" -} diff --git a/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json new file mode 100644 index 0000000000..bea6be2a7f --- /dev/null +++ b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, lock_error_logs, created_at)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, $6::text, clock_timestamp()\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637" +} diff --git a/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json new file mode 100644 index 0000000000..24f957a4e2 --- /dev/null +++ b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM draft ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c79abcb78e..d314168b35 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -981,9 +981,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -5177,9 +5177,9 @@ dependencies = [ [[package]] name = "frostem" -version = "1.20260821.3" +version = "1.20260821.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2e81eab151ba68484704bb3d21b4b4d2747314d3081fa86fa6c7300a4c42e2" +checksum = "481ace7f781f5ae54a5c0a6d6d8edb30adba737cfa1230fbd5632d63ba8dfd80" [[package]] name = "fs3" @@ -7262,9 +7262,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ "bitflags 2.13.1", "libc", @@ -7387,9 +7387,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.3" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] @@ -7776,9 +7776,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -7870,10 +7870,11 @@ dependencies = [ [[package]] name = "mysql_async" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2" +checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4" dependencies = [ + "arc-swap", "bytes", "crossbeam-queue", "crossbeam-utils", @@ -7882,7 +7883,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.3", + "lru 0.18.4", "mysql_common", "native-tls", "pem 3.0.6", @@ -11753,9 +11754,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" dependencies = [ "serde", ] @@ -12812,7 +12813,7 @@ dependencies = [ "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.18.3", + "lru 0.18.4", "lz4_flex 0.14.0", "measure_time", "memmap2", @@ -14746,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -14831,7 +14832,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.800.1" +version = "1.803.0" dependencies = [ "async-stream", "async-trait", @@ -14864,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14877,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "argon2", @@ -15017,7 +15018,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15040,7 +15041,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15057,7 +15058,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15083,7 +15084,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.800.1" +version = "1.803.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15093,7 +15094,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15110,7 +15111,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15132,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15155,7 +15156,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15171,7 +15172,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15193,7 +15194,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15214,7 +15215,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15228,7 +15229,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -15263,7 +15264,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15288,7 +15289,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15316,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15338,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15358,7 +15359,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15396,7 +15397,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15424,7 +15425,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.800.1" +version = "1.803.0" dependencies = [ "lazy_static", "serde", @@ -15436,7 +15437,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.800.1" +version = "1.803.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15460,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15474,7 +15475,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.800.1" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15509,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.800.1" +version = "1.803.0" dependencies = [ "chrono", "lazy_static", @@ -15523,7 +15524,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15542,7 +15543,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.800.1" +version = "1.803.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15646,9 +15647,10 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.800.1" +version = "1.803.0" dependencies = [ "chrono", + "futures", "itertools 0.14.0", "lazy_static", "serde", @@ -15665,7 +15667,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.800.1" +version = "1.803.0" dependencies = [ "regex", "serde", @@ -15680,7 +15682,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15707,7 +15709,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "futures", @@ -15724,7 +15726,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.800.1" +version = "1.803.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15740,7 +15742,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -15761,7 +15763,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -15792,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "arc-swap", @@ -15817,7 +15819,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-stream", @@ -15851,7 +15853,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "futures", @@ -15869,7 +15871,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.800.1" +version = "1.803.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15878,7 +15880,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15890,7 +15892,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15902,7 +15904,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "gosyn", @@ -15914,7 +15916,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15926,7 +15928,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15938,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "nu-parser", @@ -15949,7 +15951,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15960,7 +15962,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15972,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15983,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16005,7 +16007,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -16017,7 +16019,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16031,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16048,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16061,7 +16063,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde", @@ -16073,7 +16075,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16091,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16107,7 +16109,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16123,7 +16125,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16137,7 +16139,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16176,7 +16178,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "const_format", @@ -16216,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.800.1" +version = "1.803.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16227,7 +16229,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16262,7 +16264,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16286,7 +16288,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16319,7 +16321,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16346,7 +16348,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16379,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16399,7 +16401,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16433,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16469,7 +16471,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16492,7 +16494,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16516,7 +16518,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -16540,7 +16542,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16575,7 +16577,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16603,7 +16605,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16628,7 +16630,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16647,7 +16649,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-once-cell", @@ -16764,7 +16766,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.800.1" +version = "1.803.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 491f3cb48a..f432f022c5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.800.1" +version = "1.803.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.800.1" +version = "1.803.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index 0468ebf721..f5725d65a3 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -85,7 +85,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream | | EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation | | EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts | -| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | +| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers, script-controlled `wm_content_type`/`wm_headers` on `run_wait_result` and sync HTTP-route responses | stored user content → admin browser (same origin) | Admin session, account takeover | | EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | | EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | | EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS` now defaults to `true`; can still be overridden to `false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | @@ -106,7 +106,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | | T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | -| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | +| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, S3 download content-type, or a script-chosen `text/html` content type on `run_wait_result` / sync HTTP-route responses (GET-reachable with the `SameSite=Lax` session cookie) | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads and on every `result_to_response` composite result (inserted after `wm_headers`; hop-by-hop names such as `Connection` rejected so a proxy cannot strip them) | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0, WIN-2471 | | T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | | T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 | | T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 2bb93f6bd9..9e7a10edf0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -f2a31156ac08ecb02d89dbc66d72be58e9c877ff +d6297e6844dc2aab4745fce328e32ccab508969f diff --git a/backend/migrations/20260831120907_skills_resource_type.down.sql b/backend/migrations/20260831120907_skills_resource_type.down.sql new file mode 100644 index 0000000000..5cc32da37d --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.down.sql @@ -0,0 +1,13 @@ +-- The up migration only ever added: `ai_skill` still holds every skill it copied, +-- so there is nothing to restore and nothing to delete. Removing the resources +-- would destroy any a user has since edited or created, and removing a folder +-- would take whatever else was put in it. +-- +-- The seeded resource type goes. `created_by` only distinguishes this migration's +-- row from one a user created by hand: a hub sync updates the schema in place and +-- leaves `created_by` alone, so a synced-over row is still removed here and the +-- next sync puts it back. +DELETE FROM resource_type +WHERE workspace_id = 'admins' + AND name = 'ai_skill' + AND created_by = 'system'; diff --git a/backend/migrations/20260831120907_skills_resource_type.up.sql b/backend/migrations/20260831120907_skills_resource_type.up.sql new file mode 100644 index 0000000000..4e8f9cfa95 --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.up.sql @@ -0,0 +1,88 @@ +-- AI chat skills move from the `ai_skill` table onto ordinary resources, so they +-- gain folder ACLs, version history, workspace export and git-sync. An `ai_skill` +-- resource holds the SKILL.md body in `value.content`; its description lives in +-- the resource's own `description` column and its name is the path basename. +-- +-- Nothing here is destructive. `ai_skill` is left in place, unread, for a later +-- release to drop once operators have confirmed the copy. That is what lets every +-- step below skip on conflict rather than resolve one: a skipped row is still in +-- the table, so it is not lost, and the migration needs no record of what it did +-- in order to be reversible. + +-- `format_extension` makes the resource editor render `value.content` as a plain +-- .md file. Seeded under 'admins' so every workspace sees it. +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, format_extension, edited_at) +VALUES ( + 'admins', + 'ai_skill', + '{"type": "object", "properties": {"content": {"type": "string"}}}', + 'A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.', + 'system', + 'md', + now() +) +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Shared home matching the admin-only upload these skills had. A workspace that +-- already has a `skills` folder keeps it untouched, ACL and all: adopting one +-- would hand its own grants — possibly write for everyone — over a set of +-- instructions the assistant follows. +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) +SELECT DISTINCT workspace_id, 'skills', 'Skills', ARRAY[]::TEXT[], '{"g/all": false}'::jsonb +FROM ai_skill +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Copied only where the destination is free and the folder matches the one above +-- exactly, owners included: a pre-existing folder carrying the same ACL but an +-- owner would hand that owner update and delete over skills the removed API let +-- only workspace admins touch. Anything else stays in `ai_skill` for an operator +-- to place deliberately. +-- +-- What was actually inserted is recorded rather than inferred. Inferring it from +-- "is there an ai_skill resource at the destination" reports nothing when the +-- blocker is itself an ai_skill with different instructions — the one case where +-- the skipped skill is least likely to be noticed. +CREATE TEMP TABLE ai_skill_copied AS +WITH inserted AS ( + INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) + SELECT + s.workspace_id, + 'f/skills/' || s.name, + jsonb_build_object('content', s.instructions), + s.description, + 'ai_skill', + s.edited_by, + s.edited_at + FROM ai_skill s + JOIN folder f + ON f.workspace_id = s.workspace_id + AND f.name = 'skills' + AND f.extra_perms = '{"g/all": false}'::jsonb + AND cardinality(f.owners) = 0 + ON CONFLICT (workspace_id, path) DO NOTHING + RETURNING workspace_id, path +) +SELECT workspace_id, path FROM inserted; + +-- Anything not copied is still in `ai_skill`, but nothing reads that table any +-- more, so from the app's side the skill is missing until an operator places it. +-- Name them rather than leaving that to be discovered. +DO $$ +DECLARE + leftover RECORD; +BEGIN + FOR leftover IN + SELECT s.workspace_id, s.name + FROM ai_skill s + WHERE NOT EXISTS ( + SELECT 1 FROM ai_skill_copied c + WHERE c.workspace_id = s.workspace_id + AND c.path = 'f/skills/' || s.name + ) + LOOP + RAISE WARNING 'ai_skill %/% was not copied to a resource (its destination or the f/skills folder is already taken); it remains in the ai_skill table', + leftover.workspace_id, leftover.name; + END LOOP; +END $$; + +DROP TABLE ai_skill_copied; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql new file mode 100644 index 0000000000..e62fc02975 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql @@ -0,0 +1,12 @@ +-- Drafts owned by a principal with no login account cannot exist under the constraint; drop them +-- before restoring it. +DELETE FROM draft +WHERE email IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM password WHERE password.email = draft.email); + +ALTER TABLE draft + ADD CONSTRAINT draft_password_fkey + FOREIGN KEY (email) + REFERENCES password(email) + ON DELETE CASCADE + ON UPDATE CASCADE; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql new file mode 100644 index 0000000000..65a6686e95 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql @@ -0,0 +1,3 @@ +-- The delete and rename this cascaded are now explicit, at the sites that remove or rename an +-- account; `windmill_common::user_drafts::delete_drafts_of_email` carries the reasoning. +ALTER TABLE draft DROP CONSTRAINT IF EXISTS draft_password_fkey; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 1b09928903..395e692623 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.800.1" +version = "1.803.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.800.1" +version = "1.803.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.800.1" +version = "1.803.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.800.1" +version = "1.803.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 752d80a1e3..45c8b0d94e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.800.1" +version = "1.803.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index fa88cf6cdc..f84812151d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,14 +7,14 @@ */ use anyhow::Context; use monitor::{ - load_base_url, load_otel, reload_critical_alerts_on_db_oversize, - reload_delete_logs_periodically_setting, reload_indexer_config, - reload_instance_python_version_setting, reload_maven_repos_setting, + flush_pending_log_files_to_object_store, load_base_url, load_otel, + reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, + reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, reload_maven_settings_xml_setting, reload_no_default_maven_setting, reload_nuget_config_setting, reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, reload_workspace_registries_setting, - flush_pending_log_files_to_object_store, send_logs_to_object_store, WORKERS_NAMES, + send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::{Pool, Postgres}; @@ -53,9 +53,9 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, @@ -140,12 +140,12 @@ use crate::monitor::{ reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, - reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, - reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, - reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, - reload_service_log_retention_secs_setting, reload_smtp_config, + reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, + reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_sandbox_image_cache_max_setting, + reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, + reload_sandbox_image_pull_policy_setting, reload_sandbox_registry_auth_setting, + reload_scim_token_setting, reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -406,8 +406,12 @@ struct HubResourceTypeRaw { pub schema: Option, pub app: String, pub description: Option, + /// Absent from hubs predating the column, and from caches written before it. + #[serde(default)] + pub format_extension: Option, } + /// Processed resource type with parsed schema #[derive(serde::Deserialize, serde::Serialize, Clone)] pub struct HubResourceType { @@ -416,6 +420,18 @@ pub struct HubResourceType { pub schema: Option, pub app: String, pub description: Option, + /// Doubly optional on purpose. A cache written before this column has no key at + /// all (`None`) and must leave the stored extension alone; one written since + /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely + /// dropping it and must clear. A single `Option` conflates the two, and picking + /// either meaning breaks the other — as does plain serde, which folds `null` + /// into the outer `None`, hence the wrapping deserializer. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -462,6 +478,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { schema, app: rt.app, description: rt.description, + format_extension: Some(rt.format_extension), }) }) .collect(); @@ -503,9 +520,17 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh tracing::info!("Found {} cached resource types", cached_types.len()); - // Get existing resource types in admins workspace - let existing_types: Vec<(String, Option, Option)> = sqlx::query_as( - "SELECT name, schema, description FROM resource_type WHERE workspace_id = 'admins'", + // Get existing resource types in admins workspace. `format_extension` is part of + // the comparison below, so a type whose only change is gaining or losing it is + // not mistaken for unchanged; `is_fileset` decides whether it may take one. + let existing_types: Vec<( + String, + Option, + Option, + Option, + bool, + )> = sqlx::query_as( + "SELECT name, schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -513,19 +538,42 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option), + (Option, Option, Option, bool), > = existing_types .into_iter() - .map(|(name, schema, desc)| (name, (schema, desc))) + .map(|(name, schema, desc, format_extension, is_fileset)| { + (name, (schema, desc, format_extension, is_fileset)) + }) .collect(); let mut synced_count = 0; let mut skipped_count = 0; for rt in cached_types { - // Check if resource type already exists with same schema and description - if let Some((existing_schema, existing_desc)) = existing_map.get(&rt.name) { - if existing_schema == &rt.schema && existing_desc == &rt.description { + let existing = existing_map.get(&rt.name); + let is_fileset = existing.map(|(_, _, _, f)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _)| e.clone()); + // A fileset is a set of files, so it cannot also be one file. Create, update + // and the manual sync all reject the pair; this writer would otherwise + // persist it onto a same-named local fileset. + // + // A cache with no key at all leaves the stored value alone, so the target is + // what is already there — which is also what makes the comparison below + // agree with the write instead of re-upserting the row on every boot. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(from_cache) => from_cache.clone(), + None => stored_extension.clone(), + } + }; + + if let Some((existing_schema, existing_desc, _, _)) = existing { + if existing_schema == &rt.schema + && existing_desc == &rt.description + && stored_extension == format_extension + { skipped_count += 1; continue; } @@ -533,14 +581,19 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // Insert or update resource type sqlx::query( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // `format_extension` is resolved above rather than coalesced here: a + // COALESCE could never clear one, so a hub that dropped an extension + // would leave the stale value behind forever. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) + VALUES ('admins', $1, $2, $3, $4, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) + .bind(&format_extension) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; @@ -1960,6 +2013,9 @@ async fn process_notify_event( SERVICE_LOG_RETENTION_SECS_SETTING => { reload_service_log_retention_secs_setting(conn).await } + OTEL_TRACES_RETENTION_SECS_SETTING => { + reload_otel_traces_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ce09cebe71..86e75b7de9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -64,10 +64,11 @@ use windmill_common::{ JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, @@ -97,10 +98,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS, - JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, - METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, - OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, + DEFAULT_OTEL_TRACES_RETENTION_SECS, DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, + JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, + METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, + OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -514,6 +515,15 @@ pub async fn initial_load( Ordering::Relaxed, ) }); + pass.setting(OTEL_TRACES_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_otel_traces_retention_secs(parse_setting_value::( + v, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + )) + }); pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { STORE_AUDIT_LOGS_S3.store( parse_setting_value::( @@ -1682,6 +1692,57 @@ const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; /// across ticks rather than inside one, the way the neighbouring sweeps already do. const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; +/// One span per HTTP request made from a job script, so the table grows far faster than the +/// job table it is keyed against; batched for the same reason the service log sweep is. +const OTEL_TRACES_DELETE_BATCH: i64 = 10_000; +const OTEL_TRACES_DELETE_MAX_BATCHES: usize = 10; + +/// Delete HTTP request tracing spans older than `retention_secs`, returning how many went. +/// +/// `retention_secs` is a parameter rather than a read of the process-wide setting so a test can +/// pin a window without writing state the other tests in this binary run against concurrently. +async fn delete_expired_otel_traces(db: &DB, retention_secs: i64) -> u64 { + // `start_time_unix_nano` is the proto field stored verbatim, so the cutoff is built in that + // unit rather than compared against `now()`. Truncating the epoch to whole seconds first + // keeps the multiplication inside `bigint`. + // + // Batched on `ctid`, not on the `(trace_id, span_id)` primary key: with the key the planner + // hashes the LIMITed subquery and Seq Scans the whole table to probe it, which at the size + // this table reaches is the cost the batching exists to avoid. `ctid` plans as a Tid Scan, so + // each batch touches only the rows it deletes. Safe because the subquery and the delete share + // one snapshot, and spans are never updated after insert. + let mut deleted = 0; + for _ in 0..OTEL_TRACES_DELETE_MAX_BATCHES { + let batch = sqlx::query!( + "DELETE FROM otel_traces WHERE ctid IN ( + SELECT ctid FROM otel_traces + WHERE start_time_unix_nano < EXTRACT( + EPOCH FROM now() - ($1::bigint::text || ' s')::interval + )::bigint * 1000000000 + LIMIT $2 + )", + retention_secs, + OTEL_TRACES_DELETE_BATCH, + ) + .execute(db) + .await; + + match batch { + Ok(res) => { + deleted += res.rows_affected(); + if (res.rows_affected() as i64) < OTEL_TRACES_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting expired otel trace spans: {:?}", e); + break; + } + } + } + deleted +} + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1808,6 +1869,12 @@ pub async fn delete_expired_items(db: &DB) -> () { } } + let deleted_spans = + delete_expired_otel_traces(db, windmill_common::otel_traces_retention_secs()).await; + if deleted_spans > 0 { + tracing::info!("deleted {} expired otel trace spans", deleted_spans); + } + let audit_retention_days = audit_log_retention_days().await; let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; @@ -2927,6 +2994,21 @@ pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { } } +pub async fn reload_otel_traces_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_otel_traces_retention_secs(v), + Err(e) => tracing::error!("Error reloading otel traces retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, @@ -4839,7 +4921,7 @@ async fn poll_git_fork_branches( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces") .execute(db) .await?; Ok(()) @@ -6975,6 +7057,46 @@ mod zombie_worker_memory_pct_tests { } } +#[cfg(test)] +mod otel_traces_retention_tests { + use super::{delete_expired_otel_traces, DB}; + + async fn insert_span(db: &DB, id: u8, age_secs: i64) { + sqlx::query!( + "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano) + VALUES ($1, $2, 'GET /', 3, $3, $3)", + &[id; 16][..], + &[id; 8][..], + (chrono::Utc::now() - chrono::Duration::seconds(age_secs)) + .timestamp_nanos_opt() + .unwrap(), + ) + .execute(db) + .await + .unwrap(); + } + + /// The cutoff crosses two units: a retention configured in seconds against a column holding + /// nanoseconds. Getting that conversion wrong is silent in both directions — a window a + /// billion times too wide never deletes anything, one a billion times too narrow deletes + /// every span on the next tick — so pin it on either side of the boundary. + #[sqlx::test(migrations = "./migrations")] + async fn deletes_only_spans_past_the_window(db: DB) -> anyhow::Result<()> { + let day = 60 * 60 * 24; + insert_span(&db, 1, 60).await; + insert_span(&db, 2, 6 * day).await; + insert_span(&db, 3, 8 * day).await; + + assert_eq!(delete_expired_otel_traces(&db, 7 * day).await, 1); + + let kept = sqlx::query_scalar!("SELECT trace_id FROM otel_traces ORDER BY trace_id") + .fetch_all(&db) + .await?; + assert_eq!(kept, vec![vec![1u8; 16], vec![2u8; 16]]); + Ok(()) + } +} + #[cfg(test)] mod log_file_listing_tests { use super::{rotated_log_files, sorted_log_files}; diff --git a/backend/tests/fixtures/inline_preview_auth.sql b/backend/tests/fixtures/inline_preview_auth.sql index 59fe2dc917..67ca598bda 100644 --- a/backend/tests/fixtures/inline_preview_auth.sql +++ b/backend/tests/fixtures/inline_preview_auth.sql @@ -2,7 +2,9 @@ -- Layered on top of `base` (which provides test-workspace and the non-operator -- `test-user-2`/SECRET_TOKEN_2). Adds an Operator member so we can assert that -- Operators cannot reach the arbitrary-code inline preview path --- (`POST /jobs/run_inline/preview`). +-- (`POST /jobs/run_inline/preview`) with their own token, plus two deployed script +-- jobs of the operator: one running, so we can assert that its WM_TOKEN can, and +-- one queued but not yet pulled, so we can assert that "queued" is not enough. INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) VALUES ('operator@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Operator User'); @@ -12,3 +14,11 @@ INSERT INTO usr(workspace_id, email, username, is_admin, operator, role) VALUES INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('OPERATOR_TOKEN'::bytea), 'hex'), 'OPERATOR_T', 'OPERATOR_TOKEN', 'operator@windmill.dev', 'operator token', false); + +INSERT INTO v2_job(id, workspace_id, kind, runnable_path, created_by, permissioned_as, permissioned_as_email) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'); + +INSERT INTO v2_job_queue(id, workspace_id, scheduled_for, running) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', now(), true), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', now(), false); diff --git a/backend/tests/inline_preview_auth.rs b/backend/tests/inline_preview_auth.rs index 97b70fb545..8841719962 100644 --- a/backend/tests/inline_preview_auth.rs +++ b/backend/tests/inline_preview_auth.rs @@ -9,16 +9,33 @@ //! was the incomplete-fix residual of CVE-2026-22683, whose v1.615.0 patch only //! covered the entity-CRUD endpoints and left this direct inline-exec sink open. //! +//! The guard on both routes has one exemption: `wmill.datatable()` called from +//! inside a job the operator is running. Operators can only run deployed code, +//! so a request the job's WM_TOKEN authenticates comes from code a non-operator +//! authored, and the exemption is limited to the request shape the helper sends +//! (PostgreSQL against a `datatable://` database) so a leaked WM_TOKEN cannot +//! be replayed to run anything else. +//! //! This test pins down: -//! - an Operator is rejected by the operator guard (the core fix; pre-fix this -//! reached the inline executor instead of returning 401), and +//! - an Operator's own token is rejected by the operator guard (the core fix; +//! pre-fix this reached the inline executor instead of returning 401), //! - a regular non-operator passes the guard (the fix must not over-block the //! legitimate inline preview flow): in the test harness the worker inline //! utils are not registered, so a caller past the guard gets the distinct -//! "worker inline functions" error rather than the operator rejection. +//! "worker inline functions" error rather than the operator rejection, +//! - an Operator's job token passes the guard for a datatable query while its +//! job is running, on the inline route and on the `/jobs/run/preview` +//! fallback the SDKs use when the worker has no internal server, +//! - the same token is rejected for any other payload (in-process DuckDB, or a +//! `-- database` directive redirecting the query, whether written literally or +//! reached through a `WM_INTERNAL_DB` marker) and for a deferred run, +//! - an Operator's job token for a job that is not running, whether finished or +//! merely queued, is rejected. use serde_json::json; use sqlx::{Pool, Postgres}; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::Authed; use windmill_test_utils::*; fn client() -> reqwest::Client { @@ -38,11 +55,65 @@ fn inline_preview_body() -> serde_json::Value { }) } +/// The request `wmill.datatable("main")` sends: PostgreSQL against `datatable://main`. +fn datatable_query_body() -> serde_json::Value { + json!({ + "language": "postgresql", + "content": "SELECT 1 AS x;", + "args": { "database": "datatable://main" } + }) +} + +/// Mint the WM_TOKEN a job hands its own code: an internally-signed job JWT +/// (note the `job_id` claim) for the fixture's operator, exactly as the worker +/// issues it when the operator runs a deployed script. +async fn operator_job_token(job_id: uuid::Uuid) -> String { + let authed = Authed { + email: "operator@windmill.dev".to_string(), + username: "operator-user".to_string(), + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes: None, + token_prefix: None, + }; + create_jwt_token( + authed, + "test-workspace", + 3600, + Some(job_id), + Some("ephemeral-script".to_string()), + None, + None, + ) + .await + .expect("mint operator job token") +} + const OPERATOR_GUARD_MSG: &str = "Operators cannot run preview jobs"; +/// The fixture's deployed-script jobs of the operator: one running, one queued. +const RUNNING_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000001"; +const QUEUED_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000002"; + +async fn post(url: &str, token: &str, body: &serde_json::Value) -> (u16, String) { + let resp = authed(client().post(url), token) + .json(body) + .send() + .await + .expect("request"); + let status = resp.status().as_u16(); + let body = resp.text().await.expect("body"); + (status, body) +} + #[sqlx::test(fixtures("base", "inline_preview_auth"))] async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; + // The server decodes WM_TOKENs with the same in-process JWT secret, so + // setting it once lets us mint valid ones below. + set_jwt_secret().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -51,12 +122,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // 1. CORE REGRESSION: an Operator must be rejected by the operator guard. // Pre-fix this fell through to the inline executor (arbitrary code // execution); post-fix it returns 401 with the operator guard message. - let resp = authed(client().post(&url), "OPERATOR_TOKEN") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "OPERATOR_TOKEN", &inline_preview_body()).await; assert_eq!( status, 401, "Operator must be rejected from inline preview (got {status}): {body}" @@ -71,12 +137,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // the worker inline utils, so the request proceeds past the guard and // fails later with the distinct "worker inline functions" error — proving // the operator guard did not reject it. - let resp = authed(client().post(&url), "SECRET_TOKEN_2") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "SECRET_TOKEN_2", &inline_preview_body()).await; assert_ne!( status, 401, "non-operator must not be blocked by the operator guard (got {status}): {body}" @@ -86,5 +147,113 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result "non-operator must not hit the operator guard, got: {body}" ); + // 3. The WM_TOKEN of a deployed-script job the Operator is running passes the + // guard for a datatable query: this is `wmill.datatable()` called from + // inside that job. As in 2, the harness then fails with the "worker inline + // functions" error. + let running_job_token = + operator_job_token(uuid::Uuid::parse_str(RUNNING_JOB_ID).unwrap()).await; + let (status, body) = post(&url, &running_job_token, &datatable_query_body()).await; + assert_ne!( + status, 401, + "operator job token of a running job must pass the guard for a datatable query (got {status}): {body}" + ); + assert!( + !body.contains(OPERATOR_GUARD_MSG), + "operator job token of a running job must not hit the operator guard, got: {body}" + ); + + // 4. The same token is rejected for any other payload: the exemption covers + // the datatable request shape only, never in-process DuckDB, and never a + // `-- database` directive, which the executor honors over `args.database`. + let mut redirected = datatable_query_body(); + redirected["content"] = json!("-- database u/test-user/other_db\nSELECT 1 AS x;"); + let mut to_s3 = datatable_query_body(); + to_s3["content"] = json!("-- s3\nSELECT 1 AS x;"); + let mut resource_db = datatable_query_body(); + resource_db["args"]["database"] = json!("$res:u/test-user/other_db"); + // A marker is a single line the directive regexes cannot match; the directive only + // appears once the executor expands it, so the guard must check the expansion. + let mut marker = datatable_query_body(); + marker["content"] = json!(concat!( + r#"-- WM_INTERNAL_DB_SELECT {"table":"t","columnDefs":[{"field":"id","datatype":"int4"}],"#, + r#""whereClause":"true\n-- database u/test-user/other_db\n AND true"}"# + )); + for (label, payload) in [ + ("DuckDB", inline_preview_body()), + ("database directive", redirected), + ("s3 directive", to_s3), + ("resource database", resource_db), + ("marker-expanded database directive", marker), + ] { + let (status, body) = post(&url, &running_job_token, &payload).await; + assert_eq!( + status, 401, + "operator job token must be rejected for a {label} payload (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} payload must be the operator guard, got: {body}" + ); + } + + // 5. An Operator's job token whose job is not running is rejected like the + // operator's own token, whether the job is over (no queue row) or merely + // queued: a WM_TOKEN that leaked through logs cannot be replayed once the + // job is over. + for (label, job_id) in [ + ("finished", uuid::Uuid::new_v4()), + ("queued", uuid::Uuid::parse_str(QUEUED_JOB_ID).unwrap()), + ] { + let token = operator_job_token(job_id).await; + let (status, body) = post(&url, &token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token of a {label} job must be rejected (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} job must be the operator guard, got: {body}" + ); + } + + // 6. The SDKs fall back to `/jobs/run/preview` when the worker has no internal + // server (agent workers). The same exemption applies there: the running + // job's token queues the datatable query (201 with the job id), the + // operator's own token is still refused. + let fallback_url = format!("http://localhost:{port}/api/w/test-workspace/jobs/run/preview"); + let (status, body) = post(&fallback_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 201, + "operator job token of a running job must queue a datatable preview (got {status}): {body}" + ); + let (status, body) = post(&fallback_url, "OPERATOR_TOKEN", &datatable_query_body()).await; + assert_eq!( + status, 401, + "Operator must be rejected from the preview fallback (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection must be the operator guard, got: {body}" + ); + + // 7. A deferred run on the fallback would outlive the running job the + // exemption keys off, so the running job's token cannot schedule one. + for deferral in [ + "scheduled_in_secs=86400", + "scheduled_for=2099-01-01T00:00:00Z", + ] { + let deferred_url = format!("{fallback_url}?{deferral}"); + let (status, body) = post(&deferred_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token must not schedule a deferred preview with {deferral} (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for {deferral} must be the operator guard, got: {body}" + ); + } + Ok(()) } diff --git a/backend/tests/object_storage_test_ssrf.rs b/backend/tests/object_storage_test_ssrf.rs new file mode 100644 index 0000000000..aa9eaaf74c --- /dev/null +++ b/backend/tests/object_storage_test_ssrf.rs @@ -0,0 +1,97 @@ +//! `POST /api/settings/test_object_storage_config` runs the probe on the API server and reflects the +//! upstream response, so every non-super-admin must be rejected for private/loopback endpoints and +//! the Filesystem backend on every deployment (`CLOUD_HOSTED` is unset here), while a super admin's +//! Filesystem probe still round-trips. Requires the `parquet` feature, like the route. +#![cfg(feature = "parquet")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use windmill_test_utils::*; + +const SUPER_ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const USER_TOKEN: &str = "SECRET_TOKEN_2"; + +async fn test_object_storage( + url: &str, + token: &str, + body: serde_json::Value, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(url) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn object_storage_test_is_restricted_for_non_super_admins_off_cloud( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/settings/test_object_storage_config", + server.addr.port() + ); + + // A loopback "S3 endpoint" standing in for an internal service: the probe must be rejected + // before the server opens a connection to it. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let internal_port = listener.local_addr()?.port(); + let connected = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let connected = connected.clone(); + async move { + while listener.accept().await.is_ok() { + connected.store(true, Ordering::SeqCst); + } + } + }); + let internal_s3 = json!({ + "type": "S3", + "bucket": "bucket", + "region": "us-east-1", + "access_key": "key", + "secret_key": "secret", + "endpoint": format!("http://127.0.0.1:{internal_port}"), + "allow_http": true, + "path_style": true, + }); + let (status, body) = test_object_storage(&url, USER_TOKEN, internal_s3).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a loopback endpoint (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + assert!( + !connected.load(Ordering::SeqCst), + "the server must not connect to the rejected endpoint" + ); + + let tmp = tempfile::tempdir()?; + let filesystem = json!({ "type": "Filesystem", "root_path": tmp.path().to_str().unwrap() }); + let (status, body) = test_object_storage(&url, USER_TOKEN, filesystem.clone()).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a Filesystem backend (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + + // Super admins keep the unrestricted path. + let (status, body) = test_object_storage(&url, SUPER_ADMIN_TOKEN, filesystem).await?; + assert_eq!( + status, 200, + "super admin must be able to test a Filesystem backend (got {status}): {body}" + ); + Ok(()) +} diff --git a/backend/tests/relock_noop.rs b/backend/tests/relock_noop.rs new file mode 100644 index 0000000000..5cb8e3f51e --- /dev/null +++ b/backend/tests/relock_noop.rs @@ -0,0 +1,415 @@ +use sqlx::{Pool, Postgres}; +use tokio_stream::StreamExt; +use windmill_api_client::types::NewScript; +use windmill_common::scripts::{deploy_relocked_version, fetch_script_for_update}; +use windmill_test_utils::*; + +const W: &str = "test-workspace"; + +const A: &str = r#"export async function main() { return "a" }"#; +const A_COMMENTED: &str = r#"// same dependencies, different content +export async function main() { return "a" }"#; +const A_WITH_LODASH: &str = r#"import _ from "lodash@4.17.21"; +export async function main() { return _.trim(" a ") }"#; +const B: &str = r#"import { main as a } from "/f/rel/a.ts"; +export async function main() { return "b" + (await a()) }"#; +const C: &str = r#"import { main as b } from "/f/rel/b.ts"; +export async function main() { return "c" + (await b()) }"#; + +fn bun_script(path: &str, content: &str, parent_hash: Option) -> NewScript { + NewScript { + draft_only: None, + content: content.into(), + language: windmill_api_client::types::ScriptLang::Bun, + lock: None, + parent_hash, + path: path.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_secs: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + auto_kind: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + modules: None, + } +} + +#[derive(sqlx::FromRow, Debug)] +struct Version { + hash: i64, + archived: bool, + lock: Option, + created_at: chrono::DateTime, +} + +/// Every version of `path`, oldest first. +async fn versions(db: &Pool, path: &str) -> Vec { + sqlx::query_as( + "SELECT hash, archived, lock, created_at FROM script + WHERE workspace_id = $1 AND path = $2 ORDER BY created_at", + ) + .bind(W) + .bind(path) + .fetch_all(db) + .await + .unwrap() +} + +fn live(versions: &[Version]) -> &Version { + versions.iter().rev().find(|v| !v.archived).unwrap() +} + +/// `(path, status, logs)` of every dependency job created after `since`, in completion order. +async fn dependency_jobs_since( + db: &Pool, + since: chrono::DateTime, +) -> Vec<(String, String, String)> { + sqlx::query_as( + "SELECT j.runnable_path, c.status::text, COALESCE(l.logs, '') FROM v2_job_completed c + JOIN v2_job j ON j.id = c.id + LEFT JOIN job_logs l ON l.job_id = c.id + WHERE j.kind = 'dependencies' AND j.created_at > $1 + ORDER BY c.started_at", + ) + .bind(since) + .fetch_all(db) + .await + .unwrap() +} + +async fn wait_for_jobs( + completed: &mut (impl futures::Stream + Unpin), + count: usize, +) { + for _ in 0..count { + completed.next().await; + } + // Then let anything else that was queued run out, so a job the assertions say must not + // exist would have shown up here. + while let Ok(Some(_)) = + tokio::time::timeout(std::time::Duration::from_secs(2), completed.next()).await + {} +} + +/// A redeploy of an imported script whose dependencies did not move relocks its importer, +/// and that relock must deploy nothing: no new version, and no dependency job for the +/// importer's own importers. A redeploy that does change the dependencies still walks the +/// whole chain with a new version at each step. +#[sqlx::test(fixtures("base"))] +async fn relative_import_relock_deploys_only_when_the_lock_changed( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + // One at a time: each deploy's dependency job records the importer's edges, and an + // importer whose edges are recorded is what a later relock of it can skip on. + for (path, content) in [("f/rel/a", A), ("f/rel/b", B), ("f/rel/c", C)] { + client + .create_script(W, &bun_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + } + let b_before = versions(&db, "f/rel/b").await; + let c_before = versions(&db, "f/rel/c").await; + assert_eq!(b_before.len(), 1); + assert_eq!(c_before.len(), 1); + + // Content-only change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &bun_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!( + paths, + ["f/rel/a", "f/rel/b"], + "the leaf's own job and one no-op relock of its importer, and nothing for c" + ); + assert!( + jobs[1] + .2 + .contains("Lock unchanged: no new version deployed"), + "b's relock should have found its lock unchanged: {}", + jobs[1].2 + ); + let b_after = versions(&db, "f/rel/b").await; + let c_after = versions(&db, "f/rel/c").await; + assert_eq!( + b_after.len(), + 1, + "an unchanged relock must not mint a version" + ); + assert_eq!(live(&b_after).hash, live(&b_before).hash); + assert_eq!(c_after.len(), 1); + assert_eq!(live(&c_after).hash, live(&c_before).hash); + + // A dependency change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &bun_script("f/rel/a", A_WITH_LODASH, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/c"]); + for path in ["f/rel/b", "f/rel/c"] { + let vs = versions(&db, path).await; + assert_eq!( + vs.len(), + 2, + "{path}: a changed relock deploys a new version" + ); + assert!( + vs[0].archived && !vs[1].archived, + "{path}: parent archived, child live" + ); + assert!(vs[0].created_at < vs[1].created_at, "{path}: lineage order"); + assert!( + vs[1].lock.as_deref().unwrap_or("").contains("lodash"), + "{path}: the new version carries the new lock: {:?}", + vs[1].lock + ); + } + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock that has to wait on its head's row lock, because a deploy of the same path holds +/// it, must find the version that deploy left and requeue itself for it rather than fail. The +/// blocked statement re-checks only the row it selected, which the deploy archived, and comes +/// back empty; the successor is only visible to a fresh read. +#[sqlx::test(fixtures("base"))] +async fn relock_waiting_on_a_deploy_requeues_for_its_successor( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + for (path, content) in [("f/rel/a", A), ("f/rel/b", B)] { + client + .create_script(W, &bun_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + } + + // A deploy of b that holds its head's row lock for as long as this transaction lives. + let mut deploy = db.begin().await.unwrap(); + let head = fetch_script_for_update("f/rel/b", W, &mut *deploy) + .await + .unwrap() + .unwrap(); + + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &bun_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + + // b's relock skips generation and reaches its commit, where it waits on the lock. + let mut waiting = false; + for _ in 0..300 { + waiting = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' + AND query LIKE '%FROM script WHERE path = $1%FOR UPDATE%')", + ) + .fetch_one(&db) + .await + .unwrap(); + if waiting { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!(waiting, "b's relock never reached the row lock"); + + // The deploy lands: the head is archived and a successor with its own lock takes + // its place, while the relock is still waiting. + let lock = head.lock.clone().unwrap(); + let successor = + deploy_relocked_version(&mut deploy, head, None, Some(&lock), None, None) + .await + .unwrap(); + deploy.commit().await.unwrap(); + + // a's own job, the relock that waited, and the relock it queued for the successor. + wait_for_jobs(&mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/b"], "{jobs:?}"); + assert!( + jobs.iter().all(|(_, status, _)| status == "success"), + "no relock may fail on the wait: {jobs:?}" + ); + assert!( + jobs[1] + .2 + .contains("was deployed while this lock was generated"), + "the waiting relock should have seen the successor: {}", + jobs[1].2 + ); + assert!( + jobs[2] + .2 + .contains("Lock unchanged: no new version deployed"), + "the requeued relock should find the successor's lock current: {}", + jobs[2].2 + ); + let vs = versions(&db, "f/rel/b").await; + assert_eq!( + vs.len(), + 2, + "the deploy's successor and nothing else: {vs:?}" + ); + assert_eq!(live(&vs).hash, successor); + }, + port, + ) + .await; + + Ok(()) +} + +/// A multi-file importer: on a skipped relock each module gets its own last lock back, not the +/// parent script's, so an import's content-only redeploy leaves the importer alone as well. +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base"))] +async fn multi_file_importer_relock_is_a_no_op_too(db: Pool) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + let py = |path: &str, content: &str, parent_hash: Option, with_module: bool| { + let mut ns = bun_script(path, content, parent_hash); + ns.language = windmill_api_client::types::ScriptLang::Python3; + if with_module { + ns.modules = Some(std::collections::HashMap::from([( + "helper.py".to_string(), + serde_json::json!({ + "content": "def greet(x):\n return 'hi ' + x\n", + "language": "python3" + }), + )])); + } + ns + }; + async fn module_lock(db: &Pool) -> Option { + sqlx::query_scalar( + "SELECT modules->'helper.py'->>'lock' FROM script + WHERE workspace_id = $1 AND path = 'f/rel/pb' AND archived = false", + ) + .bind(W) + .fetch_one(db) + .await + .unwrap() + } + + in_test_worker( + &db, + async { + client + .create_script(W, &py("f/rel/pa", "def main():\n return 'a'\n", None, false)) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + client + .create_script( + W, + &py( + "f/rel/pb", + "from f.rel.pa import main as a\nfrom .helper import greet\n\ndef main():\n return greet(a())\n", + None, + true, + ), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 1).await; + let lock_before = module_lock(&db).await; + assert!(lock_before.is_some(), "the module got a lock of its own on deploy"); + + let since = chrono::Utc::now(); + let pa_hash = live(&versions(&db, "f/rel/pa").await).hash; + client + .create_script( + W, + &py( + "f/rel/pa", + "# same dependencies\ndef main():\n return 'a'\n", + Some(format!("{pa_hash:016x}")), + false, + ), + ) + .await + .unwrap(); + wait_for_jobs(&mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/pa", "f/rel/pb"], "{jobs:?}"); + assert!( + jobs[1].2.contains("Lock unchanged: no new version deployed"), + "the multi-file importer's relock should be a no-op: {}", + jobs[1].2 + ); + assert_eq!(versions(&db, "f/rel/pb").await.len(), 1); + assert_eq!(module_lock(&db).await, lock_before, "the module keeps its own lock"); + }, + port, + ) + .await; + + Ok(()) +} diff --git a/backend/tests/relock_skip.rs b/backend/tests/relock_skip.rs index 8262a38cf4..bf5cb24cbf 100644 --- a/backend/tests/relock_skip.rs +++ b/backend/tests/relock_skip.rs @@ -266,7 +266,10 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + // Empty content leaves every importer's lock as it was, so only the five direct + // importers of the default deps run a job: an unchanged script relock deploys no + // version and so queues nothing for its own importers. + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; // Note: within a cascade, the same script may be triggered multiple times. // After the first trigger relocks and stores the hash, subsequent triggers skip. @@ -295,7 +298,7 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; assert!( diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index d99d955de9..05de1183cd 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -288,7 +288,6 @@ pub enum ScopeDomain { Configs, OAuth, AI, - AiSkills, AiEvals, // AI agent eval datasets Indexer, @@ -349,7 +348,6 @@ impl ScopeDomain { Self::Configs => "configs", Self::OAuth => "oauth", Self::AI => "ai", - Self::AiSkills => "ai_skills", Self::AiEvals => "ai_evals", Self::Capture => "capture", Self::Drafts => "drafts", @@ -405,7 +403,6 @@ impl ScopeDomain { "configs" => Some(Self::Configs), "oauth" => Some(Self::OAuth), "ai" => Some(Self::AI), - "ai_skills" => Some(Self::AiSkills), "ai_evals" => Some(Self::AiEvals), "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), @@ -1202,12 +1199,6 @@ mod tests { assert_eq!(domain, ScopeDomain::FlowConversations); assert_eq!(kind, None); assert_eq!(route_suffix, Some("flow_conversations/list".to_string())); - - let (domain, kind, route_suffix) = - extract_domain_from_route("/api/w/test_workspace/ai_skills/list").unwrap(); - assert_eq!(domain, ScopeDomain::AiSkills); - assert_eq!(kind, None); - assert_eq!(route_suffix, Some("ai_skills/list".to_string())); } #[test] @@ -1368,11 +1359,6 @@ mod tests { ScopeDomain::from_str("flow_conversations"), Some(ScopeDomain::FlowConversations) ); - assert_eq!( - ScopeDomain::from_str("ai_skills"), - Some(ScopeDomain::AiSkills) - ); - // Test canonical string conversion assert_eq!(ScopeDomain::Acls.as_str(), "acls"); assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); @@ -1381,41 +1367,6 @@ mod tests { ScopeDomain::FlowConversations.as_str(), "flow_conversations" ); - assert_eq!(ScopeDomain::AiSkills.as_str(), "ai_skills"); - } - - #[test] - fn test_ai_skills_scope_access() { - let read_scopes = vec!["ai_skills:read".to_string()]; - assert!( - check_route_access(&read_scopes, "/api/w/test_workspace/ai_skills/list", "GET").is_ok() - ); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/get/foo", - "GET" - ) - .is_ok()); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_err()); - - let write_scopes = vec!["ai_skills:write".to_string()]; - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_ok()); - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/delete/foo", - "DELETE" - ) - .is_ok()); } #[test] diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 6353cd429a..ecb7ba405b 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -157,7 +157,8 @@ async fn list_flows( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 5a7af05cbe..c6f88dace5 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -183,9 +183,9 @@ async fn add_granular_acl( if kind == "folder" { let change_type = if write.unwrap_or(false) { - "grant_read" - } else { "grant_write" + } else { + "grant_read" }; crate::folders::log_folder_permission_change( &mut *tx, diff --git a/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs new file mode 100644 index 0000000000..3e1035ff82 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs @@ -0,0 +1,186 @@ +//! Request headers reaching a runnable's preprocessor over MCP. +//! +//! The property this pins is structural rather than a filter: the model writes +//! the tool's arguments, which become `event.body`, while the server writes +//! `event.headers`. A model that guesses a header's name can only ever land in +//! `body`, so an identity read from `headers` is one prompt injection cannot +//! forge. Nothing else in the suite exercises MCP argument shaping end to end. +//! +//! Requires: bun runtime, live database (migrations applied by sqlx::test). +#![cfg(feature = "mcp")] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const SCRIPT_PATH: &str = "u/test-user/mcp_hdr_probe"; + +/// Echoes the two halves of the event separately, so the assertions can tell +/// which one a value arrived in. +const PREPROCESSOR_SCRIPT: &str = r#" +export async function preprocessor(event: any) { + return { + kind: event.kind, + from_headers: event.headers?.["x-user-id"] ?? "", + from_body: event.body?.x_user_id ?? "", + header_names: Object.keys(event.headers ?? {}).sort(), + }; +} + +export async function main(kind: string, from_headers: string, from_body: string, header_names: string[]) { + return { kind, from_headers, from_body, header_names }; +} +"#; + +async fn insert_mcp_token(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('MCP_TOKEN'::bytea), 'hex'), 'MCP_TOK', 'MCP_TOKEN', 'test@windmill.dev', 'mcp token', true, ARRAY['mcp:all'])", + ) + .execute(db) + .await?; + Ok(()) +} + +/// POST one JSON-RPC message. The endpoint answers either `application/json` or +/// a single-event SSE stream, so strip the `data: ` framing before parsing. +async fn mcp_post(port: u16, headers: &[(&str, &str)], body: Value) -> anyhow::Result { + let mut req = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/mcp/w/test-workspace/mcp" + )) + .header("Authorization", "Bearer MCP_TOKEN") + .header("Accept", "application/json, text/event-stream") + .json(&body); + for (k, v) in headers { + req = req.header(*k, *v); + } + let text = req.send().await?.text().await?; + let payload = text + .lines() + .find_map(|l| l.strip_prefix("data: ")) + .unwrap_or(text.trim()); + serde_json::from_str(payload).map_err(|e| anyhow::anyhow!("unparseable MCP body {text:?}: {e}")) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_preprocessor_receives_the_callers_headers( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + insert_mcp_token(&db).await?; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": SCRIPT_PATH, + "summary": "mcp header probe", + "description": "", + "content": PREPROCESSOR_SCRIPT, + "language": "bun", + "lock": "", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { "x_user_id": { "type": "string" } }, + "required": [] + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 201, + "create script: {}", + resp.text().await.unwrap_or_default() + ); + + // A script counts as deployed once it has a lock, which normally arrives from + // a dependency job. Planting an empty one keeps the test to the path under + // test instead of a bun resolution whose timing it does not control. + sqlx::query("UPDATE script SET lock = '' WHERE path = $1 AND workspace_id = 'test-workspace'") + .bind(SCRIPT_PATH) + .execute(&db) + .await?; + + let tools = mcp_post( + port, + &[], + json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}), + ) + .await?; + let tool_name = tools["result"]["tools"] + .as_array() + .and_then(|list| { + list.iter() + .filter_map(|t| t["name"].as_str()) + .find(|n| n.contains("mcp__hdr__probe")) + }) + .ok_or_else(|| anyhow::anyhow!("the deployed script was not listed as a tool: {tools}"))? + .to_string(); + + let result = in_test_worker( + db.clone(), + async { + mcp_post( + port, + // Every name the withheld list covers has to be on the wire, or + // asserting its absence proves nothing. `Authorization` is already + // set by `mcp_post`, and `extract_token` reads it before the + // cookie, so sending one does not disturb auth. + &[ + ("X-User-Id", "alice@corp.example"), + ("Cookie", "session=secret"), + ("Proxy-Authorization", "Basic Zm9v"), + ], + json!({ + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + // The model names the header it wants to spoof. Its value is an + // argument, so it can only ever reach `event.body`. + "params": { "name": tool_name, "arguments": { "x_user_id": "attacker@evil.test" } } + }), + ) + .await + }, + port, + ) + .await?; + + let text = result["result"]["content"][0]["text"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("tool call returned no text content: {result}"))?; + let out: Value = serde_json::from_str(text)?; + + assert_eq!(out["kind"], "mcp", "preprocessor event kind: {out}"); + assert_eq!( + out["from_headers"], "alice@corp.example", + "the caller's header must reach event.headers: {out}" + ); + assert_eq!( + out["from_body"], "attacker@evil.test", + "the model's argument must land in event.body, not overwrite the header: {out}" + ); + + let names: Vec<&str> = out["header_names"] + .as_array() + .map(|a| a.iter().filter_map(|v| v.as_str()).collect()) + .unwrap_or_default(); + assert!( + names.contains(&"x-user-id"), + "event.headers must carry the request's own headers: {names:?}" + ); + for withheld in ["authorization", "cookie", "proxy-authorization"] { + assert!( + !names.contains(&withheld), + "{withheld} is withheld from a preprocessor: {names:?}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index 8c88c53454..3a146add27 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -38,6 +38,85 @@ fn new_script(path: &str, summary: &str, content: &str) -> serde_json::Value { }) } +/// A supplied lock queues no dependency job, so if the create does not record its hash nothing +/// ever will, and every importer of this script relocks on each of its deploys forever after. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_script_persists_supplied_lock_hash(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/supplied_lock"; + let lock = r#"{"version":"4","remote":{}}"#; + let mut script = new_script( + path, + "Supplied lock", + "export async function main() { return 42; }", + ); + script["lock"] = json!(lock); + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + let stored_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(stored_hash, windmill_common::scripts::hash_script(lock)); + + // A script deployed before the create recorded hashes has no row, and pushing it unchanged + // creates no version to hang one off. Without the write on that path it would keep its + // importers relocking until someone edited it. + sqlx::query!( + "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .execute(&db) + .await?; + + // The no-op comparison covers every field, so the push has to carry what the first deploy + // filled in by itself; `auto_parent` both resolves the parent and keeps the hash distinct. + script["auto_parent"] = json!(true); + script["ws_error_handler_muted"] = json!(false); + script["assets"] = json!([]); + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create?skip_if_noop=true" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "no-op push: {}", resp.text().await?); + + let versions: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await? + .unwrap_or_default(); + assert_eq!(versions, 1, "no-op push must not create a version"); + + let repaired_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(repaired_hash, windmill_common::scripts::hash_script(lock)); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -797,10 +876,12 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( // What a deploy leaves behind: the old head archived, a new one live at the path. // Copied through a temp table so this does not have to restate every column. - sqlx::query("CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1") - .bind(head) - .execute(&mut *winner) - .await?; + sqlx::query( + "CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1", + ) + .bind(head) + .execute(&mut *winner) + .await?; sqlx::query("UPDATE superseding SET hash = $1, archived = false, parent_hashes = ARRAY[$2]") .bind(head + 1) .bind(head) @@ -818,7 +899,10 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( let resp = tokio::time::timeout(std::time::Duration::from_secs(20), update).await??; let status = resp.status(); let body = resp.text().await?; - assert_eq!(status, 400, "losing the race should not read as success: {body}"); + assert_eq!( + status, 400, + "losing the race should not read as success: {body}" + ); assert!( body.contains("deployed to concurrently"), "the loser must say it was superseded, not that the script is missing: {body}" diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index bec59ac45a..65d44509d7 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -917,3 +917,79 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a Ok(()) } + +/// An address with no `password` row can own a draft, and the account paths carry the delete and +/// rename that no foreign key does any more. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_drafts_follow_their_owner_without_a_fkey(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/users"); + + // The destination of the rename below already holds a draft of the same item — it belongs to + // an accountless principal, so `change_email`'s "address is free" check does not see it. + sqlx::query!( + "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'), + ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')" + ) + .execute(&db) + .await?; + + // A null username is how the legacy workspace-level row is encoded, so an owner nobody can + // name must be absent from the owner circles rather than pose as one. + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list?all_users=true" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let listed = resp.json::().await?; + let ext = listed + .as_array() + .unwrap() + .iter() + .find(|d| d["path"] == "u/ext/s") + .expect("the accountless owner's draft is listed"); + assert_eq!(ext.get("draft_users"), None); + + let resp = authed(client().post(format!("{global_base}/change_email/test2@windmill.dev"))) + .json(&json!({ "new_email": "renamed@windmill.dev" })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "change_email: {}", resp.text().await?); + let moved = sqlx::query!( + "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'" + ) + .fetch_all(&db) + .await?; + assert_eq!( + moved + .iter() + .map(|r| (r.email.as_deref(), r.summary.as_deref())) + .collect::>(), + vec![(Some("renamed@windmill.dev"), Some("moving"))], + "the moving account's draft wins the unique index it now collides on" + ); + + let resp = authed(client().delete(format!("{global_base}/delete/test3@windmill.dev"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete_user: {}", resp.text().await?); + let remaining = sqlx::query_scalar!("SELECT path FROM draft ORDER BY path") + .fetch_all(&db) + .await?; + assert_eq!( + remaining, + vec!["u/ext/s".to_string(), "u/two/s".to_string()], + "the deleted account's draft goes, the accountless owner's stays" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 65fe4241cc..25312cae27 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -889,6 +889,53 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +/// A workspace with no provider of its own is served the instance config, but the +/// `copilot_disabled` flag must still come from the workspace's own row. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_get_copilot_info_keeps_workspace_copilot_disabled_over_instance_fallback( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + sqlx::query("UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2") + .bind(json!({ "copilot_disabled": true })) + .bind("test-workspace") + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind("ai_config") + .bind(json!({ + "providers": { + "openai": { + "resource_path": "u/test-user/openai_instance", + "models": ["gpt-4o-mini"] + } + } + })) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{base}/get_copilot_info"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert_eq!( + settings["providers"]["openai"]["models"][0], "gpt-4o-mini", + "instance providers are still served" + ); + assert_eq!(settings["copilot_disabled"], true); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -941,7 +988,12 @@ async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyh .send() .await .unwrap(); - assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert_eq!( + resp.status(), + 200, + "disable on fork: {}", + resp.text().await? + ); assert!(!stored().await?); Ok(()) @@ -1044,9 +1096,11 @@ async fn test_create_service_account_drops_orphaned_group_memberships( .await?; // Same username, different workspace, and very much alive — must not be touched. - sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')") - .execute(&db) - .await?; + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')", + ) + .execute(&db) + .await?; sqlx::query( "INSERT INTO group_ (workspace_id, name, summary) VALUES ('other-workspace', 'all', 'All users'), diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index fbfd656cdf..e897bf5eaf 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -416,11 +416,31 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result let mut headers = HeaderMap::new(); + // A reverse proxy consumes hop-by-hop headers instead of forwarding them and + // drops every header named by `Connection`, so a script could use one to strip + // the sandbox headers this function adds before they reach the browser. + const HOP_BY_HOP_HEADERS: [&str; 9] = [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ]; + if let Some(windmill_headers) = windmill_headers { for (k, v) in windmill_headers { let k = HeaderName::from_str(k.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header name {k}: {err}")) })?; + if HOP_BY_HOP_HEADERS.contains(&k.as_str()) { + return Err(Error::ExecutionErr(format!( + "windmill_headers cannot set the hop-by-hop header \"{k}\"" + ))); + } let v = HeaderValue::from_str(v.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header value {v}: {err}")) })?; @@ -428,6 +448,22 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result } } + // The script controls the content type and body, and run_wait_result and sync + // HTTP routes are reachable by top-level GET navigation with the session cookie: + // sandbox the document into an opaque origin so HTML can never run with the + // viewer's session. Inserted after `wm_headers` so a script cannot override it. + headers.insert( + http::header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + headers.insert( + http::header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static( + "sandbox allow-scripts allow-forms allow-popups \ + allow-popups-to-escape-sandbox allow-downloads allow-modals", + ), + ); + if let Some(content_type) = windmill_content_type { let serialized_json_result = result_value .map(|val| val.get().to_owned()) @@ -1104,6 +1140,56 @@ mod result_to_response_tests { resp.headers().get(http::header::CONTENT_TYPE).unwrap(), "text/html" ); + assert_sandboxed(resp.headers()); assert_eq!(body_bytes(resp).await, b"

hi

"); } + + fn assert_sandboxed(headers: &HeaderMap) { + assert_eq!( + headers.get(http::header::X_CONTENT_TYPE_OPTIONS).unwrap(), + "nosniff" + ); + let csp = headers + .get(http::header::CONTENT_SECURITY_POLICY) + .expect("content-security-policy") + .to_str() + .unwrap(); + assert!(csp.starts_with("sandbox "), "csp: {csp}"); + assert!(!csp.contains("allow-same-origin"), "csp: {csp}"); + } + + #[tokio::test] + async fn custom_headers_cannot_override_sandbox() { + // wm_headers is script-controlled: a content-type set there replaces the JSON + // one even without wm_content_type, and the sandbox headers must survive an + // attempt to override them. + let resp = result_to_response( + raw( + r#"{"wm_headers":{"content-type":"text/html","content-security-policy":"default-src *","x-content-type-options":"none"},"result":"

hi

"}"#, + ), + true, + ) + .expect("response"); + + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html" + ); + assert_sandboxed(resp.headers()); + } + + #[tokio::test] + async fn hop_by_hop_custom_headers_are_rejected() { + // A proxy drops every header named by `Connection`, which would strip the + // sandbox headers on the way to the browser. + for name in ["connection", "Connection", "transfer-encoding", "upgrade"] { + let res = result_to_response( + raw(&format!( + r#"{{"wm_content_type":"text/html","wm_headers":{{"{name}":"content-security-policy, x-content-type-options"}},"result":"

hi

"}}"# + )), + true, + ); + assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); + } + } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 79c6c99b9b..ccc5815aa7 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,7 +39,7 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::process_relative_imports; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ @@ -216,12 +216,15 @@ async fn list_scripts( // a member of has no `usr` row, so fall back to their instance-derived username // (`password.username`), or their email when derivation is disabled — this keeps the // raw email out of the payload whenever a derived username exists. The genuine - // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL). + // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL), + // which is why an owner that resolves to no name at all — an external JWT's subject + // has neither row — is dropped: None is read as "legacy" downstream. "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() @@ -1070,6 +1073,14 @@ fn modules_eq( } } +/// Recorded for the empty lock a codebase or a language with no lock generation carries as well as +/// for a real one: the worker writes `hash_script("")` in the same situation, and a path going from +/// a real lock to an empty one has to stop matching what its importers recorded, or they wrongly +/// skip rather than merely relock too often. +fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { + [(path.to_string(), hash_script(lock))] +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1337,6 +1348,12 @@ async fn create_script_internal<'c>( parent_hash = %p_hash.0, "Skipping no-op script deploy (identical to parent)" ); + // The version is unchanged, but the row recording its lock's hash may never have + // been written — nothing else writes it for a supplied lock, and a path only ever + // pushed unchanged would otherwise keep its importers relocking forever. + if let Some(lock) = ps.lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } return Ok((p_hash.clone(), tx, None, Vec::new())); } @@ -1884,6 +1901,13 @@ async fn create_script_internal<'c>( .execute(&mut *tx) .await?; + // A lock that is not left to a dependency job queues none, so this is the only place its hash + // can be recorded. `try_skip_relock` treats a missing hash for an imported script as changed, + // so leaving the row out makes every importer of this path relock on every deploy of it. + if let Some(lock) = lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } + // Update ci_test_reference table for test scripts // Delete by both new and old path to handle renames let old_path = parent_hashes_and_perms.as_ref().map(|x| x.p_path.as_str()); diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index cead2f4b27..18a8d52ac0 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -284,15 +284,28 @@ pub async fn test_s3_bucket( use bytes::Bytes; use futures::StreamExt; - // The probe executes on the API server itself. On multi-tenant Cloud that is a shared control - // plane, so we constrain untrusted callers to remove the SSRF / credential-exfiltration / - // local-filesystem surface (see validate_object_storage_test). On self-hosted instances the - // object store usually lives on the local/private network and all authenticated users are - // trusted, so testing there stays unrestricted. Super admins keep the unrestricted path too. + // The probe executes on the API server itself and reflects the upstream response into the + // error, so any authenticated caller could otherwise use it as an SSRF / port-scan primitive + // against the server's network, exfiltrate its ambient credentials, or write to its local + // disk (see validate_object_storage_test). That holds on self-hosted instances as much as on + // Cloud, so only super admins get the unrestricted path. let is_super_admin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; - let restrict = !is_super_admin && *CLOUD_HOSTED; + let restrict = !is_super_admin; if restrict { - validate_object_storage_test(&test_s3_bucket).await?; + validate_object_storage_test(&test_s3_bucket) + .await + .map_err(|e| match e { + // A job token never counts as a super admin (it is capped at workspace admin), so + // a super admin calling this route from a script is told why rather than that + // they lack a privilege they hold. + error::Error::NotAuthorized(msg) if authed.job_id.is_some() => { + error::Error::NotAuthorized(format!( + "{msg} A job token ($WM_TOKEN) is never treated as a super admin; call \ + this route with a user token instead." + )) + } + e => e, + })?; } let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) @@ -355,8 +368,8 @@ pub async fn test_s3_bucket( } } -// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller on -// Cloud. The probe runs on the shared API server, so without these constraints an authenticated +// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller. +// The probe runs on the API server, so without these constraints an authenticated // user could coerce the server into connecting to arbitrary internal endpoints (SSRF), signing // requests with the instance role (credential exfiltration), or reading/writing the server's local // disk (filesystem object store). @@ -366,6 +379,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul opt.as_ref().is_some_and(|s| !s.is_empty()) } + // Every refusal names the way out: the resource usually works in jobs (workers reach the + // endpoint directly), so without it the refusal reads as a broken resource. + const ALTERNATIVE: &str = + "Ask a super admin to run it, or test the resource from a script, which runs on a worker."; + // Reject backends that rely on the server's identity or local filesystem, require explicit // credentials for the rest (so the server never falls back to its own ambient credentials), and // resolve the host the client will actually connect to. We derive the *effective* endpoint here @@ -376,20 +394,25 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul let effective_endpoint: Option = match settings { ObjectSettings::Filesystem(_) => { return Err(error::Error::NotAuthorized( - "Testing a local filesystem object store requires a super admin".to_string(), + "Testing a local filesystem object store requires a super admin: it runs on the \ + Windmill server and reads and writes the server's local disk. Ask a super admin \ + to run it." + .to_string(), )); } ObjectSettings::AwsOidc(_) => { - return Err(error::Error::NotAuthorized( - "Testing OIDC-based object storage requires a super admin".to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing OIDC-based object storage requires a super admin: it runs on the \ + Windmill server with the server's own identity. {ALTERNATIVE}" + ))); } ObjectSettings::S3(s3) => { if !(non_empty(&s3.access_key) && non_empty(&s3.secret_key)) { - return Err(error::Error::NotAuthorized( - "Testing S3 storage without explicit credentials requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing S3 storage without an explicit access key and secret key requires a \ + super admin: it runs on the Windmill server, which would use its own ambient \ + credentials. {ALTERNATIVE}" + ))); } let region = s3 .region @@ -413,10 +436,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul } ObjectSettings::Azure(azure) => { if !non_empty(&azure.access_key) { - return Err(error::Error::NotAuthorized( - "Testing Azure storage without an explicit access key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing Azure storage without an explicit access key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } Some( azure @@ -432,10 +456,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul // otherwise an untrusted caller could probe with the server's identity (the very // SSRF/credential-exfil this function guards against). if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { - return Err(error::Error::NotAuthorized( - "Testing GCS storage without a service account key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing GCS storage without a service account key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } // The service-account-key JSON can override the data-plane URL (`gcs_base_url`) and the // OAuth token endpoint (`token_uri`); the GCS client connects to whatever they point at. @@ -492,10 +517,15 @@ async fn validate_public_endpoint(endpoint: &str) -> error::Result<()> { // attempts (a name resolving to both a public and a private address). for addr in addrs { if is_forbidden_ip(addr.ip()) { - return Err(error::Error::NotAuthorized( - "Testing object storage at a private, loopback, or link-local endpoint requires a super admin" - .to_string(), - )); + // The resolved address stays out of the message: it is the server's resolver's + // answer, and this message is only ever shown to the caller being constrained. + return Err(error::Error::NotAuthorized(format!( + "Testing object storage at '{host}', which resolves to a private, loopback, or \ + link-local address, requires a super admin: this test runs on the Windmill \ + server, which is not allowed to probe internal addresses for non-super-admins. \ + Ask a super admin to run it, or test the resource from a script, which runs on \ + a worker." + ))); } } Ok(()) @@ -2004,6 +2034,15 @@ struct CachedResourceType { #[allow(dead_code)] app: String, description: Option, + /// Doubly optional, and read through a wrapping deserializer: this struct also + /// decodes the on-disk cache, where an absent key means "written before the + /// column, leave the stored extension alone" and an explicit null means the hub + /// dropped it. Plain serde folds both into `None`. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + format_extension: Option>, } #[derive(serde::Deserialize)] @@ -2013,6 +2052,8 @@ struct HubResourceTypeRaw { schema: Option, app: String, description: Option, + #[serde(default)] + format_extension: Option, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2054,6 +2095,7 @@ async fn fetch_resource_types_from_hub() -> error::Result = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), ) .fetch_one(&db) .await?; @@ -2120,13 +2164,27 @@ async fn sync_cached_resource_types( } sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // Whether the payload carried the key at all is what decides: present + // (even as null) is authoritative and may clear, absent means a cache + // written before the column and must leave the stored value alone. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) + VALUES ('admins', $1, $2, $3, $4, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + -- A fileset is a set of files, so it cannot also be one file. + -- Create and update reject the pair; this writer bypasses both, so + -- it declines the extension rather than persisting the forbidden + -- combination onto a same-named local fileset. + format_extension = CASE + WHEN resource_type.is_fileset THEN NULL + WHEN $5 THEN EXCLUDED.format_extension + ELSE resource_type.format_extension END, + edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), ) .execute(&db) .await?; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8daa414b46..47b520736a 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1239,6 +1239,7 @@ async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Resu sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &authed.email).await?; audit_log( &mut *tx, @@ -1661,6 +1662,7 @@ async fn delete_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?; let usernames = sqlx::query_scalar!( "DELETE FROM usr WHERE email = $1 RETURNING username", @@ -1869,7 +1871,7 @@ async fn change_user_email( .execute(&mut *tx) .await?; - // ---- account ---- (draft.email follows through its ON UPDATE CASCADE fkey) + // ---- account ---- sqlx::query!( "UPDATE password SET email = $1 WHERE email = $2", &new_email, @@ -1883,6 +1885,7 @@ async fn change_user_email( } _ => e.into(), })?; + windmill_common::user_drafts::rename_drafts_of_email(&mut *tx, &old_email, &new_email).await?; sqlx::query!( "UPDATE usr SET email = $1 WHERE email = $2", @@ -3539,6 +3542,9 @@ async fn overwrite_global_users( require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; let mut tx = db.begin().await?; + // Replaces the account table, so — unlike the paths that remove one account — it deliberately + // does not call `delete_drafts_of_email`: the addresses are about to be reinstated, and + // dropping every draft on the instance to restore accounts would be pure collateral. sqlx::query!("DELETE FROM password") .execute(&mut *tx) .await?; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c8c49416d8..cb5c639d46 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,6 +12,7 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; +use windmill_dep_map::lock_hash::record_lock_hashes_for_workspace; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; @@ -5799,9 +5800,8 @@ async fn clone_workspace_data( // Clone the forker's own per-user drafts (plus the legacy NULL-email // workspace draft, if any) so they keep their pending edits in the // fork. Other users' drafts are intentionally NOT cloned — they don't - // own a `usr` row in the fork (see `clone_workspace_full`) so their - // drafts would dangle and the home-page `draft_users` aggregate would - // surface them as duplicate legacy entries. + // own a `usr` row in the fork (see `clone_workspace_full`), so those + // drafts would belong to someone the fork holds no membership for. clone_drafts(tx, source_workspace_id, target_workspace_id, &authed.email).await?; // Clone workspace runnable dependencies and dependency map @@ -7140,7 +7140,16 @@ async fn clone_workspace_runnable_dependencies( .execute(&mut **tx) .await?; - // Clone dependency_map to preserve import relationships + // Recorded so the clone's own relocks have something to match; with no row they record NULL + // and nothing in it ever skips. Hashed from the locks the clone holds rather than copied from + // the source's rows, which are only as current as the last write to them: one left stale by a + // supplied lock deployed before this was recorded names a lock the clone no longer has, and an + // importer that resolved against the real one would then skip a relock it needed. + record_lock_hashes_for_workspace(tx, target_workspace_id).await?; + + // Deliberately without `imported_lockfile_hash`: it records what an importer resolved against + // when it was last locked, which nothing here can establish for the version the clone got. + // Left NULL, every importer relocks once and re-anchors both sides to what the clone holds. sqlx::query!( "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) SELECT $1, importer_path, importer_kind, imported_path, importer_node_id diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 92eeaeaedb..3cba2b7da6 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -16532,202 +16532,6 @@ } } }, - "/w/{workspace}/ai_skills/list": { - "get": { - "summary": "list the workspace AI chat skills (name + description only)", - "operationId": "listAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "responses": { - "200": { - "description": "skill listing", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "required": [ - "name", - "description" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - } - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/get/{name}": { - "get": { - "summary": "get a workspace AI chat skill including its instructions", - "operationId": "getAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "skill", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "instructions": { - "type": "string" - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/upload": { - "post": { - "summary": "upsert workspace AI chat skills (admin only)", - "operationId": "uploadAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "skills" - ], - "properties": { - "skills": { - "type": "array", - "maxItems": 50, - "items": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9-]+$" - }, - "description": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "instructions": { - "type": "string", - "minLength": 1, - "maxLength": 65536 - } - } - } - } - } - } - } - } - }, - "responses": { - "200": { - "description": "uploaded", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/delete/{name}": { - "delete": { - "summary": "delete a workspace AI chat skill (admin only)", - "operationId": "deleteAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "deleted", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, "/w/{workspace}/apps/get_data/v/{secretWithExtension}": { "get": { "summary": "get raw app data by", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 235c80ccf9..d5a14a717d 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -17063,141 +17063,6 @@ paths: text/plain: schema: type: string - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - responses: - '200': - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: ^[a-z0-9-]+$ - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - '200': - description: uploaded - content: - text/plain: - schema: - type: string - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: deleted - content: - text/plain: - schema: - type: string /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index be39fddcb6..a27be7d295 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.800.1 + version: 1.803.0 title: Windmill API contact: @@ -10456,7 +10456,7 @@ paths: summary: run script by path operationId: runScriptByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -12534,133 +12534,6 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - responses: - "200": - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: "^[a-z0-9-]+$" - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - "200": - description: uploaded - content: - text/plain: - schema: - type: string - - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: deleted - content: - text/plain: - schema: - type: string - /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -13941,7 +13814,7 @@ paths: summary: run flow by path operationId: runFlowByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -27262,6 +27135,13 @@ components: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + copilot_disabled: + type: boolean + description: >- + Hides the Windmill AI assistant (chat, sessions, code generation, completion, + fixes) from the workspace UI. Read from the workspace's own settings even when + the providers served fall back to the instance config. AI agent steps and the + AI sandbox in flows are unaffected. FreeTierInfo: type: object @@ -29114,6 +28994,8 @@ components: - "igroup.delete" - "igroup.adduser" - "igroup.removeuser" + - "instance_groups.jit_adduser" + - "instance_groups.jit_removeuser" - "variables.decrypt_secret" - "workspaces.read_encryption_key" - "workspaces.edit_command_script" @@ -29612,6 +29494,12 @@ components: type: string is_fileset: type: boolean + format_extension: + type: string + nullable: true + description: >- + File extension for a type whose value is one file rather than a set + of fields. Omit to leave it unchanged; send null to clear it. TriggerHistoryEntry: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index c684830b27..e70cc19544 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -445,6 +445,12 @@ pub struct AIConfig { /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] pub model_pricing: Option>, + /// Hides the Windmill AI assistant (chat, sessions, generation, completion, fixes) from + /// the workspace UI. Only the workspace's own row is consulted: the flag holds even when + /// the providers served come from the instance config or the free tier. AI agent steps + /// and the AI sandbox are unaffected, so the providers stay in force. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub copilot_disabled: bool, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the diff --git a/backend/windmill-api/src/ai_skills.rs b/backend/windmill-api/src/ai_skills.rs deleted file mode 100644 index 11d99d55da..0000000000 --- a/backend/windmill-api/src/ai_skills.rs +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2026 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::db::{ApiAuthed, DB}; -use axum::{ - extract::{Extension, Json, Path}, - routing::{delete, get, post}, - Router, -}; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::require_admin, -}; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_skills)) - .route("/get/{name}", get(get_skill)) - .route("/upload", post(upload_skills)) - .route("/delete/{name}", delete(delete_skill)) -} - -/// Cheap listing surfaced in the AI chat system prompt — no `instructions` body. -#[derive(Serialize)] -pub struct SkillListItem { - pub name: String, - pub description: String, -} - -/// Full skill, including the SKILL.md body, fetched on demand by `read_skill`. -#[derive(Serialize)] -pub struct Skill { - pub name: String, - pub description: String, - pub instructions: String, -} - -#[derive(Deserialize)] -pub struct UploadSkills { - pub skills: Vec, -} - -#[derive(Deserialize)] -pub struct SkillUpload { - pub name: String, - pub description: String, - pub instructions: String, -} - -const MAX_SKILLS_PER_UPLOAD: usize = 50; -// Every stored skill's name + description is advertised in the global AI chat -// system prompt, so bound the total a workspace can accumulate across uploads. -const MAX_SKILLS_PER_WORKSPACE: usize = 100; -// `name` and `description` follow the Claude SKILL.md spec -// (https://platform.claude.com/docs/en/agents-and-tools/agent-skills): both are -// loaded into the AI chat system prompt and `name` is the model-facing skill id, -// so matching the upstream limits keeps skills portable with Claude Code. -const MAX_SKILL_NAME_CHARS: usize = 64; -const MAX_SKILL_DESCRIPTION_CHARS: usize = 1_024; -// Not a spec field — a payload bound on the SKILL.md body, so measured in bytes. -const MAX_SKILL_INSTRUCTIONS_BYTES: usize = 64 * 1024; - -fn validate_skill(skill: &SkillUpload) -> Result<()> { - let name = skill.name.trim(); - if name.is_empty() || name.chars().count() > MAX_SKILL_NAME_CHARS { - return Err(Error::BadRequest(format!( - "skill name must be between 1 and {MAX_SKILL_NAME_CHARS} characters, got {:?}", - skill.name - ))); - } - if !name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - return Err(Error::BadRequest(format!( - "skill name {name:?} must only contain lowercase letters, digits or '-'" - ))); - } - if skill.description.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} is missing a description (the SKILL.md frontmatter `description`)" - ))); - } - if skill.description.chars().count() > MAX_SKILL_DESCRIPTION_CHARS { - return Err(Error::BadRequest(format!( - "skill {name:?} description must be at most {MAX_SKILL_DESCRIPTION_CHARS} characters" - ))); - } - if skill.instructions.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} has an empty SKILL.md body" - ))); - } - if skill.instructions.len() > MAX_SKILL_INSTRUCTIONS_BYTES { - return Err(Error::BadRequest(format!( - "skill {name:?} instructions must be at most {MAX_SKILL_INSTRUCTIONS_BYTES} bytes" - ))); - } - Ok(()) -} - -/// Collect the trimmed skill names, rejecting duplicates within a single upload. -/// The insert upserts by name, so a duplicate would silently keep only the last -/// and make the reported/audited count wrong. -fn collect_upload_names(skills: &[SkillUpload]) -> Result> { - let mut names = Vec::with_capacity(skills.len()); - let mut seen = HashSet::with_capacity(skills.len()); - for skill in skills { - let name = skill.name.trim().to_string(); - if !seen.insert(name.clone()) { - return Err(Error::BadRequest(format!( - "duplicate skill name {name:?} in upload" - ))); - } - names.push(name); - } - Ok(names) -} - -/// Reject an upload that would push the workspace past `MAX_SKILLS_PER_WORKSPACE`. -/// Uploads upsert, so names already present (`replacing`) don't count as new. -fn check_workspace_skill_capacity( - existing_total: i64, - replacing: i64, - upload_count: usize, -) -> Result<()> { - let new_count = upload_count as i64 - replacing; - if existing_total + new_count > MAX_SKILLS_PER_WORKSPACE as i64 { - return Err(Error::BadRequest(format!( - "workspace cannot store more than {MAX_SKILLS_PER_WORKSPACE} skills" - ))); - } - Ok(()) -} - -async fn list_skills( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query!( - "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - &w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json( - rows.into_iter() - .map(|r| SkillListItem { name: r.name, description: r.description }) - .collect(), - )) -} - -async fn get_skill( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let row = sqlx::query!( - "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - row.map(|r| { - Json(Skill { name: r.name, description: r.description, instructions: r.instructions }) - }) - .ok_or_else(|| Error::NotFound(format!("no skill named {name:?} in workspace {w_id}"))) -} - -/// Bulk upsert the uploaded skills by name. Existing skills not in the payload -/// are left untouched — removal goes through `delete_skill`. -async fn upload_skills( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(payload): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - if payload.skills.is_empty() { - return Err(Error::BadRequest("no skills to upload".to_string())); - } - if payload.skills.len() > MAX_SKILLS_PER_UPLOAD { - return Err(Error::BadRequest(format!( - "cannot upload more than {MAX_SKILLS_PER_UPLOAD} skills at a time" - ))); - } - for skill in &payload.skills { - validate_skill(skill)?; - } - let names = collect_upload_names(&payload.skills)?; - - let mut tx = db.begin().await?; - let counts = sqlx::query!( - r#"SELECT - COUNT(*)::bigint AS "total!", - COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS "replacing!" - FROM ai_skill - WHERE workspace_id = $1"#, - &w_id, - &names - ) - .fetch_one(&mut *tx) - .await?; - check_workspace_skill_capacity(counts.total, counts.replacing, names.len())?; - - for (skill, name) in payload.skills.iter().zip(names.iter()) { - sqlx::query!( - r#"INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by) - VALUES ($1, $2, $3, $4, now(), $5) - ON CONFLICT (workspace_id, name) DO UPDATE - SET description = EXCLUDED.description, - instructions = EXCLUDED.instructions, - edited_at = now(), - edited_by = EXCLUDED.edited_by"#, - &w_id, - name, - skill.description, - skill.instructions, - &authed.username, - ) - .execute(&mut *tx) - .await?; - } - - let audit_resource = names.join(","); - audit_log( - &mut *tx, - &authed, - "ai_skills.upload", - ActionKind::Update, - &w_id, - Some(&audit_resource), - Some([("skill_count", &names.len().to_string()[..])].into()), - ) - .await?; - tx.commit().await?; - - Ok(format!( - "Uploaded {} skill(s) to workspace {}", - payload.skills.len(), - &w_id - )) -} - -async fn delete_skill( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - let deleted = sqlx::query_scalar!( - "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - - if deleted.is_none() { - tx.commit().await?; - return Err(Error::NotFound(format!( - "no skill named {name:?} in workspace {w_id}" - ))); - } - - audit_log( - &mut *tx, - &authed, - "ai_skills.delete", - ActionKind::Delete, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Deleted skill {name} from workspace {w_id}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn skill() -> SkillUpload { - SkillUpload { - name: "test-skill".to_string(), - description: "Useful for tests".to_string(), - instructions: "# Test\n\nDo the thing.".to_string(), - } - } - - #[test] - fn validate_skill_rejects_oversized_description() { - let mut skill = skill(); - skill.description = "x".repeat(MAX_SKILL_DESCRIPTION_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_instructions() { - let mut skill = skill(); - skill.instructions = "x".repeat(MAX_SKILL_INSTRUCTIONS_BYTES + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_name() { - let mut skill = skill(); - skill.name = "a".repeat(MAX_SKILL_NAME_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_non_slug_name() { - // Uppercase, underscore, space and punctuation are all outside the - // Claude SKILL.md `[a-z0-9-]` name charset. - for bad in ["My-Skill", "my_skill", "my skill", "skill!"] { - let mut skill = skill(); - skill.name = bad.to_string(); - - assert!( - matches!(validate_skill(&skill), Err(Error::BadRequest(_))), - "{bad:?} should be rejected" - ); - } - } - - #[test] - fn validate_skill_counts_description_in_characters() { - // 1024 two-byte chars exceed the byte limit but sit exactly on the - // character limit, so they must be accepted. - let mut skill = skill(); - skill.description = "é".repeat(MAX_SKILL_DESCRIPTION_CHARS); - - assert!(validate_skill(&skill).is_ok()); - } - - #[test] - fn workspace_capacity_allows_replacement_at_cap() { - // Already at the cap, but the upload only replaces an existing skill. - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(check_workspace_skill_capacity(at_cap, 1, 1).is_ok()); - } - - #[test] - fn workspace_capacity_rejects_new_skill_over_cap() { - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(matches!( - check_workspace_skill_capacity(at_cap, 0, 1), - Err(Error::BadRequest(_)) - )); - } - - #[test] - fn collect_upload_names_trims_and_collects() { - let names = collect_upload_names(&[skill()]).unwrap(); - assert_eq!(names, vec!["test-skill".to_string()]); - } - - #[test] - fn collect_upload_names_rejects_duplicates() { - // Names are compared after trimming, so whitespace can't smuggle a dup in. - let dup = SkillUpload { name: " test-skill ".to_string(), ..skill() }; - assert!(matches!( - collect_upload_names(&[skill(), dup]), - Err(Error::BadRequest(_)) - )); - } -} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 7bc995844d..0d29c1c369 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -491,7 +491,8 @@ async fn list_apps( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app')) as draft_users", + WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app') \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(app.workspace_id, app.path) as inherited_labels", ]) .left() diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index b44b0e64be..e5928f761e 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -214,6 +214,9 @@ fn list_drafts_query(all_users: bool) -> String { // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled (`password.username` is NULL). This // keeps the raw email out of the payload whenever a derived username exists. + // A null username means the legacy row downstream, so an owner that resolves to + // no name at all — an external JWT's subject has neither row — is dropped rather + // than surfaced as a second legacy entry. let draft_users = r#"CASE WHEN d.typ::text IN ('script', 'flow', 'app', 'raw_app') THEN ( SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END) NULLS LAST) @@ -221,6 +224,7 @@ fn list_drafts_query(all_users: bool) -> String { LEFT JOIN usr u ON u.workspace_id = du.workspace_id AND u.email = du.email LEFT JOIN password p ON p.email = du.email AND p.super_admin = true WHERE du.workspace_id = d.workspace_id AND du.path = d.path AND du.typ = d.typ + AND (du.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ) ELSE NULL END"#; // Default lists the user's own drafts AND the legacy NULL-email rows; with // `all_users` the filter is dropped to list every workspace draft. diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a30cffbfbf..53044ee020 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -18,6 +18,7 @@ use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; use sha2::{Digest, Sha256}; +use std::borrow::Cow; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; @@ -108,6 +109,7 @@ use windmill_common::{ flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, + query_builders, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, @@ -8131,6 +8133,79 @@ pub async fn run_wait_result_flow_by_version( .await } +/// Whether request-supplied SQL from an operator may run. Operators can only run deployed +/// code, so a request their job token (`WM_TOKEN`) authenticates comes from code a +/// non-operator authored. The job must still be running, and the request must have the +/// shape `wmill.datatable()` sends (PostgreSQL against a `datatable://` database), so a +/// WM_TOKEN that leaked into job logs cannot be replayed to reach another target while the +/// job lives, in particular DuckDB, which runs in-process in the worker. +/// +/// What it does permit is any statement against the workspace's data tables, writes and DDL +/// included: the helper's body is an unrestricted SQL template and data tables carry no +/// per-user ACL. Narrowing that is a separate decision from this exemption. +/// +/// The database argument is only half the target: the executor honors a `-- database` +/// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused. +/// Check them against the code the executor runs rather than the request's `content`, which +/// is not the same string once a `WM_INTERNAL_DB` marker expands. +async fn operator_may_run_datatable_query( + db: &DB, + w_id: &str, + job_id: Option, + language: Option<&ScriptLang>, + content: &str, + args: Option<&HashMap>>, +) -> error::Result { + let Some(job_id) = job_id else { + return Ok(false); + }; + if language != Some(&ScriptLang::Postgresql) { + return Ok(false); + } + // Parse the directives out of the code the executor actually runs: it expands a + // `WM_INTERNAL_DB` marker first, and a directive can be embedded in the expansion. + // An expansion that overrides the language would run something other than the SQL the + // language check above cleared, so it is refused along with a malformed marker. + let executed = + match query_builders::try_expand_internal_db_query(content, &ScriptLang::Postgresql) { + Some(Ok(expanded)) if expanded.language_override.is_none() => Cow::Owned(expanded.code), + Some(_) => return Ok(false), + None => Cow::Borrowed(content), + }; + if windmill_parser_sql::parse_db_resource(&executed).is_some() + || !matches!(windmill_parser_sql::parse_s3_mode(&executed), Ok(None)) + { + return Ok(false); + } + let targets_datatable = args + .and_then(|args| args.get("database")) + .and_then(|database| serde_json::from_str::(database.get()).ok()) + .is_some_and(|database| database.starts_with("datatable://")); + if !targets_datatable { + return Ok(false); + } + Ok(sqlx::query_scalar!( + "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} + +/// The refusal an operator gets from a preview route. Inside a job the caller never ran a +/// preview themselves, so name the one thing the job's token may do. +fn operator_preview_refusal(job_id: Option) -> error::Error { + let reason = if job_id.is_some() { + "Operators cannot run preview jobs for security reasons: from inside a job, an \ + operator may only run a wmill.datatable() query while that job is running" + } else { + "Operators cannot run preview jobs for security reasons" + }; + error::Error::NotAuthorized(reason.to_string()) +} + async fn run_preview_script( authed: ApiAuthed, Extension(db): Extension, @@ -8142,9 +8217,20 @@ async fn run_preview_script( #[cfg(feature = "enterprise")] check_license_key_valid().await?; if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // A deferred run would outlive the running job the exemption keys off. + if run_query.get_scheduled_for(&db).await?.is_some() + || !operator_may_run_datatable_query( + &db, + &w_id, + authed.job_id, + preview.language.as_ref(), + preview.content.as_deref().unwrap_or_default(), + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(authed.job_id)); + } } // Preview runs arbitrary, request-supplied code. require_path_read_access_for_preview // only checks folder/namespace *read* access (and is a no-op when path is null), so a @@ -8239,13 +8325,21 @@ async fn run_inline_preview_script( Path(w_id): Path, Json(preview): Json, ) -> error::Result { - // Same arbitrary-code class as run_preview_script: operators are blocked from - // running request-supplied code, and a narrowly-scoped token must not escape - // its scope through inline preview. - if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // Same arbitrary-code class as run_preview_script, and every worker and standalone + // server exposes this route, so an operator is refused on the same terms. A + // narrowly-scoped token must not escape its scope through inline preview either. + if authed.is_operator + && !operator_may_run_datatable_query( + &db, + &w_id, + job_id, + Some(&preview.language), + &preview.content, + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(job_id)); } check_scopes(&authed, || format!("jobs:run"))?; if let Some(job_id) = job_id { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fc98020d1b..054af0f2fa 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,7 +69,6 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; -mod ai_skills; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; @@ -453,15 +452,15 @@ pub async fn run_server( // unless they are allowed — hence a separate layer rather than widening the // one every other route shares. (`Mcp-Param-*` is only sent for tool inputs // annotated with `x-mcp-header`, which no tool here declares.) + // + // The request's own header list is mirrored rather than enumerated: a browser + // MCP client may send any custom name for a preprocessor to read, and no fixed + // list could cover them. Nothing is granted by echoing it: the origin is + // `Any`, so browsers never attach credentials, and the endpoint authenticates + // each request on its own. let mcp_cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST, http::Method::DELETE]) - .allow_headers([ - http::header::CONTENT_TYPE, - http::header::AUTHORIZATION, - http::HeaderName::from_static("mcp-protocol-version"), - http::HeaderName::from_static("mcp-method"), - http::HeaderName::from_static("mcp-name"), - ]) + .allow_headers(tower_http::cors::AllowHeaders::mirror_request()) // The 401 challenge is how a client discovers where to authorize (RFC 9728), // and it is not a safelisted response header, so without this a browser // client sees an empty one and has no way to begin the OAuth flow. @@ -713,7 +712,6 @@ pub async fn run_server( Router::new() }) .nest("/ai", ai::workspaced_service()) - .nest("/ai_skills", ai_skills::workspaced_service()) .nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service()) .nest( "/path_autocomplete", diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 568ad854f9..aa2415f664 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -878,7 +878,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runScriptByPath"), description: Cow::Borrowed("run script by path"), - instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/p/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ @@ -1419,7 +1419,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runFlowByPath"), description: Cow::Borrowed("run flow by path"), - instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/f/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 2b4350bcfe..480e3c0841 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -12,7 +12,9 @@ use windmill_mcp::common::transform::transform_property_keys; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend, PathFilter}; +use windmill_mcp::server::{ + BackendResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter, +}; use crate::auth::AuthCache; use crate::db::ApiAuthed; @@ -214,8 +216,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, false, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_script_by_path_internal( self.db.clone(), @@ -238,8 +243,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, true, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_flow_by_path_internal( self.db.clone(), diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 94271bb662..65c8069b85 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -11,15 +11,19 @@ use serde_json::Value; use sql_builder::prelude::*; use windmill_common::auth::create_jwt_token; use windmill_common::db::{Authed, UserDB}; +use windmill_common::error::Error; use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; +use windmill_common::triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}; use windmill_common::utils::{query_elems_from_hub, StripPath}; use windmill_common::worker::to_raw_value; use windmill_common::{DB, HUB_BASE_URL}; use windmill_mcp::server::{ - non_empty_body_fields, BackendResult, EndpointTool, ErrorData, PathFilter, + non_empty_body_fields, BackendResult, EndpointTool, ErrorData, McpRequest, PathFilter, }; use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; +use windmill_trigger::trigger_helpers::{get_runnable_format, RunnableId}; +use crate::args::build_headers; use crate::db::ApiAuthed; use crate::HTTP_CLIENT; @@ -641,7 +645,7 @@ fn selects_endpoint_tool(caller_scopes: &[String], tool: &str) -> bool { .is_ok_and(|config| config.endpoints.iter().any(|e| e == tool)) } -/// Create HTTP request with authentication +/// Create HTTP request with authentication. pub async fn create_http_request( method: &str, url: &str, @@ -702,17 +706,113 @@ pub async fn create_http_request( .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) } -/// Convert a JSON Value into PushArgsOwned for job execution -pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned { +/// The `kind` an MCP-invoked runnable sees on its preprocessor event, alongside +/// `webhook`, `http` and the trigger kinds. +const MCP_TRIGGER_KEY: &str = "mcp"; + +/// A preprocessor's view of the MCP request that ran it. Mirrors the HTTP +/// trigger event: `body` is what the model sent, everything else describes the +/// call itself. +#[derive(serde::Serialize)] +struct McpPreprocessorEvent<'a> { + kind: &'a str, + body: Box, + headers: HashMap>, + tool_name: &'a str, +} + +/// Headers withheld from a preprocessor because they authenticate the connection. +/// +/// Not a security boundary: a webhook preprocessor receives all three. Withheld +/// because nothing needs them yet, and releasing one later is additive while +/// withdrawing one after runnables read it is not. +const WITHHELD_FROM_PREPROCESSOR: &[&str] = &["authorization", "cookie", "proxy-authorization"]; + +/// Every header a preprocessor may see. +fn preprocessor_headers( + headers: &http::HeaderMap, +) -> HashMap> { + let mut selected = build_headers(headers, None, true); + selected.retain(|name, _| { + !WITHHELD_FROM_PREPROCESSOR + .iter() + .any(|withheld| withheld.eq_ignore_ascii_case(name)) + }); + selected +} + +/// Build the job arguments for a script or flow run as an MCP tool. +/// +/// Shaped by the runnable's own format: a preprocessor receives the request as +/// an event, and a runnable without one receives only what the model sent. +pub async fn prepare_push_args( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + args: Value, + request: &McpRequest<'_>, +) -> Result { + let mut main_args = HashMap::new(); if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); for (k, v) in map { - args_hash.insert(k, to_raw_value(&v)); + main_args.insert(k, to_raw_value(&v)); } - windmill_queue::PushArgsOwned { extra: None, args: args_hash } - } else { - windmill_queue::PushArgsOwned::default() } + + let runnable_id = if is_flow { + RunnableId::from_flow_path(path) + } else { + // Resolves a `hub/` path to the hub script on its own. + RunnableId::from_script_path(path) + }; + + // MCP is not one of the `TRIGGER_KIND` enum values and does not need to be: + // the per-kind arms of the no-preprocessor heuristic are payload-shape + // special cases for message triggers, and `Webhook` reaches the same generic + // arm MCP wants while sharing that kind's format cache. + let runnable_format = get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; + + Ok(match runnable_format { + // Without a preprocessor there is nowhere for a header to go that the + // model does not also write: its arguments *are* the runnable's + // parameters, so a header bound to one of them would be a value the model + // could set. The request is reachable through a preprocessor, where it + // arrives in a key of the event the model never fills. + RunnableFormat { has_preprocessor: false, .. } => { + windmill_queue::PushArgsOwned { args: main_args, extra: None } + } + RunnableFormat { has_preprocessor: true, version } => { + let headers = preprocessor_headers(request.headers); + match version { + RunnableFormatVersion::V2 => { + let event = McpPreprocessorEvent { + kind: MCP_TRIGGER_KEY, + body: to_raw_value(&main_args), + headers, + tool_name: request.tool_name, + }; + windmill_queue::PushArgsOwned { + args: HashMap::from([("event".to_string(), to_raw_value(&event))]), + extra: None, + } + } + RunnableFormatVersion::V1 => windmill_queue::PushArgsOwned { + args: main_args, + extra: Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": MCP_TRIGGER_KEY, + MCP_TRIGGER_KEY: { + "headers": headers, + "tool_name": request.tool_name, + } + })), + )])), + }, + } + } + }) } /// Parse an HTTP response body into a JSON Value diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index a69441ea75..2998faa7f3 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -618,6 +618,7 @@ pub(crate) async fn offboard_global_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email).await?; sqlx::query!("DELETE FROM workspace_invite WHERE email = $1", &email) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index c145252251..ea6d0c5bba 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -261,7 +261,8 @@ fn branch_sqls() -> Branches { FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred}) as draft_users" + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred} \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users" ) }; diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index 2268a63894..a1a9206196 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -99,7 +99,6 @@ fn build_standard_scope_domains() -> Vec { ("configs", "Configs", "Configuration management", false), ("oauth", "OAuth", "OAuth management", false), ("ai", "AI", "AI feature management", false), - ("ai_skills", "AI Skills", "AI skill management", false), ( "ai_evals", "AI Evals", diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 76378e335a..da8532e9d1 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -146,6 +146,7 @@ async fn edit_copilot_config( .await?; let workspace_has_config = ai_config.has_providers(); + let copilot_disabled = ai_config.copilot_disabled; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -158,7 +159,7 @@ async fn edit_copilot_config( .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()) .filter(|c| c.has_providers()); - let effective_ai_config = if workspace_has_config { + let mut effective_ai_config = if workspace_has_config { ai_config } else if let Some(instance_config) = instance_config_with_providers { instance_config @@ -172,6 +173,7 @@ async fn edit_copilot_config( } else { AIConfig::default() }; + effective_ai_config.copilot_disabled = copilot_disabled; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -207,6 +209,9 @@ async fn get_copilot_info( )) })?; + let copilot_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.copilot_disabled); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -215,20 +220,23 @@ async fn get_copilot_info( // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the // free-tier fallback, matching the proxy and edit_copilot_config paths. .filter(|c| c.has_providers()); - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = instance_config { - Ok(Json(instance_config)) - } else if let Some(free_config) = - crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? - { - // Nothing configured: fall back to Windmill's free tier (EE-only). The config - // carries a `free_tier` marker even once the user's grant is spent — with no - // providers, but telling the client *why* AI is off. - Ok(Json(free_config)) - } else { - Ok(Json(AIConfig::default())) - } + let mut effective = + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + workspace_ai_config.0 + } else if let Some(instance_config) = instance_config { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + free_config + } else { + AIConfig::default() + }; + effective.copilot_disabled = copilot_disabled; + Ok(Json(effective)) } #[cfg(feature = "enterprise")] diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b1ad2edd01..d38396562b 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -94,6 +94,9 @@ pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation pub const DISABLE_WORKSPACE_INVITE_EMAILS_SETTING: &str = "disable_workspace_invite_emails"; pub const DISABLE_PASSWORD_LOGIN_SETTING: &str = "disable_password_login"; pub const AUTO_LOGIN_PROVIDER_SETTING: &str = "auto_login_provider"; +/// Name of the SAML attribute or OIDC userinfo claim carrying the user's IdP groups. Unset or +/// empty leaves instance-group membership entirely to SCIM. +pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; @@ -108,6 +111,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; +pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index f1784e9841..b4128f80d2 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -148,43 +148,76 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; /// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower /// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` /// the cleanup queries build overflows Postgres' microsecond field. -const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; +const MAX_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; -/// Apply a configured service log retention, in seconds. +/// Clamp a configured retention window, in seconds, to one a cutoff can be built from. /// -/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a -/// cutoff. The two unusable directions are not the same mistake and must not share a landing -/// point: too large still says "keep these for a very long time", so it is capped and the -/// intent survives, whereas falling back would delete logs the operator meant to keep. A -/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at -/// or after `now` and the next sweep expires the entire history, rows and object-storage files -/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an -/// operator types by analogy with it, and what the settings UI writes into a field that was -/// merely focused — falls back to the default. -pub fn set_service_log_retention_secs(configured: i64) { - let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS { +/// Shared by the retention windows that have no "keep forever" spelling, so that an unusable +/// value can never reach a cutoff. The two unusable directions are not the same mistake and must +/// not share a landing point: too large still says "keep these for a very long time", so it is +/// capped and the intent survives, whereas falling back would delete data the operator meant to +/// keep. A non-positive value has no such reading — every cutoff is `now - retention`, so it +/// lands at or after `now` and the next sweep expires the entire history. `0` is both what an +/// operator types by analogy with job retention, where it does mean keep forever, and what the +/// settings UI writes into a field that was merely focused, so it falls back to the default. +fn clamp_retention_secs(configured: i64, default: i64, what: &str) -> i64 { + if configured > MAX_RETENTION_SECS { tracing::warn!( - "service log retention of {configured}s exceeds the maximum of \ - {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there" + "{what} retention of {configured}s exceeds the maximum of {MAX_RETENTION_SECS}s, \ + capping it there" ); - MAX_SERVICE_LOG_RETENTION_SECS + MAX_RETENTION_SECS } else if configured >= 1 { configured } else { tracing::warn!( - "service log retention of {configured}s would expire every service log, \ - falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s" + "{what} retention of {configured}s would expire the entire history, \ + falling back to the default of {default}s" ); - DEFAULT_SERVICE_LOG_RETENTION_SECS - }; + default + } +} + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`]. Expiry reaches every copy of a log line: +/// the row, the file on disk, and the object-storage object. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + "service log", + ); SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); } +/// Apply a configured OTEL trace retention, in seconds. +/// +/// The only way into [`OTEL_TRACES_RETENTION_SECS`]. +pub fn set_otel_traces_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_OTEL_TRACES_RETENTION_SECS, + "otel traces", + ); + OTEL_TRACES_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long an HTTP request tracing span stays in `otel_traces`, in seconds. +/// +/// Spans are keyed by the job they were captured for and read back by the job detail view, so +/// this is the outer bound on how far back that view can show a job's HTTP requests. It is +/// independent of job retention: a span can outlive its job, or be swept while the job remains. +pub fn otel_traces_retention_secs() -> i64 { + OTEL_TRACES_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// How long a service log line stays retrievable, in seconds. /// /// The outer bound on everything service-log: the `log_file` rows, the raw files in object @@ -423,6 +456,10 @@ lazy_static::lazy_static! { /// would expire every service log cannot reach a cutoff. Read it with /// [`service_log_retention_secs`]. static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); + /// Private on purpose, same as [`SERVICE_LOG_RETENTION_SECS`]: + /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the + /// only reader. + static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 31219621e9..d71bc50e60 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -358,31 +358,38 @@ pub async fn fetch_script_for_update<'a>( .map_err(crate::error::Error::from) } -pub struct ClonedScript { - pub old_script: NewScript, - pub new_hash: i64, -} -// TODO: What if dependency job fails, there is script with NULL in the lock -pub async fn clone_script<'c>( - path: &str, - w_id: &str, +/// Deploys the outcome of a relative-import relock as a new version of `head`, the path's live +/// version that the caller holds `FOR UPDATE`, and archives `head`. A `lock` of `None` records +/// a failed generation: the version carries `lock_error_logs` instead and runs keep resolving +/// to the last version that has a lock. A `modules` of `None` keeps the head's module locks. +/// +/// Writes whatever `head` names and checks nothing: callers are responsible for having +/// established access to its workspace and path, as a dependency job's push already has. +/// +/// `created_at` is stamped when the insert runs, not at transaction start. The row lock on +/// `head` is what orders one relock after another, and with `now()` a transaction that began +/// first but locked second commits a live child older than its archived parent, which every +/// "latest version" read then mis-orders. +pub async fn deploy_relocked_version( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + head: Script, deployment_message: Option, - db: &DB, -) -> crate::error::Result { - let mut tx = db.begin().await?; - let s = if let Some(s) = fetch_script_for_update(path, w_id, &mut *tx).await? { - s - } else { - return Err(crate::error::Error::NotFound(format!( - "Non-archived script with path '{}' not found", - path - ))); - }; + lock: Option<&str>, + modules: Option<&std::collections::HashMap>, + lock_error_logs: Option<&str>, +) -> crate::error::Result { + let s = head; + let w_id = s.workspace_id.as_str(); - let rs = runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut *tx) - .await?; + let rs = + runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut **tx) + .await?; let (debouncing_settings, concurrency_settings) = - runnable_settings::prefetch_cached_tx(&rs, &mut tx).await?; + runnable_settings::prefetch_cached_tx(&rs, &mut *tx).await?; + + // What the row stores is what the hash covers: the new module locks when there are any. + let modules = modules.cloned().or(s.modules); + let modules_json = modules.as_ref().map(serde_json::to_value).transpose()?; let ns = NewScript { path: s.path.clone(), @@ -392,7 +399,7 @@ pub async fn clone_script<'c>( content: s.content, schema: s.schema, is_template: s.is_template, - lock: None, + lock: lock.map(str::to_string), language: s.language, kind: Some(s.kind), tag: s.tag, @@ -424,7 +431,7 @@ pub async fn clone_script<'c>( on_behalf_of: s.on_behalf_of, preserve_on_behalf_of: None, assets: s.assets, - modules: s.modules, + modules, auto_parent: None, labels: s.labels, skip_draft_deletion: None, @@ -433,7 +440,7 @@ pub async fn clone_script<'c>( let new_hash = hash_script(&ns); tracing::debug!( - "cloning script at path {} from '{}' to '{}'", + "deploying relocked version of script at path {} from '{}' to '{}'", s.path, *s.hash, new_hash @@ -446,17 +453,19 @@ pub async fn clone_script<'c>( envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels) + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, \ + lock_error_logs, created_at) SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ - content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ + content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, \ envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, \ + $6::text, clock_timestamp() FROM script WHERE hash = $2 AND workspace_id = $3; - ", new_hash, s.hash.0, w_id).execute(&mut *tx).await?; + ", new_hash, s.hash.0, w_id, lock, modules_json, lock_error_logs).execute(&mut **tx).await?; // Archive base. sqlx::query!( @@ -464,9 +473,8 @@ pub async fn clone_script<'c>( *s.hash, w_id ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - tx.commit().await?; - Ok(ClonedScript { old_script: ns, new_hash }) + Ok(new_hash) } diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 84a0e8c3ea..115dae34e7 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -245,7 +245,9 @@ async fn fetch_other_drafts_users( // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled. Else a real teammate's draft renders // as a phantom "Legacy draft". The genuine NULL-email legacy row keeps - // `username = None` (no `usr`/`password` match and `d.email` is NULL). + // `username = None` (no `usr`/`password` match and `d.email` is NULL), which is + // why an owner that resolves to no name at all — an external JWT's subject has + // neither row — is dropped instead: `None` is taken to mean "legacy" downstream. let rows = sqlx::query_as!( OtherDraftUser, r#"SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as "username?", @@ -261,6 +263,7 @@ async fn fetch_other_drafts_users( AND d.path = $2 AND d.typ = $3 AND (d.email IS NULL OR d.email <> $4) + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ORDER BY d.email NULLS LAST"#, w_id, path, @@ -426,6 +429,67 @@ pub async fn overlay_or_draft_only( } } +/// Delete the drafts an address owns, across every workspace. +/// +/// `draft.email` carries no foreign key to `password`: a draft's owner is any principal the +/// instance authenticates, and an external JWT's subject never has a `password` row. Deleting an +/// account is therefore what has to delete its drafts — a delete path that skips this leaves them +/// behind forever, addressed to someone who no longer exists. Call it in the same transaction as +/// the account removal. +/// +/// No authorization of its own: it acts instance-wide on whatever address it is handed, so the +/// caller must already have authorized removing that account (superadmin, the account's own +/// holder, or SCIM). +pub async fn delete_drafts_of_email<'c>( + executor: impl sqlx::PgExecutor<'c>, + email: &str, +) -> Result<()> { + sqlx::query!("DELETE FROM draft WHERE email = $1", email) + .execute(executor) + .await?; + Ok(()) +} + +/// Move the drafts an address owns onto its new address, for the same reason +/// [`delete_drafts_of_email`] exists: no foreign key follows the rename, so drafts left behind are +/// stranded on an address that no longer authenticates. Same authorization contract, for a rename. +/// +/// The two addresses may each already hold a draft of the same item, since the destination can +/// belong to a principal with no account and so is not covered by the caller's "address is free" +/// check. `draft_pkey_with_user` admits only one, so the moving account's wins — which is also why +/// a rename onto the same address returns early: every row would collide with itself and be +/// cleared. Callers need not compare first (an IdP re-sending an unchanged `userName` does not). +pub async fn rename_drafts_of_email( + conn: &mut sqlx::PgConnection, + old_email: &str, + new_email: &str, +) -> Result<()> { + if old_email == new_email { + return Ok(()); + } + sqlx::query!( + "DELETE FROM draft dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.path = dest.path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Delete EVERY user's draft (and the legacy NULL-email row) at a path+kind. /// Use when the item is DELETED outright: it's gone for everyone, so leaving /// teammates' drafts behind would orphan them forever. Discarding one's OWN diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index c1d8fc00bc..697fc69e44 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -175,7 +175,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -183,7 +183,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28910/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28930/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/backend/windmill-common/tests/user_drafts_rename.rs b/backend/windmill-common/tests/user_drafts_rename.rs new file mode 100644 index 0000000000..35d2b53858 --- /dev/null +++ b/backend/windmill-common/tests/user_drafts_rename.rs @@ -0,0 +1,28 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::user_drafts::rename_drafts_of_email; + +/// A rename onto the same address has to be a no-op: the helper clears a draft the destination +/// already holds at the same item, and every row would be its own destination. SCIM PATCH sends +/// `userName` unconditionally, so an IdP re-sending an unchanged one reaches this. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn renaming_onto_the_same_address_keeps_the_drafts(db: Pool) { + sqlx::query( + "INSERT INTO draft(workspace_id, path, typ, value, email) \ + VALUES ('test-workspace', 'u/test-user/s', 'script', '{}'::json, 'test@windmill.dev')", + ) + .execute(&db) + .await + .expect("failed to seed draft"); + + let mut conn = db.acquire().await.unwrap(); + rename_drafts_of_email(&mut conn, "test@windmill.dev", "test@windmill.dev") + .await + .unwrap(); + + let kept: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM draft WHERE email = 'test@windmill.dev'") + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(kept, 1); +} diff --git a/backend/windmill-dep-map/Cargo.toml b/backend/windmill-dep-map/Cargo.toml index 28b8552e8e..a93376d167 100644 --- a/backend/windmill-dep-map/Cargo.toml +++ b/backend/windmill-dep-map/Cargo.toml @@ -26,4 +26,5 @@ tracing.workspace = true lazy_static.workspace = true chrono.workspace = true itertools.workspace = true +futures.workspace = true uuid.workspace = true diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index ee6a3c3fd4..aab50795a9 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -1,6 +1,7 @@ pub mod ci_tests; #[cfg(feature = "private")] pub mod ci_tests_ee; +pub mod lock_hash; pub mod scoped_dependency_map; pub mod trigger_dependents; pub mod workspace_dependencies; @@ -127,6 +128,40 @@ pub fn extract_referenced_paths( } } +/// Re-records which paths `script_path` imports and what each one's lock hashes to right now. +/// That snapshot is what a later relock-skip check of this importer compares against, so it +/// has to move whenever the imports may have, whether or not the importer's own lock did. +/// +/// Writes for any path in `w_id` and checks nothing: callers are responsible for having +/// established access to that workspace and script, as a dependency job's push already has. +pub async fn refresh_dependency_map( + db: &sqlx::Pool, + w_id: &str, + script_path: &str, + parent_path: &Option, + code: &str, + script_lang: &Option, +) -> error::Result<()> { + use scoped_dependency_map::ScopedDependencyMap; + + let mut tx = db.begin().await?; + let mut dependency_map = + ScopedDependencyMap::fetch_maybe_rearranged(w_id, script_path, "script", parent_path, db) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + + dependency_map.dissolve(tx).await.commit().await?; + Ok(()) +} + pub async fn process_relative_imports( db: &sqlx::Pool, _job_id: Option, @@ -144,29 +179,7 @@ pub async fn process_relative_imports( use scoped_dependency_map::ScopedDependencyMap; use trigger_dependents::trigger_dependents_to_recompute_dependencies; - // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled - { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, - ) - .await?; - - tx = dependency_map - .patch( - extract_referenced_paths(&code, script_path, *script_lang), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - - dependency_map.dissolve(tx).await.commit().await?; - } + refresh_dependency_map(db, w_id, script_path, &parent_path, code, script_lang).await?; { let mut already_visited = args diff --git a/backend/windmill-dep-map/src/lock_hash.rs b/backend/windmill-dep-map/src/lock_hash.rs new file mode 100644 index 0000000000..50bd18a8ee --- /dev/null +++ b/backend/windmill-dep-map/src/lock_hash.rs @@ -0,0 +1,79 @@ +use std::collections::HashMap; + +use futures::TryStreamExt; +use sqlx::{Postgres, Transaction}; +use windmill_common::error::Result; +use windmill_common::scripts::hash_script; + +/// Records what the lock now at each path hashes to, which is one half of the comparison a relock +/// skip makes against what each importer resolved against. +/// +/// Writes any path in `w_id` and checks nothing: callers are responsible for having established +/// the caller's access to that workspace. A path repeated in `entries` keeps its last hash. +/// +/// Callers that write the lock itself in the same statement fold the upsert into that statement +/// instead; this is for the ones with nothing to fold it into. +pub async fn record_lock_hashes( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + entries: &[(String, i64)], +) -> Result<()> { + // Postgres rejects a whole statement that resolves a conflict on one key twice, so a path + // given more than once keeps its last hash, as it would if the two were written in order. + let mut deduped: HashMap<&str, i64> = HashMap::with_capacity(entries.len()); + for (path, hash) in entries { + deduped.insert(path.as_str(), *hash); + } + if deduped.is_empty() { + return Ok(()); + } + let (paths, hashes): (Vec, Vec) = deduped + .into_iter() + .map(|(path, hash)| (path.to_string(), hash)) + .unzip(); + // Recording a hash a path already has would still cut a row version, and the no-op push this + // is reached from is the mode a git-sync of an unchanged workspace runs in. + sqlx::query!( + "INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + SELECT $1, * FROM UNNEST($2::text[], $3::bigint[]) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash + WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + w_id, + &paths[..], + &hashes[..] + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Records the hash of every live lock in `w_id`, for a workspace whose scripts arrived without +/// going through a deploy — a clone, which copies their locks verbatim and so would otherwise hold +/// none of the hashes describing them. +/// +/// Carries the same caller obligation as [`record_lock_hashes`]. +/// +/// `script.lock` is unbounded and a workspace holds one per script, so the rows are streamed and +/// each lock is hashed and dropped before the next arrives; only the hashes accumulate. +pub async fn record_lock_hashes_for_workspace( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, +) -> Result<()> { + let mut entries: Vec<(String, i64)> = Vec::new(); + { + let mut rows = sqlx::query!( + "SELECT DISTINCT ON (path) path, lock FROM script + WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL + ORDER BY path, created_at DESC", + w_id + ) + .fetch(&mut **tx); + + while let Some(row) = rows.try_next().await? { + if let Some(lock) = row.lock { + entries.push((row.path, hash_script(&lock))); + } + } + } + record_lock_hashes(tx, w_id, &entries).await +} diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs index 3f88f7e781..d49fa468ae 100644 --- a/backend/windmill-mcp/src/common/schema.rs +++ b/backend/windmill-mcp/src/common/schema.rs @@ -101,7 +101,7 @@ fn apply_resource_enrichment( let resources_count = resource_cache.len(); let description = match resource_type { Some(rt) => format!( - "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", + "This is a resource named `{}` with the following description: `{}`.\nPass it as the bare string `$res:` — the whole value of this argument, never an object wrapper like {{\"$res\": \"\"}} and never a plain path.\n{}", rt.name, rt.description.as_deref().unwrap_or("No description"), if resources_count == 0 { @@ -138,7 +138,7 @@ fn apply_resource_enrichment( ) }) .collect::>() - .join("\\n"); + .join("\n"); let prior_description = prop_map .get("description") .and_then(Value::as_str) @@ -147,7 +147,7 @@ fn apply_resource_enrichment( prop_map.insert( "description".to_string(), Value::String(format!( - "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", + "{}\nHere are the available resources, one per line as `title: $res:path`. The title is only a label; pass the `$res:path` part verbatim as this argument's value:\n{}", prior_description, resources_description )), ); @@ -804,6 +804,10 @@ mod tests { let desc = node["description"].as_str().unwrap(); assert!(desc.contains("c_aws_account")); assert!(desc.contains("$res:f/platform/aws_dev")); + // MCP clients render this description verbatim, so the separators must be + // real newlines rather than the two-character escape. + assert!(desc.contains('\n')); + assert!(!desc.contains("\\n")); } #[test] diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index e42c0dacfc..7ca1eb08a1 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -16,6 +16,14 @@ use crate::server::endpoints::EndpointTool; /// Result type for backend operations using rmcp's ErrorData directly pub type BackendResult = Result; +/// What the backend needs about the HTTP request a tool call arrived on, in order +/// to hand a runnable the headers of the call that triggered it. +pub struct McpRequest<'a> { + pub headers: &'a http::HeaderMap, + /// The MCP tool name the caller invoked, reported to preprocessors. + pub tool_name: &'a str, +} + /// How a script/flow listing is narrowed by path at the SQL layer, *before* the /// `ITEMS_FETCH_MAX_LIMIT` cap applies. /// @@ -157,6 +165,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Run a flow and wait for result @@ -166,6 +175,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Call an endpoint tool (generated API endpoint) diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index b6fb7a5b0a..b97e374e98 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -12,7 +12,7 @@ pub mod tools; // Re-export main types pub use crate::common::types::{McpToken, MultiWorkspaceMcp, WorkspaceInfo}; -pub use backend::{BackendResult, McpAuth, McpBackend, PathFilter}; +pub use backend::{BackendResult, McpAuth, McpBackend, McpRequest, PathFilter}; pub use endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only, list_workspaces_tool, non_empty_body_fields, EndpointTool, diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index f407504c4b..e27eaa8470 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -9,8 +9,10 @@ use crate::common::transform::{ extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, }; -use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId}; -use crate::server::backend::{McpAuth, McpBackend, PathFilter}; +use crate::common::types::{ + McpToken, MultiWorkspaceMcp, ResourceInfo, SchemaType, ToolableItem, WorkspaceId, +}; +use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter}; use crate::server::endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; @@ -101,16 +103,24 @@ enum McpMode { Multi(String), } +/// Everything a request carries besides its MCP payload. +struct McpContext { + auth: A, + mode: McpMode, + headers: http::HeaderMap, +} + impl Runner { /// Create a new Runner with the given backend pub fn new(backend: B) -> Self { Self { backend: Arc::new(backend) } } - /// Extract authentication and the workspace mode from request context + /// Extract authentication, the workspace mode and the HTTP request itself + /// from the request context fn extract_context( context: &RequestContext, - ) -> Result<(B::Auth, McpMode), ErrorData> { + ) -> Result, ErrorData> { let http_parts = context.extensions.get::().ok_or_else(|| { tracing::error!("http::request::Parts not found"); ErrorData::internal_error("http::request::Parts not found", None) @@ -148,7 +158,7 @@ impl Runner { McpMode::Single(workspace_id) }; - Ok((auth.clone(), mode)) + Ok(McpContext { auth: auth.clone(), mode, headers: http_parts.headers.clone() }) } } @@ -391,6 +401,18 @@ fn authorize_endpoint_call( Ok(()) } +/// Map the model's argument keys back to the runnable's original parameter names. +fn transform_call_args(args: Value, item_schema: &Option) -> Value { + let Value::Object(map) = args else { + return args; + }; + let mut args_hash = HashMap::new(); + for (k, v) in map { + args_hash.insert(reverse_transform_key(&k, item_schema), v); + } + Value::Object(args_hash.into_iter().collect()) +} + fn find_matching_path(candidates: Vec, request_name: &str) -> Option { candidates .into_iter() @@ -427,7 +449,7 @@ impl ServerHandler for Runner { _request: Option, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, .. } = Self::extract_context(&context)?; // Parse MCP scopes to determine what to expose let scopes = auth.scopes().unwrap_or(&[]); @@ -455,7 +477,7 @@ impl ServerHandler for Runner { request: CallToolRequestParams, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, headers } = Self::extract_context(&context)?; // Parse MCP scopes for authorization let scopes = auth.scopes().unwrap_or(&[]); @@ -464,6 +486,7 @@ impl ServerHandler for Runner { let read_only = auth.read_only(); let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); + let mcp_request = McpRequest { headers: &headers, tool_name: request.name.as_ref() }; // Every tool here runs to completion in one round trip: none of them ask the // client for input, so the MRTR variants of `CallToolResponse` are never built. @@ -474,14 +497,22 @@ impl ServerHandler for Runner { &workspace_id, &scope_config, read_only, - request.name, + request.name.clone(), args, + &mcp_request, ) .await } McpMode::Multi(token) => { - self.call_tool_multi(&auth, &token, &scope_config, read_only, request.name, args) - .await + self.call_tool_multi( + &auth, + &token, + &scope_config, + read_only, + request.name.clone(), + args, + ) + .await } }?; Ok(result.into()) @@ -665,6 +696,7 @@ impl Runner { read_only: bool, name: std::borrow::Cow<'static, str>, args: Value, + request: &McpRequest<'_>, ) -> Result { // Check if this is an endpoint tool let endpoint_tools = self.backend.all_endpoint_tools(); @@ -777,17 +809,7 @@ impl Runner { .map_err(|e| ErrorData::internal_error(e.message, None))? }; - // Transform arguments back to original key names - let transformed_args = if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); - for (k, v) in map { - let original_key = reverse_transform_key(&k, &item_schema); - args_hash.insert(original_key, v); - } - Value::Object(args_hash.into_iter().collect()) - } else { - args - }; + let transformed_args = transform_call_args(args, &item_schema); let script_or_flow_path = if is_hub { format!("hub/{}", path) @@ -798,11 +820,23 @@ impl Runner { // Execute script or flow let result = if tool_type == "script" { self.backend - .run_script(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_script( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await } else { self.backend - .run_flow(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_flow( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await }; diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c7c119a5a7..8ac4fd390a 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4007,33 +4007,10 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { { let maybe_new_id = match j.kind { - JobKind::Dependencies => { - let deployment_message = j - .args - .clone() - .map(|hashmap| { - hashmap - .get("deployment_message") - .map(|map_value| serde_json::from_str::(map_value.get()).ok()) - .flatten() - }) - .flatten(); - - // This way we tell downstream which script we should archive when the resolution is finished. - // (not used at the moment) - j.args - .as_mut() - .map(|args| args.insert("base_hash".to_owned(), to_raw_value(&*base_hash))); - - windmill_common::scripts::clone_script( - j.runnable_path(), - &j.workspace_id, - deployment_message, - db, - ) - .await? - .new_hash - } + // A script gets its new version from the worker, once the generated lock is known + // to differ from the live version's: minting one here would deploy, and walk the + // importers of, a version whose lock turns out byte-identical to its parent's. + JobKind::Dependencies => *base_hash, JobKind::FlowDependencies => { sqlx::query_scalar!( "INSERT INTO flow_version diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index b107d34be4..f9bfe2f4df 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -130,6 +130,12 @@ pub struct EditResourceType { pub schema: Option, pub description: Option, pub is_fileset: Option, + /// Doubly optional so an edit can distinguish the two things a plain + /// `Option` conflates: an absent field leaves the extension alone, while an + /// explicit `null` clears it. A hub pull relies on both — a type that stops + /// being a file type has to stop being one locally too. + #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + pub format_extension: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -2803,10 +2809,42 @@ async fn update_resource_type( if let Some(is_fileset) = ns.is_fileset { sqlb.set("is_fileset", if is_fileset { "TRUE" } else { "FALSE" }); } + if let Some(format_extension) = ns.format_extension.clone() { + match format_extension { + Some(ext) => sqlb.set_str("format_extension", ext), + None => sqlb.set("format_extension", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + // Creation refuses the pair outright, so an edit must too — otherwise the same + // impossible type (a set of files that is also one file) is reachable by setting + // either half on an existing row. Whichever half the request omits is read from + // the row being edited, inside this transaction and with the row locked: read + // outside it, two concurrent edits each supplying one half would both pass. + let current = sqlx::query!( + "SELECT is_fileset, format_extension FROM resource_type + WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + &name, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + let effective_is_fileset = ns + .is_fileset + .unwrap_or_else(|| current.as_ref().map(|c| c.is_fileset).unwrap_or(false)); + let effective_format_extension = match &ns.format_extension { + Some(value) => value.clone(), + None => current.and_then(|c| c.format_extension), + }; + if effective_is_fileset && effective_format_extension.is_some() { + return Err(Error::BadRequest( + "A fileset resource type cannot have a format_extension".to_string(), + )); + } + sqlx::query(&sql).execute(&mut *tx).await?; audit_log( &mut *tx, diff --git a/backend/windmill-types/src/more_serde.rs b/backend/windmill-types/src/more_serde.rs index d4b648d2d5..f9cbe739c5 100644 --- a/backend/windmill-types/src/more_serde.rs +++ b/backend/windmill-types/src/more_serde.rs @@ -38,6 +38,25 @@ pub fn is_default(t: &T) -> bool { &T::default() == t } +pub fn maybe_number<'de, T, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: FromStr + serde::Deserialize<'de>, + ::Err: Display, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumericOrString { + String(String), + RawT(T), + } + + match NumericOrString::::deserialize(deserializer)? { + NumericOrString::String(s) => T::from_str(&s).map_err(serde::de::Error::custom), + NumericOrString::RawT(i) => Ok(i), + } +} + pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -66,3 +85,60 @@ where NumericOrNull::Null => Ok(None), } } + +/// Deserializer for a doubly-optional field, so a struct can tell an absent key +/// (`None`) from an explicit `null` (`Some(None)`). +/// +/// Plain serde collapses both into the outer `None`, which makes the distinction +/// unusable exactly where it matters: a payload that omits a field means "leave it +/// alone", while one that sends `null` means "clear it". +/// +/// ```ignore +/// #[serde(default, deserialize_with = "double_option", skip_serializing_if = "Option::is_none")] +/// pub field: Option>, +/// ``` +pub fn double_option<'de, T, D>(deserializer: D) -> Result>, D::Error> +where + T: serde::Deserialize<'de>, + D: serde::Deserializer<'de>, +{ + serde::Deserialize::deserialize(deserializer).map(Some) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + #[derive(Deserialize)] + struct WithMaybeNumber { + #[serde(deserialize_with = "super::maybe_number")] + n: i64, + } + + #[test] + fn maybe_number_accepts_number() { + let v: WithMaybeNumber = serde_json::from_value(serde_json::json!({ "n": 12345 })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_accepts_string() { + let v: WithMaybeNumber = + serde_json::from_value(serde_json::json!({ "n": "12345" })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_rejects_non_numeric_string() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": "abc" })).is_err() + ); + } + + #[test] + fn maybe_number_rejects_null() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": null })).is_err() + ); + } +} diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 414650e846..19f6a06021 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -16,7 +16,7 @@ use crate::{ runnable_settings::{ConcurrencySettings, DebouncingSettings}, }; -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct ScriptModule { pub content: String, pub language: ScriptLang, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 57b859075a..9bba9a99e6 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -19,14 +19,17 @@ use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::jobs::JobKind; use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; -use windmill_common::scripts::ScriptHash; +use windmill_common::scripts::{ + deploy_relocked_version, fetch_script_for_update, hash_script, ScriptHash, ScriptModule, +}; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; -use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; +use windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -38,8 +41,10 @@ use windmill_common::{ scripts::ScriptLang, DB, }; +use windmill_dep_map::lock_hash::record_lock_hashes; pub use windmill_dep_map::{ extract_referenced_paths, extract_relative_imports, process_relative_imports, + refresh_dependency_map, }; use windmill_git_sync::{ handle_deployment_metadata, tally_deployed_object_changes, DeployedObject, @@ -86,10 +91,13 @@ use crate::{ /// has the toolchain and, since the cache key is per OS/arch, the platform the runtime /// workers use. Deploys that supply their own lock never reach a dependency job at all and /// queue theirs from `create_script_internal` instead. -async fn maybe_queue_binary_prebuild(db: &DB, job: &MiniPulledJob, lock: &str) -> Result<()> { - let (Some(hash), Some(path), Some(lang)) = - (job.runnable_id, job.runnable_path.clone(), job.script_lang) - else { +async fn maybe_queue_binary_prebuild( + db: &DB, + job: &MiniPulledJob, + hash: ScriptHash, + lock: &str, +) -> Result<()> { + let (Some(path), Some(lang)) = (job.runnable_path.clone(), job.script_lang) else { return Ok(()); }; let Some(prebuild) = @@ -283,6 +291,13 @@ pub async fn handle_dependency_job( job.runnable_path() ); let script_path = job.runnable_path(); + let w_id = &job.workspace_id; + + let triggered_by_relative_import = job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default(); // A build pass reads the same script data but writes none of the deploy state below, // including the `lock_error_logs` stamp on a fetch failure: the version it builds is @@ -296,14 +311,48 @@ pub async fn handle_dependency_job( *deployment_tallied = true; } + // A relative-import relock deploys nothing until `commit_relock` says so, while the + // caller's fallback tally assumes a failed dependency job left a deployed version behind. + // Claim the tally here; the failure path hands it back once it has minted the version + // that carries the error. + if triggered_by_relative_import { + *deployment_tallied = true; + } + + // A relative-import relock locks the path's live version as of now, not the hash captured + // when the job was pushed: a deploy can land during the debounce delay, after which that + // hash names an archived version. What it generates is committed against the live version + // re-read under a row lock, so a deploy landing mid-generation is caught there too. + let target_hash = if triggered_by_relative_import { + Some(ScriptHash(live_head_hash(db, w_id, script_path).await?)) + } else { + job.runnable_id + }; + // `JobKind::Dependencies` job store either: // - A saved script `hash` in the `script_hash` column. // - Preview raw lock and code in the `queue` or `job` table. - let script_data = &match job.runnable_id { + let script_data = &match target_hash { + // Read straight from the database: the cache pins a version's data under its hash for + // as long as this worker lives, and the live version may still be waiting on its own + // dependency job's lock, which lands in place. A run resolving to it on this worker + // would then get no lock from the cache at all. + Some(hash) if triggered_by_relative_import => { + let raw = cache::script::fetch_script_from_db(db, hash, std::panic::Location::caller()) + .await?; + Cow::Owned(std::sync::Arc::new(cache::ScriptData { + lock: raw.lock, + code: raw.content, + modules: raw.modules, + })) + } Some(hash) => match cache::script::fetch(&Connection::from(db.clone()), hash).await { Ok(d) => Cow::Owned(d.0), Err(e) => { - if !is_build_job { + // The live version of a relative-import relock is what runs resolve to, and + // `lock_error_logs` on it takes it out of resolution; the job carries the + // error instead, since it deployed nothing. + if !is_build_job && !triggered_by_relative_import { let logs2 = sqlx::query_scalar!( "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2", &job.id, @@ -348,12 +397,6 @@ pub async fn handle_dependency_job( .await; } - let triggered_by_relative_import = job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default(); - // Extract temp_script_refs from job args (path -> hash mapping for temp storage) let temp_script_refs: Option> = job .args @@ -391,20 +434,17 @@ pub async fn handle_dependency_job( ) .await; + let (deployment_message, parent_path) = + get_deployment_msg_and_parent_path_from_args(job.args.clone()); + match content { Ok(content) => { - if job.runnable_id.is_none() { + let Some(current_hash) = target_hash else { // it a one-off raw script dependency job, no need to update the db return Ok(to_raw_value_owned( json!({ "status": "Successful lock file generation", "lock": content }), )); - } - - let current_hash = job.runnable_id.unwrap_or(ScriptHash(0)); - let w_id = &job.workspace_id; - - let (deployment_message, parent_path) = - get_deployment_msg_and_parent_path_from_args(job.args.clone()); + }; // Generate lockfiles for module files (if any). // @@ -441,7 +481,9 @@ pub async fn handle_dependency_job( occupancy_metrics, &raw_workspace_dependencies_o, module.lock.as_deref(), - triggered_by_relative_import, + // A module that was never locked has nothing a skip could hand + // back; the path's lock is the parent script's, not its own. + triggered_by_relative_import && module.lock.is_some(), script_path, None, "script", @@ -464,34 +506,83 @@ pub async fn handle_dependency_job( None }; - // We do not create new row for this update - // That means we can keep current hash and just update lock - // Also store lockfile hash for dependency change detection - let lockfile_hash = windmill_common::scripts::hash_script(&content); - let updated_modules_json = updated_modules - .as_ref() - .and_then(|m| serde_json::to_value(m).ok()); - sqlx::query!( - "WITH update_lock AS ( - UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + let deployed_hash = if triggered_by_relative_import { + match commit_relock( + db, + w_id, + script_path, + current_hash, + Some(&content), + updated_modules.as_ref(), + None, + deployment_message.clone(), ) - INSERT INTO lock_hash (workspace_id, path, lockfile_hash) - VALUES ($3, $4, $5) - ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - &content, - ¤t_hash.0, - w_id, - script_path, - &lockfile_hash, - updated_modules_json - ) - .execute(db) - .await?; + .await? + { + RelockOutcome::Deployed(hash) => hash, + RelockOutcome::Unchanged => { + let log_msg = "\nLock unchanged: no new version deployed"; + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + // The imports may have moved even though the result did not, and the + // map is what this importer's next skip check reads. + refresh_dependency_map( + db, + w_id, + script_path, + &parent_path, + &script_data.code, + &job.script_lang, + ) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock unchanged, no new version deployed", "lock": content }), + )); + } + RelockOutcome::Superseded(head) => { + let log_msg = format!( + "\nVersion {head} was deployed while this lock was generated; discarding it and queueing a relock of that version" + ); + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock generation superseded by a newer version", "lock": content }), + )); + } + } + } else { + // We do not create new row for this update + // That means we can keep current hash and just update lock + // Also store lockfile hash for dependency change detection + let lockfile_hash = windmill_common::scripts::hash_script(&content); + let updated_modules_json = updated_modules + .as_ref() + .and_then(|m| serde_json::to_value(m).ok()); + sqlx::query!( + "WITH update_lock AS ( + UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + ) + INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + VALUES ($3, $4, $5) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + &content, + ¤t_hash.0, + w_id, + script_path, + &lockfile_hash, + updated_modules_json + ) + .execute(db) + .await?; - // `lock` has been updated; invalidate the cache. - // Since only worker that ran this Dependency Job has the cache - // we do not need to think about invalidating cache for other workers. - cache::script::invalidate(current_hash); + // `lock` has been updated; invalidate the cache. + // Since only worker that ran this Dependency Job has the cache + // we do not need to think about invalidating cache for other workers. + cache::script::invalidate(current_hash); + current_hash + }; // The version only became runnable now, so this process still resolves the path to // the one before it. Only the runnable-hash cache: the import-side caches ignore the // lock, so evicting this process' half of that pair here would key a bundle by a @@ -504,7 +595,7 @@ pub async fn handle_dependency_job( &db, &w_id, DeployedObject::Script { - hash: current_hash, + hash: deployed_hash, path: script_path.to_string(), parent_path: parent_path.clone(), }, @@ -565,7 +656,7 @@ pub async fn handle_dependency_job( }); } - if let Err(e) = maybe_queue_binary_prebuild(db, job, &content).await { + if let Err(e) = maybe_queue_binary_prebuild(db, job, deployed_hash, &content).await { tracing::error!(%e, "error queueing the auto-build binary job for {script_path}"); } @@ -583,14 +674,49 @@ pub async fn handle_dependency_job( .await? .flatten() .unwrap_or_else(|| "no logs".to_string()); - sqlx::query!( - "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", - &format!("{logs2}\n{error}"), - &job.runnable_id.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; + let error_logs = format!("{logs2}\n{error}"); + if let (true, Some(hash)) = (triggered_by_relative_import, target_hash) { + // The same shape a failed deploy leaves: a version without a lock that carries + // the error, so it shows on the script while runs keep resolving to the last + // version that has one. Only that version is the caller's fallback to tally; + // one that landed meanwhile owns its own lock, and a commit that failed left + // nothing. + match commit_relock( + db, + w_id, + script_path, + hash, + None, + None, + Some(&error_logs), + deployment_message.clone(), + ) + .await + { + Ok(RelockOutcome::Deployed(_)) => *deployment_tallied = false, + Ok(RelockOutcome::Superseded(_)) => { + if let Err(e) = + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await + { + tracing::error!(%e, "error queueing a relock of {script_path}") + } + } + Ok(RelockOutcome::Unchanged) => {} + Err(e) => { + tracing::error!(%e, "error recording the failed relock of {script_path}") + } + } + } else { + sqlx::query!( + "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", + &error_logs, + &job.runnable_id.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + } Err(Error::ExecutionErr(format!( "Error locking file: {error}\n\nlogs:\n{}", remove_ansi_codes(&logs2) @@ -598,6 +724,136 @@ pub async fn handle_dependency_job( } } } + +/// The version of `script_path` that runs resolve to, which is what a relative-import relock +/// locks. `NotFound` when the path holds none, which a job pushed for a path since archived or +/// deleted reports as its own failure. +async fn live_head_hash(db: &DB, w_id: &str, script_path: &str) -> error::Result { + sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + )) + }) +} + +enum RelockOutcome { + /// A new version carrying the result is the live one. + Deployed(ScriptHash), + /// The live version already holds this lock and these module locks; nothing was written. + Unchanged, + /// The live version is no longer the one the lock was generated for; nothing was written. + Superseded(ScriptHash), +} + +/// Commits what a relative-import relock produced against the path's live version, read under +/// a row lock so relocks of one path serialize on it. +/// +/// A result equal to the live version's lock and module locks writes nothing: the importer's +/// dependencies did not move, and a new version would deploy byte-identical content and then +/// walk its own importers for nothing. A `lock` of `None` is a failed generation and always +/// deploys, as the version that carries the error. +async fn commit_relock( + db: &DB, + w_id: &str, + script_path: &str, + generated_for: ScriptHash, + lock: Option<&str>, + modules: Option<&HashMap>, + lock_error_logs: Option<&str>, + deployment_message: Option, +) -> error::Result { + let mut tx = db.begin().await?; + let mut head = None; + for _ in 0..4 { + head = fetch_script_for_update(script_path, w_id, &mut *tx).await?; + if head.is_some() { + break; + } + // Having waited on the live version's row lock, the statement re-checked that row + // once the holder committed, found it archived, and returned nothing: the successor + // the holder inserted is not in the statement's snapshot. A fresh statement sees it, + // unless yet another writer got there first, so this goes around a few times before + // concluding the path holds no live version. + } + let Some(head) = head else { + return Err(Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + ))); + }; + if head.hash != generated_for { + // A deploy landed while the lock was generated. It carried its own lock or queued its + // own dependency job, and this lock describes content that is no longer live. + return Ok(RelockOutcome::Superseded(head.hash)); + } + let lock_hash_entry = lock.map(|lock| (script_path.to_string(), hash_script(lock))); + if let Some(lock) = lock { + let modules_unchanged = modules.map_or(true, |m| head.modules.as_ref() == Some(m)); + if head.lock.as_deref() == Some(lock) && modules_unchanged { + // The hash row is still written, and under the same row lock: a version deployed + // before lock hashes were recorded has none, so its importers cannot skip until + // it does, and a deploy that takes the lock next must not have the hash it records + // overwritten by this one. + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + return Ok(RelockOutcome::Unchanged); + } + } + let new_hash = deploy_relocked_version( + &mut tx, + head, + deployment_message, + lock, + modules, + lock_error_logs, + ) + .await?; + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + Ok(RelockOutcome::Deployed(ScriptHash(new_hash))) +} + +/// Queues another relative-import relock of `script_path`, through the same push the fan-out +/// uses. The version live now was deployed while a lock was generated for its predecessor; +/// when that deploy was a sibling relock it queued nothing for this path, and the result just +/// discarded may have been the one generated against the current imports. +async fn requeue_relock( + db: &DB, + job: &MiniPulledJob, + script_path: &str, + deployment_message: Option, + parent_path: Option, +) -> error::Result<()> { + let already_visited = job + .args + .as_ref() + .and_then(|x| x.get("already_visited")) + .and_then(|v| serde_json::from_str::>(v.get()).ok()) + .unwrap_or_default(); + trigger_dependents_to_recompute_dependencies( + &job.workspace_id, + vec![DependencyDependent { + importer_path: script_path.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + }], + deployment_message, + parent_path, + &job.permissioned_as_email, + &job.created_by, + &job.permissioned_as, + db, + already_visited, + ) + .await +} + fn remove_ansi_codes(s: &str) -> String { lazy_static::lazy_static! { static ref ANSI_REGEX: regex::Regex = regex::Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap(); @@ -2892,8 +3148,12 @@ async fn try_skip_relock( } // Fetch existing lock based on runnable type - let lock = match runnable_type { - "script" => sqlx::query_scalar!( + let lock = match (runnable_type, existing_lock) { + // A script's module asks with the script's own type and hands over the lock it last + // deployed with. The path's lock below is the parent script's, and a module given + // that loses whatever it resolves on its own. + ("script", Some(module_lock)) => Some(module_lock.to_string()), + ("script", None) => sqlx::query_scalar!( "SELECT lock FROM script WHERE path = $1 AND workspace_id = $2 AND lock IS NOT NULL AND deleted = false ORDER BY created_at DESC LIMIT 1", base_path, @@ -2903,7 +3163,7 @@ async fn try_skip_relock( .await? .flatten(), - "flow" | "app" => existing_lock.map(|s| s.to_string()), + ("flow" | "app", existing_lock) => existing_lock.map(|s| s.to_string()), _ => None, }; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 9085d6a365..6a165901e4 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.800.1"; +export const VERSION = "v1.803.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index b45666a1e1..2dac8639e2 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -603,7 +603,7 @@ async function dev(opts: DevOptions, appFolder?: string) { build.onLoad( { filter: /.*/, namespace: "wmill-virtual" }, (args: any) => { - const contents = wmillTs(port); + const contents = wmillTs(); log.info( colors.yellow( `[wmill-virtual] Loading virtual module: ${args.path}`, diff --git a/cli/src/commands/app/wmillTsDev.ts b/cli/src/commands/app/wmillTsDev.ts index 7cb2ea328b..821060d8e4 100644 --- a/cli/src/commands/app/wmillTsDev.ts +++ b/cli/src/commands/app/wmillTsDev.ts @@ -1,5 +1,5 @@ //comment this line and last to dev -export function wmillTsDev(port: number) { return ` +export function wmillTsDev() { return ` let reqs: Record = {} let ws: WebSocket | null = null let wsReady: Promise @@ -10,7 +10,7 @@ function initWebSocket() { wsReadyResolve = resolve }) - ws = new WebSocket('ws://localhost:${port}') + ws = new WebSocket((window.location.protocol === 'https:' ? 'wss:' : 'ws:') + '//' + window.location.host) ws.onopen = () => { console.log('[wmill] WebSocket connected') @@ -157,4 +157,4 @@ export function streamJob( ws?.send(JSON.stringify({ jobId, type: 'streamJob', reqId })) }) } -`} \ No newline at end of file +`} diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 6ace1eaf48..7a383fcfeb 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -1144,7 +1144,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", @@ -1162,7 +1162,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index 1a1f144de1..f752147811 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -17,6 +17,9 @@ interface HubResourceType { app: string; description: string; is_fileset?: boolean; + // Absent from hubs predating the column, so a missing value is "ordinary type", + // not "unset it". + format_extension?: string | null; } export async function pull(opts: GlobalOptions) { @@ -116,7 +119,8 @@ export async function pull(opts: GlobalOptions) { typeof y.schema !== "string" && deepEqual(y.schema, x.schema) && y.description === x.description && - (y.is_fileset ?? false) === (x.is_fileset ?? false) + (y.is_fileset ?? false) === (x.is_fileset ?? false) && + (y.format_extension ?? null) === (x.format_extension ?? null) ) ) { log.info("skipping " + x.name + " (same as current)"); diff --git a/cli/src/commands/resource-type/resource-type.ts b/cli/src/commands/resource-type/resource-type.ts index 96c8429eb4..fd4b72108e 100644 --- a/cli/src/commands/resource-type/resource-type.ts +++ b/cli/src/commands/resource-type/resource-type.ts @@ -25,6 +25,9 @@ export interface ResourceTypeFile { schema?: any; description?: string; is_fileset?: boolean; + // Extension for a type whose value is one file rather than a set of fields; it + // is what makes the resource editor a file editor for that language. + format_extension?: string | null; } export async function pushResourceType( diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index edbedc2612..2eb6ba42b8 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -106,7 +106,8 @@ export async function pushSchedule( path: string, schedule: Schedule | ScheduleFile | undefined, localSchedule: ScheduleFile, - permissionedAsContext?: PermissionedAsContext + permissionedAsContext?: PermissionedAsContext, + enabledOwnedByParent?: boolean ): Promise { path = removeType(path, "schedule").replaceAll(SEP, "/"); log.debug(`Processing local schedule ${path}`); @@ -123,6 +124,21 @@ export async function pushSchedule( // Strip CLI-only boolean marker before sending to API delete (localSchedule as any).has_permissioned_as; + // In a fork, the file's `enabled` is the parent's for a path the parent + // also has (see sync push's `parentOwnedScheduleEnabled`): the fork's own + // flag stays as it is. + if (enabledOwnedByParent && schedule) { + if ( + localSchedule.enabled !== undefined && + localSchedule.enabled !== schedule.enabled + ) { + log.warnAlways( + `Schedule ${path} stays ${schedule.enabled ? "enabled" : "disabled"}: the file says ${localSchedule.enabled ? "enabled" : "disabled"}, but in a fork that flag is the parent workspace's` + ); + } + delete localSchedule.enabled; + } + const preserveFields: { permissioned_as?: string; preserve_permissioned_as?: boolean } = {}; if (permissionedAsContext?.userIsAdminOrDeployer) { if (schedule) { @@ -153,13 +169,9 @@ export async function pushSchedule( ...preserveFields, }, }); - // Tarball export from a fork strips `enabled` from schedule YAMLs so - // the fork→parent git-sync round-trip can't flip the parent's state. - // Skip the secondary setScheduleEnabled call when the local YAML - // doesn't carry `enabled` — sending `{ enabled: undefined }` would - // serialize to `{}` and the backend (`SetEnabled.enabled` is required) - // would reject the request. Preserving the target's existing flag is - // exactly the round-trip-safe behavior. + // No `enabled` in the file (absent from the YAML, or set aside above) + // leaves the remote flag alone: `SetEnabled.enabled` is required, so + // `{ enabled: undefined }` would be rejected rather than ignored. if ( localSchedule.enabled !== undefined && localSchedule.enabled !== schedule.enabled @@ -167,13 +179,12 @@ export async function pushSchedule( log.info(colors.bold.yellow( `Schedule ${path} is ${localSchedule.enabled ? "enabled" : "disabled"} locally but not on remote, updating remote` )); - await wmill.setScheduleEnabled({ - workspace: workspace, + await setEnabledUnlessParentOwned( + workspace, path, - requestBody: { - enabled: localSchedule.enabled, - }, - }); + localSchedule.enabled, + schedule.enabled + ); } } catch (e) { console.error((e as any).body); @@ -194,6 +205,44 @@ export async function pushSchedule( console.error((e as any).body); throw e; } + // A create in a fork lands disabled whatever the request says. A fork-only + // path the file wants enabled is enabled here, so one push converges; a + // parent-owned one stays disabled. + if (enabledOwnedByParent !== undefined && localSchedule.enabled === true) { + if (enabledOwnedByParent) { + log.warnAlways( + `Schedule ${path} created disabled: the file says enabled, but in a fork that flag is the parent workspace's` + ); + } else { + await setEnabledUnlessParentOwned(workspace, path, true, false); + } + } + } +} + +// The parent listing behind `enabledOwnedByParent` sees only what the pusher +// may read; the backend's `fork-conflict` refusal is the last word, so a path +// it says the parent has keeps the fork's flag rather than failing the push. +async function setEnabledUnlessParentOwned( + workspace: string, + path: string, + enabled: boolean, + remoteEnabled: boolean +): Promise { + try { + await wmill.setScheduleEnabled({ + workspace, + path, + requestBody: { enabled }, + }); + } catch (e) { + const conflict = parseForkConflict(e); + if (!conflict) { + throw e; + } + log.warnAlways( + `Schedule ${path} left ${remoteEnabled ? "enabled" : "disabled"}: the parent workspace '${conflict.parentWorkspaceId}' has the same schedule, so its flag is the parent's to set` + ); } } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 8c93bdb963..fada434e0d 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -2213,7 +2213,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", @@ -2231,7 +2231,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6aaa72da0d..bab9967b3c 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -25,7 +25,7 @@ import { } from "yaml"; import JSZip from "jszip"; import { minimatch } from "minimatch"; -import { yamlParseContent } from "../../utils/yaml.ts"; +import { yamlParseContent, yamlParseFile } from "../../utils/yaml.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { @@ -1138,7 +1138,7 @@ export function rawAppPathWithinFolder( return resolved; } -function ZipFSElement( +export function ZipFSElement( zip: JSZip, useYaml: boolean, defaultTs: "bun" | "deno", @@ -1146,6 +1146,14 @@ function ZipFSElement( resourceTypeToIsFileset: Record, ignoreCodebaseChanges: boolean, stripOnBehalfOf: boolean, + // Names a flow's rendered inline-script files after the checkout's own + // `!inline` references (module id -> file). The export carries script + // source, never a reference, so without a checkout to defer to every file + // is named from the step summary, and a file the checkout names otherwise + // reads as a delete + add on every push while the resolved flows are equal. + localFlowInlineMapping?: ( + flowDir: string, + ) => Promise>, ): DynFSElement { // Pre-scan: find zip base paths of scripts that have modules. // These scripts use the folder layout: {basePath}__mod/script.{ext} @@ -1249,59 +1257,71 @@ function ZipFSElement( log.error(`Failed to parse flow.yaml at path: ${p}`); throw error; } - let inlineScripts; + let inlineScripts: InlineScript[]; try { - const assigner = newPathAssigner(defaultTs, { - skipInlineScriptSuffix: getNonDottedPaths(), - }); - // Preserve original !inline filenames from the flow to avoid phantom renames - const inlineMapping = extractCurrentMapping( - flow.value.modules as any, - {}, - flow.value.failure_module, - flow.value.preprocessor_module, - ); - inlineScripts = extractInlineScriptsForFlows( - flow.value.modules as any, - inlineMapping, - SEP, - defaultTs, - assigner, - { + // Extraction rewrites the modules' content into `!inline` refs, + // so each attempt works on its own copy of the flow. + const render = ( + source: OpenFlow, + inlineMapping: Record, + ): [OpenFlow, InlineScript[]] => { + const f: OpenFlow = structuredClone(source); + const assigner = newPathAssigner(defaultTs, { + skipInlineScriptSuffix: getNonDottedPaths(), + }); + const options = { skipInlineScriptSuffix: getNonDottedPaths(), failOnInlineDirective: true, - }, - ); - if (flow.value.failure_module) { - inlineScripts.push( - ...extractInlineScriptsForFlows( - [flow.value.failure_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { - skipInlineScriptSuffix: getNonDottedPaths(), - failOnInlineDirective: true, - }, - ), - ); - } - if (flow.value.preprocessor_module) { - inlineScripts.push( - ...extractInlineScriptsForFlows( - [flow.value.preprocessor_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { - skipInlineScriptSuffix: getNonDottedPaths(), - failOnInlineDirective: true, - }, - ), + }; + const scripts = extractInlineScriptsForFlows( + f.value.modules as any, + inlineMapping, + SEP, + defaultTs, + assigner, + options, ); + if (f.value.failure_module) { + scripts.push( + ...extractInlineScriptsForFlows( + [f.value.failure_module], + inlineMapping, + SEP, + defaultTs, + assigner, + options, + ), + ); + } + if (f.value.preprocessor_module) { + scripts.push( + ...extractInlineScriptsForFlows( + [f.value.preprocessor_module], + inlineMapping, + SEP, + defaultTs, + assigner, + options, + ), + ); + } + return [f, scripts]; + }; + const inlineMapping = localFlowInlineMapping + ? await localFlowInlineMapping(finalPath) + : {}; + let rendered = render(flow, inlineMapping); + // The assigner keeps the names it hands out unique, not the + // checkout's: one of those equal to another step's + // summary-derived name would leave two files at one path, so + // such a flow renders the export's way. + if ( + new Set(rendered[1].map((s) => s.path)).size !== + rendered[1].length + ) { + rendered = render(flow, {}); } + [flow, inlineScripts] = rendered; } catch (error) { log.error( `Failed to extract inline scripts for flow at path: ${p}`, @@ -2581,7 +2601,7 @@ export function preservePendingScriptLocks( } } -async function compareDynFSElement( +export async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, ignore: (path: string, isDirectory: boolean) => boolean, @@ -2594,6 +2614,9 @@ async function compareDynFSElement( branchOverride?: string, isEls1Remote?: boolean, caseInsensitiveFs?: boolean, + // Which schedule files carry an `enabled` that is not the target's to set + // (see push's `parentOwnedScheduleEnabled`): those compare without it. + parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, ): Promise<{ changes: Change[]; localMap: Record }> { let [m1, m2] = els2 ? await Promise.all([ @@ -2785,12 +2808,28 @@ async function compareDynFSElement( ); throw error; } + if ( + parentOwnsScheduleEnabled && + getTypeStrFromPath(k) === "schedule" && + parentOwnsScheduleEnabled(k) + ) { + delete parsedV?.enabled; + delete parsedM2?.enabled; + } if (deepEqual(parsedV, parsedM2)) { continue; } } else if (k.endsWith(".yaml")) { const before = parseYaml(k, m2[k]); const after = parseYaml(k, v); + if ( + parentOwnsScheduleEnabled && + getTypeStrFromPath(k) === "schedule" && + parentOwnsScheduleEnabled(k) + ) { + delete before?.enabled; + delete after?.enabled; + } if (deepEqual(before, after)) { continue; } @@ -4543,6 +4582,86 @@ async function checkServerLockJobs( } } +// The checkout's `!inline` references of one flow (module id -> file), as +// `ZipFSElement`'s `localFlowInlineMapping` names the remote render. Empty +// when the flow has no local flow.yaml. +export async function checkoutInlineNames( + flowYamlPath: string, +): Promise> { + let flow: any; + try { + flow = await yamlParseFile(flowYamlPath); + } catch { + return {}; + } + const mapping = extractCurrentMapping( + flow?.value?.modules, + {}, + flow?.value?.failure_module, + flow?.value?.preprocessor_module, + ); + // A reference that leaves the flow folder would render the remote step + // onto another item's path; such a step keeps its summary-derived name. + for (const [id, ref] of Object.entries(mapping)) { + if (path.isAbsolute(ref) || ref.split(/[\\/]/).includes("..")) { + delete mapping[id]; + } + } + return mapping; +} + +// For a path the parent also has, a fork's export writes the parent's +// `enabled` and the backend refuses to enable the fork's copy: the file's flag +// is the parent's. A parent that cannot be listed (a fork-scoped job token) +// may own every path. Undefined when the target is not a fork. +async function parentOwnedScheduleEnabled( + workspaceId: string, +): Promise<((scheduleFilePath: string) => boolean) | undefined> { + let parentWorkspaceId: string | null | undefined; + let known = false; + try { + const { workspaces } = await wmill.listUserWorkspaces(); + const entry = workspaces?.find((w) => w.id === workspaceId); + known = entry !== undefined; + parentWorkspaceId = entry?.parent_workspace_id; + } catch { + // A fork-scoped token cannot list workspaces. + } + // No parent on record (a fork whose parent was deleted keeps its + // `wm-fork-` id): nothing defers to a parent any more. + if (known && !parentWorkspaceId) { + return undefined; + } + // Without the listing only the `wm-fork-` prefix says fork: a dev + // workspace (custom id) reached with a fork-scoped token counts as none. + if (!isForkWorkspace(workspaceId, parentWorkspaceId)) { + return undefined; + } + let parentPaths: Set | undefined; + if (parentWorkspaceId) { + try { + parentPaths = new Set(); + const perPage = 100; + for (let page = 1; ; page++) { + const batch = await wmill.listSchedules({ + workspace: parentWorkspaceId, + page, + perPage, + }); + batch.forEach((s) => parentPaths!.add(s.path)); + if (batch.length < perPage) break; + } + } catch { + parentPaths = undefined; + } + } + return (scheduleFilePath) => + parentPaths === undefined || + parentPaths.has( + removeType(scheduleFilePath, "schedule").replaceAll(SEP, "/"), + ); +} + export async function push( opts: GlobalOptions & SyncOptions & { @@ -4633,6 +4752,9 @@ export async function push( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + const parentOwnsScheduleEnabled = opts.includeSchedules + ? await parentOwnedScheduleEnabled(workspace.workspaceId) + : undefined; if (opts.lint) { log.info("Running lint validation before push..."); @@ -4696,6 +4818,10 @@ export async function push( // ignore } + // See ZipFSElement's `localFlowInlineMapping`. + const localFlowInlineMapping = (flowDir: string) => + checkoutInlineNames(path.join(process.cwd(), flowDir, "flow.yaml")); + const remote = ZipFSElement( (await downloadZip( workspace, @@ -4721,6 +4847,7 @@ export async function push( resourceTypeToIsFileset, false, parseSyncBehavior(opts.syncBehavior) >= 1, + localFlowInlineMapping, ); const local = await FSFSElement( @@ -4741,6 +4868,7 @@ export async function push( wsNameForFiles, false, // els1 (local) is not the remote source await isCaseInsensitiveFilesystem(process.cwd()), + parentOwnsScheduleEnabled, ); // Detect resources/variables that the local config flags as ws_specific @@ -5807,6 +5935,9 @@ export async function push( originalLocalPath: originalWorkspaceSpecificPath, permissionedAsContext, wsSpecific: isWsSpecific ? true : undefined, + enabledOwnedByParent: parentOwnsScheduleEnabled?.( + change.path, + ), keyPushOpts: { noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, @@ -5942,6 +6073,9 @@ export async function push( originalLocalPath: localFilePath, permissionedAsContext, wsSpecific: isAddedWsSpecific ? true : undefined, + enabledOwnedByParent: parentOwnsScheduleEnabled?.( + change.path, + ), keyPushOpts: { noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 52b000d4c2..7017136a66 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.800.1"; +export const VERSION = "1.803.0"; diff --git a/cli/src/core/log.ts b/cli/src/core/log.ts index e78450597f..c4cf256e9d 100644 --- a/cli/src/core/log.ts +++ b/cli/src/core/log.ts @@ -41,6 +41,12 @@ export function warnStderr(msg: unknown) { console.error(`\x1b[33m${String(msg)}\x1b[39m`); } +// A notice that must reach a log even in silent (`--json-output`) mode: +// stderr keeps stdout parseable, and a job that runs the CLI records both. +export function warnAlways(msg: unknown) { + console.error(`\x1b[33m${String(msg)}\x1b[39m`); +} + export function error(msg: unknown) { console.error(`\x1b[31m${String(msg)}\x1b[39m`); } diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 100bea6e53..2a9bb34fe0 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -115,6 +115,8 @@ Local previews exist for every entity type and don't deploy: - \`wmill flow preview -d ''\` — run a local flow.yaml. - \`wmill app dev\` — live-reload dev server for raw apps. +An argument typed as a resource takes the bare string \`"$res:"\` as its whole value (a variable takes \`"$var:"\`) — never an object wrapper like \`{"$res": ""}\`, and never a plain path. See the \`resources\` skill. + Argument shapes and per-language details live in the \`write-script-\`, \`write-flow\`, and \`raw-app\` skills. ## Keeping metadata in sync diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 62027ab2ef..5271167a07 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5116,6 +5116,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute \`wmill flow preview -d ''\` directly — pick plausible args from the flow's input schema. +An input typed as a resource (\`format: resource-\` in the schema) takes the bare string \`"$res:"\` as its whole value — \`-d '{"db": "$res:f/databases/postgres_prod"}'\`, not \`{"db": {"$res": "..."}}\` and not a plain path. Same for a variable, with \`"$var:"\`. See the \`resources\` skill. + \`wmill flow preview\` is safe to run yourself (it does not deploy). \`wmill generate-metadata\` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's \`AGENTS.md\` opts into automatic metadata. After running it, check the regenerated \`.lock\` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only \`wmill sync push\` deploys; run it only when the user explicitly asks. ### Visual preview @@ -6188,6 +6190,41 @@ Reference other resources: } \`\`\` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema \`format: resource-\`) is passed as +the **bare string** \`$res:\` — the whole argument value. Same for a variable, with +\`$var:\`. This applies everywhere job arguments are supplied: \`wmill script run/preview\`, +\`wmill flow run/preview\`, the \`runScriptByPath\` / \`runFlowByPath\` API, a schedule's \`args\`, a +trigger's configured static args. + +\`\`\`json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +\`\`\` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with \`$res:\` / \`$var:\`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +\`\`\`json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +\`\`\` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(\`{ "gh_auth": { "token": "$var:g/all/gh_token" } }\`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL @@ -7074,11 +7111,11 @@ flow related commands - \`flow push \` - push a local flow spec. This overrides any remote versions. - \`--message \` - Deployment message - \`flow run \` - run a flow by path. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`--step \` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -7474,11 +7511,11 @@ script related commands - \`--json\` - Output as JSON (for piping to jq) - \`script show \` - show a script's content (alias for get) - \`script run \` - run a script by path - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script preview \` - preview a local script without deploying it. Supports both regular and codebase scripts. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other than the final output. Useful for scripting. - \`--tag \` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script new \` - create a new script diff --git a/cli/src/types.ts b/cli/src/types.ts index c9244a8f4c..9c517ff3c7 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -189,6 +189,8 @@ export interface PushObjOptions { keyPushOpts?: PushWorkspaceKeyOptions; /** TypeScript runtime a bare `.ts` denotes, for raw-app runnables */ defaultTs?: "bun" | "deno"; + /** schedule push into a fork: the file's `enabled` is the parent's */ + enabledOwnedByParent?: boolean; } /** @@ -217,6 +219,7 @@ export async function pushObj( wsSpecific, keyPushOpts, defaultTs, + enabledOwnedByParent, } = opts; const typeEnding = getTypeStrFromPath(p); @@ -250,7 +253,7 @@ export async function pushObj( } else if (typeEnding === "resource-type") { await pushResourceType(workspace, p, befObj, newObj); } else if (typeEnding === "schedule") { - await pushSchedule(workspace, p, befObj, newObj, permissionedAsContext); + await pushSchedule(workspace, p, befObj, newObj, permissionedAsContext, enabledOwnedByParent); } else if (typeEnding === "http_trigger") { await pushTrigger("http", workspace, p, befObj, newObj, permissionedAsContext); } else if (typeEnding === "websocket_trigger") { diff --git a/cli/test/push_diff_convergence_unit.test.ts b/cli/test/push_diff_convergence_unit.test.ts new file mode 100644 index 0000000000..14e248c03e --- /dev/null +++ b/cli/test/push_diff_convergence_unit.test.ts @@ -0,0 +1,298 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import JSZip from "jszip"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, sep } from "node:path"; +import { + checkoutInlineNames, + compareDynFSElement, + ZipFSElement, +} from "../src/commands/sync/sync.ts"; + +// The differ also reads the working tree (shared lockfiles, dependency files); +// an empty one keeps that out of the picture. +const originalCwd = process.cwd(); +beforeAll(() => { + process.chdir(mkdtempSync(join(tmpdir(), "wmill-push-diff-"))); +}); +afterAll(() => { + process.chdir(originalCwd); +}); + +// A push is only useful when a second run of it finds nothing left to do. +// These pin the two shapes that used to be listed on every run of a push into +// a fork while the push itself either applied nothing or aborted. + +type Mock = { + isDirectory: boolean; + path: string; + getContentText(): Promise; + getChildren(): AsyncIterable; +}; + +// Both sides of the differ use the OS separator; fixtures are written with +// "/" and rows are read back the same way. +const osPath = (p: string) => p.split("/").join(sep); +const slashPath = (p: string) => p.split(sep).join("/"); + +function local(files: Record): Mock { + return { + isDirectory: true, + path: "", + async getContentText() { + return ""; + }, + async *getChildren() { + for (const [path, content] of Object.entries(files)) { + yield { + isDirectory: false, + path: osPath(path), + async getContentText() { + return content; + }, + async *getChildren() {}, + }; + } + }, + }; +} + +const noIgnore = () => false; + +async function diff( + localEl: Mock, + remoteEl: Mock, + skips: Record, + parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, +) { + const { changes } = await compareDynFSElement( + localEl as any, + remoteEl as any, + noIgnore, + false, + skips as any, + true, + [], + false, + undefined, + undefined, + false, + false, + parentOwnsScheduleEnabled, + ); + return changes.map((c) => `${c.name} ${slashPath(c.path)}`); +} + +const SCHEDULE = (enabled: string) => + `summary: nightly\nargs: {}\nenabled: ${enabled}\nis_flow: true\nschedule: 0 0 0 * * *\nscript_path: f/mail/flow\ntimezone: UTC\n`; + +test("push into a fork: a schedule the parent also has compares without `enabled`", async () => { + const remote = local({ "f/mail/nightly.schedule.yaml": SCHEDULE("true") }); + const skips = { includeSchedules: true }; + // The parent has `f/mail/nightly`; any other schedule is the fork's own. + const parentHas = (filePath: string) => + slashPath(filePath) === "f/mail/nightly.schedule.yaml"; + + // Not a fork: `enabled` is compared like any other field. + expect( + await diff( + local({ "f/mail/nightly.schedule.yaml": SCHEDULE("false") }), + remote, + skips, + ), + ).toEqual(["edited f/mail/nightly.schedule.yaml"]); + expect( + await diff( + local({ "f/mail/nightly.schedule.yaml": SCHEDULE("false") }), + remote, + skips, + parentHas, + ), + ).toEqual([]); + // The key being absent is the same case as it differing. + expect( + await diff( + local({ + "f/mail/nightly.schedule.yaml": SCHEDULE("false").replace( + "enabled: false\n", + "", + ), + }), + remote, + skips, + parentHas, + ), + ).toEqual([]); + // Only `enabled` is set aside. + expect( + await diff( + local({ + "f/mail/nightly.schedule.yaml": SCHEDULE("false").replace( + "0 0 0 * * *", + "0 0 1 * * *", + ), + }), + remote, + skips, + parentHas, + ), + ).toEqual(["edited f/mail/nightly.schedule.yaml"]); + // A schedule only the fork has keeps toggling from the file. + expect( + await diff( + local({ "f/mail/fork_only.schedule.yaml": SCHEDULE("true") }), + local({ "f/mail/fork_only.schedule.yaml": SCHEDULE("false") }), + skips, + parentHas, + ), + ).toEqual(["edited f/mail/fork_only.schedule.yaml"]); +}); + +const SUMMARY = "process one mail end-to-end (spam check, classify)"; + +function remoteFlow(content: string) { + const zip = new JSZip(); + zip.file( + "f/mail/flow_v2.flow.json", + JSON.stringify({ + summary: "Flow V2", + description: "", + value: { + modules: [ + { + id: "a", + summary: SUMMARY, + value: { + type: "rawscript", + content, + input_transforms: {}, + language: "python3", + }, + }, + ], + }, + schema: { type: "object", properties: {} }, + }), + // The backend's archive carries no directory entries. + { createFolders: false }, + ); + return zip; +} + +function localFlow(content: string) { + return local({ + "f/mail/flow_v2.flow/flow.yaml": `summary: Flow V2\ndescription: ''\nvalue:\n modules:\n - id: a\n summary: ${SUMMARY}\n value:\n type: rawscript\n content: '!inline process_mail.inline_script.py'\n input_transforms: {}\n language: python3\nschema:\n type: object\n properties: {}\n`, + "f/mail/flow_v2.flow/process_mail.inline_script.py": content, + }); +} + +// The checkout's `!inline` references, as `push` reads them from its flow.yaml. +const checkoutNames = async (flowDir: string) => + slashPath(flowDir) === "f/mail/flow_v2.flow" + ? { a: "process_mail.inline_script.py" } + : {}; + +test("push: an inline script the checkout names differently from the step summary is not a rename", async () => { + const skips = { includeSchedules: false }; + const render = (content: string, withCheckout: boolean) => + ZipFSElement( + remoteFlow(content), + true, + "bun", + {}, + {}, + false, + true, + withCheckout ? checkoutNames : undefined, + ) as any; + + // Same content, file named by hand: three rows before, none after. + expect( + await diff( + localFlow("def main():\n return 1\n"), + render("def main():\n return 1\n", false), + skips, + ), + ).toEqual([ + "deleted f/mail/flow_v2.flow/process_one_mail_end-to-end_(spam_check,_classify).inline_script.py", + "edited f/mail/flow_v2.flow/flow.yaml", + "added f/mail/flow_v2.flow/process_mail.inline_script.py", + ]); + expect( + await diff( + localFlow("def main():\n return 1\n"), + render("def main():\n return 1\n", true), + skips, + ), + ).toEqual([]); + + // A real edit is still one. + expect( + await diff( + localFlow("def main():\n return 2\n"), + render("def main():\n return 1\n", true), + skips, + ), + ).toEqual(["edited f/mail/flow_v2.flow/process_mail.inline_script.py"]); +}); + +test("push: a checkout name that collides with another step's summary-derived name keeps two files", async () => { + const zip = new JSZip(); + zip.file( + "f/mail/flow_v2.flow.json", + JSON.stringify({ + summary: "Flow V2", + description: "", + value: { + modules: [ + { + id: "a", + summary: SUMMARY, + value: { + type: "rawscript", + content: "a", + input_transforms: {}, + language: "python3", + }, + }, + { + id: "b", + summary: "process_mail", + value: { + type: "rawscript", + content: "b", + input_transforms: {}, + language: "python3", + }, + }, + ], + }, + schema: { type: "object", properties: {} }, + }), + { createFolders: false }, + ); + // Nothing local: every rendered file is a "deleted" row, one per path. + const rows = await diff( + local({}), + ZipFSElement(zip, true, "bun", {}, {}, false, true, checkoutNames) as any, + { includeSchedules: false }, + ); + expect(rows.filter((r) => r.endsWith(".py")).sort()).toEqual([ + "deleted f/mail/flow_v2.flow/process_mail.inline_script.py", + "deleted f/mail/flow_v2.flow/process_one_mail_end-to-end_(spam_check,_classify).inline_script.py", + ]); +}); + +test("push: checkout inline names stay inside the flow folder", async () => { + const flowYaml = join(process.cwd(), "flow.yaml"); + writeFileSync( + flowYaml, + `summary: x\nvalue:\n modules:\n - id: a\n value:\n type: rawscript\n content: '!inline a.inline_script.py'\n language: python3\n - id: b\n value:\n type: rawscript\n content: '!inline ../shared/b.py'\n language: python3\n - id: c\n value:\n type: rawscript\n content: '!inline /tmp/c.py'\n language: python3\n`, + ); + expect(await checkoutInlineNames(flowYaml)).toEqual({ + a: "a.inline_script.py", + }); + expect( + await checkoutInlineNames(join(process.cwd(), "missing.yaml")), + ).toEqual({}); +}); diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 5d7f0b2c29..5cef3c9364 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,9 +4,9 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 28 registered actions across fourteen features (`ai_session`, `ai_chat`, +It currently carries 32 registered actions across fifteen features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`, -`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`). Nearly all of the +`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, `sso_groups_claim`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 536b178852..6fd6413d59 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.800.1", + "version": "1.803.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.800.1", + "version": "1.803.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index ed8c18768f..f8f952ede6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.800.1", + "version": "1.803.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/aiStore.test.ts b/frontend/src/lib/aiStore.test.ts index fed6147b62..be8980e93a 100644 --- a/frontend/src/lib/aiStore.test.ts +++ b/frontend/src/lib/aiStore.test.ts @@ -35,6 +35,24 @@ describe('setCopilotInfo legacy /thinking migration', () => { expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) }) + it('keeps the models but turns the assistant off when the workspace disabled it', () => { + setCopilotInfo({ + providers: { + anthropic: { + resource_path: 'u/admin/anthropic', + models: ['claude-sonnet-4-6'] + } + }, + copilot_disabled: true + }) + + const info = get(copilotInfo) + expect(info.enabled).toBe(false) + expect(info.workspaceDisabled).toBe(true) + // The providers still describe what AI agent steps can run on. + expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) + }) + it('defaults provider web search on unless explicitly disabled', () => { setCopilotInfo({ providers: { diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 2d6a98c625..6ac98d69e8 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -41,6 +41,10 @@ export const copilotSessionModel = writable( export const copilotInfo = writable<{ enabled: boolean + // The workspace hid the assistant (`ai_config.copilot_disabled`). `enabled` is then false + // whatever the providers say, and the AI entry points that nudge "configure AI" when + // `enabled` is off render nothing at all instead. + workspaceDisabled: boolean codeCompletionModel?: AIProviderModel defaultModel?: AIProviderModel metadataModel?: AIProviderModel @@ -56,6 +60,7 @@ export const copilotInfo = writable<{ freeTier?: FreeTierInfo }>({ enabled: false, + workspaceDisabled: false, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, @@ -71,7 +76,7 @@ export const copilotInfo = writable<{ aiUserDisabled.subscribe((disabled) => { copilotInfo.update((info) => ({ ...info, - enabled: info.aiModels.length > 0 && !disabled + enabled: info.aiModels.length > 0 && !disabled && !info.workspaceDisabled })) }) @@ -126,9 +131,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { return model }) + const workspaceDisabled = aiConfig.copilot_disabled === true copilotInfo.set({ - // Providers are configured; the per-user opt-out is the only thing that can gate it off. - enabled: !get(aiUserDisabled), + // Providers are configured; only the workspace or per-user opt-outs can gate it off. + enabled: !workspaceDisabled && !get(aiUserDisabled), + workspaceDisabled, // Strip the deprecated /thinking suffix from the configured model slots too, // otherwise a workspace whose default still carries it sends an invalid model id. codeCompletionModel: stripModelSuffix(aiConfig.code_completion_model), @@ -146,6 +153,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { copilotInfo.set({ enabled: false, + workspaceDisabled: aiConfig.copilot_disabled === true, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index a9a0837d1f..3d73cea72f 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -21,9 +21,10 @@ } from '$lib/gen' import { emptyString, truncateRev, urlize } from '$lib/utils' import { registryEntryFor, registryCcCapableFor, stripSandboxSuffix } from './oauthRegistry' - import { createEventDispatcher, onDestroy, tick, untrack } from 'svelte' + import { createEventDispatcher, onDestroy, tick } from 'svelte' import Path from './Path.svelte' - import { Button, RadioCard, Skeleton } from './common' + import { ListRow, RadioCard, Skeleton } from './common' + import { useListHighlight } from './common/listRow/listHighlight.svelte' import ApiConnectForm from './ApiConnectForm.svelte' import SearchItems from './SearchItems.svelte' import WhitelistIp from './WhitelistIp.svelte' @@ -42,7 +43,6 @@ import SyncResourceTypes from './SyncResourceTypes.svelte' import Label from './Label.svelte' import ResourcePathHint from './ResourcePathHint.svelte' - import { twMerge } from 'tailwind-merge' interface Props { step?: number @@ -1027,15 +1027,8 @@ // Both lists start undefined and render skeletons; "nothing found" only means something // once they have landed. let listsLoaded = $derived(rankedConnectsManual !== undefined && rankedConnects !== undefined) - let highlightedIndex = $state(-1) const rowDomId = (index: number) => `resource-type-row-${index}` - // Set at hover time rather than up front, so only the descriptions the row actually cut - // off carry a tooltip. - function titleIfTruncated(e: MouseEvent & { currentTarget: HTMLElement }) { - const el = e.currentTarget - el.title = el.scrollWidth > el.clientWidth ? (el.textContent?.trim() ?? '') : '' - } const oauthRowOffset = $derived(customKeys.length) const otherRowOffset = $derived(customKeys.length + (rankedConnects?.length ?? 0)) @@ -1054,53 +1047,23 @@ return best } - // Filtering reshuffles the rows under the highlight: point it at the best match so Enter - // takes the top hit, and drop it entirely once the filter is cleared. - $effect(() => { - navItems - filter - untrack(() => (highlightedIndex = searching ? bestMatchIndex() : -1)) + const highlight = useListHighlight({ + count: () => navItems.length, + rowId: rowDomId, + // Sections are rendered in a fixed order, so the best match is not necessarily the + // first row; Enter should still take the top hit. + restingIndex: () => (searching ? bestMatchIndex() : -1), + onActivate: (index) => { + const item = navItems[index] + if (!item) return + item.oauth ? connectOauth(item.key) : selectFromOthers(item.key) + }, + activateEnterFrom: [SEARCH_INPUT_ID] }) - // Scrolling rows under a resting pointer makes the browser fire `mouseenter` on each one, - // which would drag the highlight back under the cursor as the arrow keys move it. Only a - // real pointer move hands the highlight back to the mouse. - let pointerOwnsHighlight = $state(true) - - function highlightHovered(index: number) { - if (pointerOwnsHighlight) highlightedIndex = index - } - - function moveHighlight(delta: number) { - const count = navItems.length - if (count === 0) return - pointerOwnsHighlight = false - // Rows are tabbable buttons, so focus can sit on one. Enter then activates whatever is - // focused, which has to stay the highlighted row. - const rowWasFocused = document.activeElement?.id?.startsWith('resource-type-row-') ?? false - highlightedIndex = - highlightedIndex < 0 - ? delta > 0 - ? 0 - : count - 1 - : (highlightedIndex + delta + count) % count - const row = document.getElementById(rowDomId(highlightedIndex)) - row?.scrollIntoView({ block: 'nearest' }) - if (rowWasFocused) row?.focus() - } - function onListKeydown(e: KeyboardEvent) { if (step !== 1) return - if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { - e.preventDefault() - moveHighlight(e.key === 'ArrowDown' ? 1 : -1) - } else if (e.key === 'Enter' && (e.target as HTMLElement)?.id === SEARCH_INPUT_ID) { - // A focused row activates itself on Enter; this covers Enter typed in the search field. - const item = navItems[highlightedIndex] - if (!item) return - e.preventDefault() - item.oauth ? connectOauth(item.key) : selectFromOthers(item.key) - } + highlight.onKeydown(e) } let editScopes = $state(false) @@ -1132,7 +1095,7 @@
(pointerOwnsHighlight = true)} + onpointermove={highlight.pointerMoved} >
@@ -1146,28 +1109,6 @@
- {#snippet resourceRow(key: string)} -
-
- -
-
-
- {resourceTypeDisplayName(key)} - {key} -
- {#if resourceTypeDescriptions[key]} - - {plainDescription(resourceTypeDescriptions[key])} - - {/if} -
-
- {/snippet} - {#snippet sectionHeading(title: string, count: number)}

{title}{#if searching}{count}{/if} @@ -1175,26 +1116,29 @@ {/snippet} {#snippet resourceButton(key: string, index: number, oauth: boolean)} - + {icon} + {title} + subtitle={resourceTypeDescriptions[key] ? subtitle : undefined} + highlighted={index === highlight.index} + onMouseEnter={() => highlight.hovered(index)} + onClick={() => (oauth ? connectOauth(key) : selectFromOthers(key))} + /> {/snippet}
@@ -1213,7 +1157,7 @@ {#if customKeys.length > 0}
{@render sectionHeading('Custom resource types', customKeys.length)} -
+
{#each customKeys as key, i} {@render resourceButton(key, i, false)} {/each} @@ -1227,7 +1171,7 @@ 'Instance-configured OAuth APIs', rankedConnects?.length ?? 0 )} -
+
{#if rankedConnects} {#each rankedConnects as { key }, i} {@render resourceButton(key, oauthRowOffset + i, true)} @@ -1259,7 +1203,7 @@
{/if} -
+
{#if rankedConnectsManual} {#each otherKeys as key, i} {@render resourceButton(key, otherRowOffset + i, false)} diff --git a/frontend/src/lib/components/FolderEditor.svelte b/frontend/src/lib/components/FolderEditor.svelte index dede17b953..e8f37a625c 100644 --- a/frontend/src/lib/components/FolderEditor.svelte +++ b/frontend/src/lib/components/FolderEditor.svelte @@ -6,19 +6,23 @@ FolderService, UserService, GranularAclService, - GroupService + GroupService, + type User } from '$lib/gen' - import TableCustom from './TableCustom.svelte' + import DataTable from './table/DataTable.svelte' + import Head from './table/Head.svelte' + import Row from './table/Row.svelte' + import Cell from './table/Cell.svelte' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' - import { Alert, Button, Drawer, DrawerContent } from './common' + import { Alert, Button } from './common' import Skeleton from './common/skeleton/Skeleton.svelte' - import GroupEditor from './GroupEditor.svelte' + import GroupEditorDrawer from './GroupEditorDrawer.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import { ArrowDown, ArrowUp, Eye, Plus, Trash } from 'lucide-svelte' + import { ArrowDown, ArrowUp, Eye, Pen, Plus, Trash } from 'lucide-svelte' import Label from './Label.svelte' import { sendUserToast } from '$lib/toast' - import { createEventDispatcher, untrack } from 'svelte' + import { onMount, tick, untrack } from 'svelte' import Select from './select/Select.svelte' import { safeSelectItems } from './select/utils.svelte' import TextInput from './text_input/TextInput.svelte' @@ -28,97 +32,251 @@ import CollapseLink from './CollapseLink.svelte' import LabelsInput from './LabelsInput.svelte' import Badge from './common/badge/Badge.svelte' + import InputError from './InputError.svelte' + import Popover from './meltComponents/Popover.svelte' + import { deepEqual } from 'fast-equals' + import { + folderPermissionDiff, + isFolderDraftDirty, + type FolderDraft, + type FolderRole + } from '$lib/folderDraft' + + const VALID_FOLDER_NAME = /^[a-zA-Z_0-9-]+$/ + + const ROLE_TOOLTIPS = { + viewer: + 'A viewer of a folder has read-only access to all the elements (scripts/flows/apps/schedules/resources/variables) inside the folder', + writer: + 'A writer of a folder has read AND write access to all the elements (scripts/flows/apps/schedules/resources/variables) inside the folder', + admin: + 'An admin of a folder has read AND write access to all the elements inside the folders and can manage the permissions as well as add new admins' + } + + const MEMBERS_EXPLAINER = + "A member is a user or group with a role on this folder. The role applies to every script, flow, app, resource, variable and schedule inside it: viewers can read them, writers can also edit them, and admins can additionally manage the folder's members." + + // Edits mutate `draft` only; `save()` is the sole writer to the backend, and `baseline` is + // what the folder held when it was loaded, so comparing the two gives both the dirty state + // and the permission calls to replay. Both live in `folderDraft.ts`, with tests. + type Role = FolderRole interface Props { + /** In `new` mode this is the name being typed, hence bindable. */ name: string + mode?: 'edit' | 'new' + /** Drives the parent drawer's Save button, which lives above this component. */ + onCanSaveChange?: (canSave: boolean) => void + /** Drives the parent drawer's discard confirmation on close. Unlike `canSave` + * this stays true for edits that cannot be saved yet (an invalid rule, a name + * already taken) — closing would still throw them away. */ + onUnsavedChange?: (unsaved: boolean) => void + /** False while Save would create rather than update, which an `edit` drawer reaches + * when the folder turns out not to exist. The drawer labels itself from this. */ + onExistsChange?: (exists: boolean) => void + /** Edit a folder of this workspace rather than the active one. The folder picker + * can be aimed elsewhere (the project import wizard picks a destination workspace + * before entering it), and the folder must be written where it was listed. */ + workspace?: string } - let { name }: Props = $props() - let can_write = $state(false) + let { + name = $bindable(), + mode = 'edit', + onCanSaveChange, + onUnsavedChange, + onExistsChange, + workspace + }: Props = $props() - type Role = 'viewer' | 'writer' | 'admin' - let folder: Folder | undefined - let perms: { owner_name: string; role: Role }[] | undefined = $state(undefined) - let usernames: string[] = $state([]) - let groups: string[] = $state([]) - let ownerItem: string = $state('') + const targetWorkspace = $derived(workspace ?? $workspaceStore ?? '') + const aimedElsewhere = $derived(!!workspace && workspace !== $workspaceStore) - let newGroup: Drawer | undefined = $state(undefined) - let viewGroup: Drawer | undefined = $state(undefined) - - async function loadUsernames(): Promise { - usernames = await UserService.listUsernames({ workspace: $workspaceStore! }) - } - - async function loadGroups(): Promise { - groups = await GroupService.listGroupNames({ workspace: $workspaceStore! }) - } - - async function load() { - loadUsernames() - loadGroups() - await loadFolder() - } - - async function addToFolder() { - await GranularAclService.addGranularAcls({ - workspace: $workspaceStore ?? '', - path: name, - kind: 'folder', - requestBody: { - owner: (ownerKind == 'user' ? 'u/' : 'g/') + ownerItem - } - }) - ownerItem = '' - loadFolder() - } - - let folderNotFound: boolean | undefined = $state(undefined) - - async function loadFolder(): Promise { - try { - folder = await FolderService.getFolder({ workspace: $workspaceStore!, name }) - summary = folder.summary ?? '' - labels = [...(folder.labels ?? [])] - defaultPermissionedAs = (folder.default_permissioned_as ?? []).map((r) => ({ ...r })) - can_write = - $userStore != undefined && - (folder?.owners.includes('u/' + $userStore.username) || - ($userStore.is_admin ?? false) || - ($userStore.is_super_admin ?? false) || - $userStore.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1) - - perms = Array.from( - new Set( - Object.entries(folder?.extra_perms ?? {}) - .map((x) => x[0]) - .concat(folder?.owners ?? []) - ) - ).map((x) => { - return { - owner_name: x, - role: getRole(x) + // `$userStore` describes the workspace the app is *in*. Aimed at another one it answers + // the wrong question — a folder admin there would get read-only controls, and a + // non-member would get write ones — so resolve the membership of the workspace being + // edited. `whoami` returns group names unprefixed; `owners` holds them `g/`-prefixed. + let targetUser: User | undefined = $state(undefined) + const membership = $derived.by(() => { + if (!aimedElsewhere) { + return $userStore + ? { + username: $userStore.username, + is_admin: $userStore.is_admin ?? false, + is_super_admin: $userStore.is_super_admin ?? false, + pgroups: $userStore.pgroups ?? [], + groups: $userStore.groups ?? [] + } + : undefined + } + return targetUser + ? { + username: targetUser.username, + is_admin: targetUser.is_admin ?? false, + is_super_admin: targetUser.is_super_admin ?? false, + pgroups: (targetUser.groups ?? []).map((g) => 'g/' + g), + groups: targetUser.groups ?? [] } - }) - reloadHistory++ - } catch (e) { - folderNotFound = true + : undefined + }) + + async function loadTargetUser(): Promise { + if (!aimedElsewhere || !workspace) return + try { + targetUser = await UserService.whoami({ workspace }) + } catch { + // Not a member, or the call failed: no membership means read-only controls, + // which is the safe reading — the write would be refused anyway. + targetUser = undefined } } - // --- default_permissioned_as rules editor --- - let defaultPermissionedAs: FolderDefaultPermissionedAs = $state([]) + let can_write = $state(false) + let folder: Folder | undefined + let usernames: string[] = $state([]) + let groups: string[] = $state([]) + let folderNames: string[] = $state([]) + let ownerItem: string = $state('') + + let groupEditorDrawer: GroupEditorDrawer | undefined = $state(undefined) + let addMemberPopover: Popover | undefined = $state(undefined) + let nameInput: TextInput | undefined = $state(undefined) + + let baseline: FolderDraft | undefined = $state(undefined) + // Empty, not `emptyDraft()`: that one seeds the caller as an admin, which is true of a + // folder being created and a lie about one whose read failed. Every path that wants the + // seeded row calls `emptyDraft()` itself. + let draft: FolderDraft = $state({ + summary: '', + labels: [], + defaultPermissionedAs: [], + perms: [] + }) + let labelsInput: LabelsInput | undefined = $state() + let pendingLabel = $state('') + let folderNotFound: boolean | undefined = $state(undefined) + let loaded = $state(false) + + // A name typed in `new` mode, and one whose folder turned out not to exist, both + // end up at `createFolder` on save. + const isNew = $derived(mode === 'new' || folderNotFound === true) + + function emptyDraft(): FolderDraft { + return { + summary: '', + labels: [], + defaultPermissionedAs: [], + // The backend makes the creator an owner whatever we send, so the table + // shows that from the start rather than after the first reload. + perms: membership ? [{ owner_name: 'u/' + membership.username, role: 'admin' as Role }] : [] + } + } + + function setDraft(value: FolderDraft) { + baseline = structuredClone(value) + draft = structuredClone(value) + } + + async function loadUsernames(): Promise { + usernames = await UserService.listUsernames({ workspace: targetWorkspace }) + } + + async function loadGroups(): Promise { + groups = await GroupService.listGroupNames({ workspace: targetWorkspace }) + } + + async function loadFolderNames(): Promise { + folderNames = await FolderService.listFolderNames({ workspace: targetWorkspace }) + } + + /** Fills a picker or a validation list. The editor is usable before these land, so they + * run alongside the folder read — but a rejection has to be reported: unhandled, it + * leaves the list silently empty and duplicate names stop being caught. */ + function loadAside(load: () => Promise): void { + load().catch((e) => sendUserToast(e?.body ?? String(e), true)) + } + + async function load() { + loadAside(loadUsernames) + loadAside(loadGroups) + // Before the folder read: `can_write` is computed from this membership. + await loadTargetUser() + if (mode === 'new') { + loadAside(loadFolderNames) + can_write = true + setDraft(emptyDraft()) + loaded = true + } else { + await loadFolder() + } + } + + function grant(close: () => void) { + const owner = (ownerKind == 'user' ? 'u/' : 'g/') + ownerItem + if (!draft.perms.some((p) => p.owner_name === owner)) { + draft.perms.push({ owner_name: owner, role: newMemberRole }) + } + ownerItem = '' + close() + } + + /** `baselineOnly` re-reads the folder without touching the draft: after a save that + * committed some of its calls and then failed, the baseline must become what the server + * actually holds while the draft stays the user's intent — the applied changes then stop + * counting as dirty, and the ones still missing stay dirty and retryable. */ + async function loadFolder(opts?: { baselineOnly?: boolean }): Promise { + const apply = (value: FolderDraft) => + opts?.baselineOnly ? (baseline = structuredClone(value)) : setDraft(value) + try { + folder = await FolderService.getFolder({ workspace: targetWorkspace, name }) + folderNotFound = false + can_write = + membership != undefined && + (folder?.owners.includes('u/' + membership.username) || + membership.is_admin || + membership.is_super_admin || + membership.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1) + + apply({ + summary: folder.summary ?? '', + labels: [...(folder.labels ?? [])], + defaultPermissionedAs: (folder.default_permissioned_as ?? []).map((r) => ({ ...r })), + perms: Array.from( + new Set( + Object.entries(folder?.extra_perms ?? {}) + .map((x) => x[0]) + .concat(folder?.owners ?? []) + ) + ).map((x) => ({ owner_name: x, role: getRole(x) })) + }) + reloadHistory++ + } catch (e) { + // Only a folder that is genuinely absent may replace the draft — it can be created + // from here, so the editor opens on an empty one rather than a dead end. Any other + // failure (network, 5xx) must leave the draft alone: overwriting it here would + // discard the user's edits and clear `unsaved` with them. + if (e?.status === 404) { + folderNotFound = true + can_write = true + apply(emptyDraft()) + } else { + sendUserToast(e?.body ?? String(e), true) + } + } finally { + loaded = true + } + } const restricted = $derived( - isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + isDemoWorkspaceRestricted(targetWorkspace, membership?.is_admin, membership?.is_super_admin) ) const canEditDefaults = $derived( can_write && !restricted && - ($userStore?.is_admin || - $userStore?.is_super_admin || - ($userStore?.groups ?? []).includes('wm_deployers')) + (membership?.is_admin || + membership?.is_super_admin || + (membership?.groups ?? []).includes('wm_deployers')) ) function isValidGlob(glob: string): boolean { @@ -135,58 +293,47 @@ return /^[ug]\/.+/.test(value) || value.includes('@') } - // Split a permissioned_as value like "u/alice" or "g/prod" into its kind and name. - function ruleKind(value: string): 'user' | 'group' { + // Split an owner value like "u/alice" or "g/prod" into its kind and name. + function ownerKindOf(value: string): 'user' | 'group' { return value.startsWith('g/') ? 'group' : 'user' } - function ruleName(value: string): string { + function ownerNameOf(value: string): string { if (value.startsWith('u/') || value.startsWith('g/')) return value.slice(2) return value } function setRulePermissionedAs(idx: number, kind: 'user' | 'group', name: string) { const prefix = kind === 'user' ? 'u/' : 'g/' - defaultPermissionedAs[idx].permissioned_as = prefix + name + draft.defaultPermissionedAs[idx].permissioned_as = prefix + name } + // Only blocks a save for someone who can see the rules. The backend accepts values this + // rejects (`u/` alone passes `validate_default_permissioned_as`), so a folder admin who + // is not a workspace admin could otherwise meet a permanently disabled Save with no rule + // on screen to explain it. const defaultRulesInvalid = $derived( - defaultPermissionedAs.some( - (r) => !isValidGlob(r.path_glob) || !isValidPermissionedAs(r.permissioned_as) - ) + canEditDefaults && + draft.defaultPermissionedAs.some( + (r) => !isValidGlob(r.path_glob) || !isValidPermissionedAs(r.permissioned_as) + ) ) function addDefaultRule() { - defaultPermissionedAs = [...defaultPermissionedAs, { path_glob: '**', permissioned_as: '' }] + draft.defaultPermissionedAs = [ + ...draft.defaultPermissionedAs, + { path_glob: '**', permissioned_as: '' } + ] } function removeDefaultRule(idx: number) { - defaultPermissionedAs = defaultPermissionedAs.filter((_, i) => i !== idx) + draft.defaultPermissionedAs = draft.defaultPermissionedAs.filter((_, i) => i !== idx) } function moveDefaultRule(idx: number, delta: -1 | 1) { - const next = [...defaultPermissionedAs] + const next = [...draft.defaultPermissionedAs] const target = idx + delta if (target < 0 || target >= next.length) return ;[next[idx], next[target]] = [next[target], next[idx]] - defaultPermissionedAs = next - } - - async function saveDefaultRules() { - if (defaultRulesInvalid) { - sendUserToast('Some rules have invalid globs or permissioned_as values', true) - return - } - try { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { default_permissioned_as: defaultPermissionedAs } - }) - sendUserToast('Default permissioned_as rules updated') - dispatch('update') - loadFolder() - } catch (e) { - sendUserToast(e.body ?? String(e), true) - } + draft.defaultPermissionedAs = next } function getRole(x: string): Role { @@ -204,51 +351,208 @@ } let ownerKind: 'user' | 'group' = $state('user') - let groupCreated: string | undefined = $state(undefined) - let newGroupName: string = $state('') - let summary: string = $state('') - let labels: string[] | undefined = $state(undefined) + let newMemberRole: Role = $state('viewer') - async function saveLabels() { + // Set when the group editor is opened from the add-member form, so that saving returns + // there. Opened from a member row instead, that group is already a member and reopening + // the form on it would offer to add it twice. + let groupEditorInterruptedPicker = false + + function openGroupEditor(groupName: string, fromPicker: boolean) { + groupEditorInterruptedPicker = fromPicker + if (groupName) groupEditorDrawer?.initEdit(groupName) + else groupEditorDrawer?.initNew() + } + + async function onGroupSaved(groupName: string) { + // The group has to be in `groups` before the picker reopens, or the value set below + // has no matching item to show. try { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { labels: labels ?? [] } - }) - sendUserToast('Folder labels updated') - dispatch('update') + await loadGroups() } catch (e) { - sendUserToast(e.body ?? String(e), true) - loadFolder() + sendUserToast(e?.body ?? String(e), true) + } + if (!groupEditorInterruptedPicker) return + // Editing a group was a detour from adding a member: come back to the form on that + // group so the interrupted job can be finished. + ownerKind = 'group' + ownerItem = groupName + addMemberPopover?.open() + } + + // Guarded on `mode`, not `isNew`: the name field is rendered only in `new` mode, so on the + // not-found branch there is no input to annotate and no name the user could correct. + const nameError = $derived( + mode !== 'new' + ? '' + : !name + ? '' + : !VALID_FOLDER_NAME.test(name) + ? 'Folder name can only contain alphanumeric characters, underscores, and hyphens' + : folderNames.includes(name) + ? 'A folder with this name already exists' + : '' + ) + + // `create_folder` folds the caller into `owners` with write whatever the payload says, so + // on create their own row is fixed: offering to demote or remove it would be a change the + // backend silently discards. + // An invalid rule disables Save, so the section holding it is held open rather than merely + // opened once: collapsing it would hide the only explanation for the disabled button. + let defaultRulesOpen = $state(false) + + function isFixedCreatorRow(owner: string): boolean { + return isNew && owner === 'u/' + membership?.username + } + + // The label input holds typed text until Enter or a blur, and that text is an edit like + // any other: it has to count as dirty here, or Save stays disabled when it is the only + // change and closing drops it without asking. `save()` flushes it into `draft.labels`. + const dirty = $derived(isFolderDraftDirty(draft, baseline) || pendingLabel !== '') + // A typed name is progress too, even before any other field is touched. + const unsaved = $derived(dirty || (mode === 'new' && !!name)) + + $effect(() => { + onCanSaveChange?.( + isNew + ? loaded && !!name && !nameError && !restricted && !defaultRulesInvalid + : can_write && dirty && !defaultRulesInvalid + ) + }) + + $effect(() => { + onUnsavedChange?.(unsaved) + }) + + $effect(() => { + onExistsChange?.(!isNew) + }) + + /** Replays the permission rows the user changed. `updateFolder` could write + * `owners`/`extra_perms` wholesale in the same call as the settings, but it only + * logs a single "update owners"/"update acl" entry, so the permission history + * would stop naming who was granted what. The diff itself is in `folderDraft.ts`. */ + async function applyPermissionChanges(next: FolderDraft['perms'], prev: FolderDraft['perms']) { + const workspace = targetWorkspace + const callerOwners = membership + ? ['u/' + membership.username, ...(membership.pgroups ?? [])] + : [] + for (const call of folderPermissionDiff(prev, next, callerOwners)) { + switch (call.kind) { + case 'grantAdmin': + await FolderService.addOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner } + }) + break + case 'demoteAdmin': + await FolderService.removeOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner, write: call.write } + }) + break + case 'setAcl': + await GranularAclService.addGranularAcls({ + workspace, + path: name, + kind: 'folder', + requestBody: { owner: call.owner, write: call.write } + }) + break + case 'remove': + // Sequential, and `removeowner` first: the write policy refuses it when the + // member being removed is the caller's last admin handle. Failing there leaves + // the folder untouched, where the other order strands a member with no grant + // but still in `owners`. + await FolderService.removeOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner } + }) + await GranularAclService.removeGranularAcls({ + workspace, + path: name, + kind: 'folder', + requestBody: { owner: call.owner } + }) + break + } } } - async function addGroup() { - await GroupService.createGroup({ - workspace: $workspaceStore ?? '', - requestBody: { name: newGroupName } - }) - groupCreated = newGroupName - $userStore?.folders?.push(newGroupName) - loadGroups() - ownerItem = newGroupName + export async function save(): Promise<{ name: string; created: boolean } | undefined> { + // Clicking Save blurs the label input, which commits its text on a delay — after the + // snapshot below. Take the label first or it is dropped as the drawer closes. + labelsInput?.flushPendingLabel() + const next = $state.snapshot(draft) as FolderDraft + const prev = baseline as FolderDraft + // Captured before the write: an edit-branch save reloads, which clears `folderNotFound`. + const created = isNew + try { + if (created) { + await FolderService.createFolder({ + workspace: targetWorkspace, + requestBody: { + name, + summary: next.summary, + labels: next.labels, + default_permissioned_as: next.defaultPermissionedAs, + owners: next.perms.filter((p) => p.role === 'admin').map((p) => p.owner_name), + extra_perms: Object.fromEntries( + next.perms.map((p) => [p.owner_name, p.role !== 'viewer']) + ) + } + }) + sendUserToast(`Folder ${name} created`) + } else { + const requestBody: { + summary?: string + labels?: string[] + default_permissioned_as?: FolderDefaultPermissionedAs + } = {} + if (next.summary !== prev.summary) requestBody.summary = next.summary + if (!deepEqual(next.labels, prev.labels)) requestBody.labels = next.labels + if (!deepEqual(next.defaultPermissionedAs, prev.defaultPermissionedAs)) { + requestBody.default_permissioned_as = next.defaultPermissionedAs + } + if (Object.keys(requestBody).length > 0) { + await FolderService.updateFolder({ workspace: targetWorkspace, name, requestBody }) + } + await applyPermissionChanges(next.perms, prev.perms) + await loadFolder() + sendUserToast('Folder updated') + } + return { name, created } + } catch (e) { + sendUserToast(e.body ?? String(e), true) + // A failed create is not proof the folder is absent: `create_folder` commits before a + // git-sync step that can still fail the request. Only the name conflict says it was + // never written. Report rather than resolve — a folder found by name may be someone + // else's, and adopting it would send this draft's writes there. + const nameTaken = String(e?.body ?? '').includes('already exists') + if (created && !nameTaken) { + sendUserToast(`Folder ${name} may have been created anyway — reopen it to check`, true) + } + // Reconcile after any edit-path failure rather than tracking which calls landed: + // these handlers commit before a git-sync step that can still fail the request, so + // a rejection is not proof nothing was written. The baseline moves to what the + // server now holds and the draft stays, so a retry re-sends only what is missing. + if (!created) await loadFolder({ baselineOnly: true }) + return undefined + } } - const dispatch = createEventDispatcher() - - async function updateFolder() { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { summary } - }) - sendUserToast('Folder summary updated') - dispatch('update') - loadFolder() - } + // The stores are read only to wait until they are populated, and the load runs once: this + // editor holds an unsaved draft, and the layout re-`set`s `$userStore` periodically — a + // second `load()` would overwrite the draft with the server's state and lose the edits + // silently, `unsaved` included. The drawer remounts this component per opening. + let loadStarted = false $effect.pre(() => { + if (loadStarted) return if ($workspaceStore && $userStore) { + loadStarted = true untrack(() => { load() }) @@ -256,47 +560,41 @@ }) let reloadHistory = $state(0) + + onMount(async () => { + if (mode !== 'new') return + // The editor is remounted per drawer opening, so mount is the moment the + // create form appears; the input only exists after the first render. + await tick() + nameInput?.focus() + }) - - { - newGroup?.closeDrawer() - groupCreated = undefined - }} - > - {#if !groupCreated} -
- - -
- {:else} - - {/if} - - - - - - - - +
- + {/if} + +
{#if can_write} - + (pendingLabel = v)} + /> {:else}
- {#each labels ?? [] as label (label)} + {#each draft.labels as label (label)} {label} {:else} No labels @@ -319,257 +622,261 @@
-
diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 4a0bd58715..b321e76693 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1070,8 +1070,10 @@
  • feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions - are started for, and the plan tier and quota shown when the execution meter is - opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1123,8 +1125,10 @@
  • feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions - are started for, and the plan tier and quota shown when the execution meter is - opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1152,20 +1156,19 @@
    Instance object storage is not configured, so every server and worker keeps its log - files on its own disk. This page lists what each host wrote, but can only open the - files belonging to the replica serving the request — another host's are listed and - not readable — and a host's files go with it when it is replaced. Retention below - still governs the entries in the database and the files on disk. + files on its own disk. This page lists what each host wrote, but can only open the files + belonging to the replica serving the request — another host's are listed and not + readable — and a host's files go with it when it is replaced. Retention below still + governs the entries in the database and the files on disk.
    {:else if !$enterpriseLicense}
    Log files are uploaded to instance object storage, and the indexer that would ingest - them into the columnar store and delete each one afterwards is an enterprise - feature. Retention below expires the database entries and the local files; the - uploaded copies are only removed when Delete logs from s3 periodically is on - under Object Storage. + them into the columnar store and delete each one afterwards is an enterprise feature. + Retention below expires the database entries and the local files; the uploaded copies + are only removed when Delete logs from s3 periodically is on under Object Storage.
    {/if} diff --git a/frontend/src/lib/components/LabelsInput.svelte b/frontend/src/lib/components/LabelsInput.svelte index 933fe38ea1..c7eda30d66 100644 --- a/frontend/src/lib/components/LabelsInput.svelte +++ b/frontend/src/lib/components/LabelsInput.svelte @@ -8,9 +8,22 @@ labels: string[] | undefined onchange?: () => void class?: string + /** Suggest the labels of this workspace rather than the active one, for an editor + * aimed elsewhere (the folder drawer opened from a cross-workspace picker). */ + workspace?: string + /** Text typed into the input but not yet added to `labels`. An editor with a Save + * button needs it: without it that text is invisible to the editor's dirty state, + * so it is silently dropped on close and cannot even enable Save on its own. */ + onPendingChange?: (pending: string) => void } - let { labels = $bindable(), onchange, class: clazz = '' }: Props = $props() + let { + labels = $bindable(), + onchange, + class: clazz = '', + workspace, + onPendingChange + }: Props = $props() let adding = $state(false) let inputValue = $state('') @@ -34,9 +47,13 @@ !(labels ?? []).includes(trimmedInput) ) + $effect(() => { + onPendingChange?.(adding ? trimmedInput : '') + }) + async function loadExistingLabels() { try { - const resp = await fetch(`/api/w/${$workspaceStore}/labels/list`) + const resp = await fetch(`/api/w/${workspace ?? $workspaceStore}/labels/list`) if (resp.ok) existingLabels = await resp.json() } catch {} } @@ -82,8 +99,15 @@ addLabel() // either "Create new" selected or free text } } else if (e.key === 'Escape') { + // Escape cancels the label, and nothing else. Left to bubble it also reaches + // whatever encloses us — a drawer or dialog closes on it, and one guarding on + // unsaved changes reads `pending` before this clears it, so it prompts to + // discard work this key just discarded. + e.preventDefault() + e.stopPropagation() inputValue = '' adding = false + onPendingChange?.('') } else if (e.key === 'ArrowDown') { e.preventDefault() const maxIdx = suggestions.length + (showCreateNew ? 1 : 0) - 1 @@ -100,6 +124,14 @@ if (adding) addLabel() }, 150) } + + /** Add whatever is typed but not yet committed, right now. Blur commits on a 150ms + * grace period, so a caller that reads `labels` in the same tick as the blur — a Save + * button, which blurs this input by being clicked — would miss the last label. + * `adding` is cleared here, so the pending timer then finds nothing to do. */ + export function flushPendingLabel(): void { + if (adding) addLabel() + }
    diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index c7f7065565..14658a4228 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -305,6 +305,7 @@ resourceType="s3_bucket" workspaceOverride="admins" buttonTextOverride="Test from a worker" + viaWorker />
    diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 14dc4efb72..e087fadfba 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -390,7 +390,9 @@ current.path = npath } - export async function save(): Promise { + /** Whether the write landed. It toasts its own failure, so most callers ignore this; + * one that follows the save with bookkeeping of its own has to know not to. */ + export async function save(): Promise { const dirty = dirtyWorkspaces try { for (const ws of dirty) { @@ -438,8 +440,10 @@ dirty.length > 1 ? `Saved resource in ${dirty.length} workspaces` : `Saved resource` ) dispatch('refresh', current?.path ?? path) + return true } catch (err) { sendUserToast(`Could not save resource: ${err.body ?? err.message}`, true) + return false } } diff --git a/frontend/src/lib/components/Section.svelte b/frontend/src/lib/components/Section.svelte index f1425c51b7..badc7c9531 100644 --- a/frontend/src/lib/components/Section.svelte +++ b/frontend/src/lib/components/Section.svelte @@ -102,7 +102,7 @@ transition:slide={animate || collapsable ? { duration: 200 } : { duration: 0 }} > {#if description} -
    {@html description}
    +
    {@html description}
    {/if}
    diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 29da96cf98..471ed74099 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -252,7 +252,7 @@ {/if}
    Extra permissions ({acls?.length ?? 0})Extra members ({acls?.length ?? 0}) {#if linkedVarPaths.length > 0}
    @@ -299,7 +299,7 @@ size="lg" variant="accent" disabled={!newOwner} - on:click={() => addAcl(newOwner, write)}>Add permission addAcl(newOwner, write)}>Add member
    {/if} @@ -307,7 +307,7 @@ {#snippet headerRow()} - owner + member diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 65713e19f7..8133494f18 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -63,6 +63,10 @@ /** Trailing debounce window (ms) on Monaco's onDidChangeModelContent. */ const CHANGE_TIMEOUT = 200 + /** Gap between the line numbers and the first character. Zero puts them flush, + * so a two-digit line reads as one token with the code. */ + const LINE_DECORATIONS_WIDTH = 6 + let changeTimeoutId: number | undefined = undefined // Monaco fires onDidChangeModelContent synchronously from within `setValue`, so without // this an authoritative overwrite reads as a user edit on the `input` event. @@ -112,7 +116,8 @@ minHeight = 1000, renderLineHighlight = 'none', suggestion, - leadingChangeSync = false + leadingChangeSync = false, + lineNumbersMinChars = 3 }: { lang: string code?: string @@ -149,6 +154,9 @@ * `code`; leave it off where each extra sync costs work downstream (an app * code input feeding an autoRefresh runnable re-runs a job per sync). */ leadingChangeSync?: boolean + /** Width of the line-number gutter, in characters. Same name, and same + * default, as `Editor`, so the two render line numbers alike. */ + lineNumbersMinChars?: number } = $props() let yPadding = MONACO_Y_PADDING @@ -312,10 +320,12 @@ if (model.getLanguageId() !== lang) { const currentCode = model.getValue() const uri = `file:///${hash}.${langToExt(lang)}` - const oldModel = model - const newModel = meditor.createModel(currentCode, lang, mUri.parse(uri)) - editor?.setModel(newModel) - oldModel.dispose() + // The old model goes first: `langToExt` maps anything it does not know to + // `unknown`, so the new uri is usually the one this model already holds, + // and creating over an occupied uri throws ("model already exists"). + editor?.setModel(null) + model.dispose() + editor?.setModel(meditor.createModel(currentCode, lang, mUri.parse(uri))) } // Update editor options for suggestions, validation decorations, and line numbers @@ -334,8 +344,8 @@ snippetsPreventQuickSuggestions: disableSuggestions }, lineNumbers: hideLineNumbers ? 'off' : 'on', - lineDecorationsWidth: hideLineNumbers ? 0 : 6, - lineNumbersMinChars: hideLineNumbers ? 0 : 2, + lineDecorationsWidth: hideLineNumbers ? 0 : LINE_DECORATIONS_WIDTH, + lineNumbersMinChars: hideLineNumbers ? 0 : lineNumbersMinChars, // Hide validation squiggles and decorations renderValidationDecorations: disableLinting ? 'off' : 'on', // Hide the validation margin indicators @@ -397,8 +407,11 @@ ...(yPadding !== undefined ? { padding: { bottom: yPadding, top: yPadding } } : {}), readOnly, renderLineHighlight, - lineDecorationsWidth: 0, - lineNumbersMinChars: 2, + // Same conditional as `updateModelAndOptions`: created correct rather than + // created wide and narrowed a tick later, which a caller hiding the gutter + // would see as a flash of indent. + lineDecorationsWidth: hideLineNumbers ? 0 : LINE_DECORATIONS_WIDTH, + lineNumbersMinChars: hideLineNumbers ? 0 : lineNumbersMinChars, fontSize: fontSize, quickSuggestions: disableSuggestions ? { other: false, comments: false, strings: false } diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 37e3e152db..6a58af8031 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -1,5 +1,13 @@ @@ -22,6 +24,7 @@ on:click={() => (dispatch('close'), onClick?.())} on:pointerdown={(e) => e.stopPropagation()} {id} + {title} startIcon={{ icon: Icon ?? X }} iconOnly unifiedSize="sm" diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index 56244632ef..c858f9f08f 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -23,6 +23,12 @@ /** Tailwind z-index class for the modal root. Override to stack this modal * above another modal that's already open (both default to `z-[9999]`). */ zIndexClass?: string + /** Render into `body` instead of where this component sits. Needed when an ancestor + * creates a stacking context the dialog has to escape — a drawer paints over the page + * whatever the dialog's z-index, and a `transform`, `filter` or `overflow` on the way + * up confines it. Off by default: it moves the dialog out of its DOM position, so opt + * in per call site rather than assuming every caller wants it. */ + alwaysPortal?: boolean children?: Snippet onConfirmed?: () => void | Promise onCanceled?: () => void @@ -40,6 +46,7 @@ id, trashbin = false, zIndexClass = 'z-[9999]', + alwaysPortal = false, children, onConfirmed, onCanceled @@ -141,7 +148,11 @@ - + {#if open}
    + + +{#snippet body()} +
    + {#if icon} +
    {@render icon()}
    + {/if} +
    +
    + {@render title()} +
    + {#if subtitle} + + + {@render subtitle()} + + {/if} +
    +
    +{/snippet} + +{#if trailing} + +
    + {#if onClick} + + + {:else} + +
    {@render body()}
    + {/if} + {@render trailing()} +
    +{:else if !onClick} + + +
    + {@render body()} +
    +{:else} + +{/if} diff --git a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts new file mode 100644 index 0000000000..4da6f18f32 --- /dev/null +++ b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts @@ -0,0 +1,84 @@ +import { untrack } from 'svelte' + +/** + * The highlighted row of a searchable list: the one the arrow keys move and Enter + * activates, rendered by passing `highlighted` to `ListRow`. + * + * Pairs with a search field above the list — the arrows and Enter are answered while + * focus stays in it, so a query and a choice are one uninterrupted sequence. + */ +export function useListHighlight(opts: { + /** How many rows the list holds right now. */ + count: () => number + /** The DOM id of the row at this index — the same `id` given to its `ListRow`. */ + rowId: (index: number) => string + /** Where the highlight belongs when the list changes underneath it: the top hit while + * a search is on, and typically -1 (nothing lit) when it is not. */ + restingIndex: () => number + /** Open the row at this index. */ + onActivate: (index: number) => void + /** Ids of the elements whose Enter also activates the highlighted row — the search + * field. A focused row activates itself, so it is not one of these. */ + activateEnterFrom?: string[] +}) { + let index = $state(-1) + // Scrolling rows under a resting pointer makes the browser fire `mouseenter` on each + // one, which would drag the highlight back under the cursor as the arrow keys move it. + // Only a real pointer move hands the highlight back to the mouse. + let pointerOwns = $state(true) + + // Filtering reshuffles the rows under the highlight, so it goes back where the caller + // says it belongs rather than staying on a position that now means another row. + $effect(() => { + opts.count() + const resting = opts.restingIndex() + untrack(() => (index = resting)) + }) + + function move(delta: number) { + const count = opts.count() + if (count === 0) return + pointerOwns = false + // Rows are tabbable, so focus can sit on one. Enter then activates whatever is + // focused, which has to stay the highlighted row — so any row counts, not just + // the lit one. Tab from the search field lands on the first row while the + // highlight rests on the best match, and testing only the lit row would leave + // focus behind and activate the wrong one. + const focusedId = document.activeElement?.id + const rowWasFocused = + !!focusedId && Array.from({ length: count }, (_, i) => opts.rowId(i)).includes(focusedId) + index = index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count + const row = document.getElementById(opts.rowId(index)) + row?.scrollIntoView({ block: 'nearest' }) + if (rowWasFocused) row?.focus() + } + + return { + get index() { + return index + }, + /** Wire to each row's `onMouseEnter`. */ + hovered(i: number) { + if (pointerOwns) index = i + }, + /** Wire to the list container's `onpointermove`. */ + pointerMoved() { + pointerOwns = true + }, + /** Wire to the container that holds the search field and the rows, so the keys are + * answered whichever of the two has focus. */ + onKeydown(e: KeyboardEvent) { + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { + e.preventDefault() + move(e.key === 'ArrowDown' ? 1 : -1) + } else if ( + e.key === 'Enter' && + opts.activateEnterFrom?.includes((e.target as HTMLElement)?.id) && + index >= 0 + ) { + e.preventDefault() + opts.onActivate(index) + } + } + } +} diff --git a/frontend/src/lib/components/common/modal/Modal2.svelte b/frontend/src/lib/components/common/modal/Modal2.svelte index f33b1dec04..b9e57740b9 100644 --- a/frontend/src/lib/components/common/modal/Modal2.svelte +++ b/frontend/src/lib/components/common/modal/Modal2.svelte @@ -26,6 +26,10 @@ * and clicks "outside" the child would otherwise propagate * here and close the underlying modal. */ closeOnOutsideClick?: boolean + /** Close on Escape. Default true. Every open modal listens on the + * window, so a stacked pair would both close on one press; set it + * false on the underlying modal while its child is up. */ + closeOnEscape?: boolean /** Wider side padding and a lighter title, for a dialog whose body is a form rather * than a list. Opt-in: every other Modal2 keeps the padding and heading it had. */ formStyling?: boolean @@ -46,6 +50,7 @@ fixedHeight = 'md', contentClasses = '', closeOnOutsideClick = true, + closeOnEscape = true, formStyling = false, headerLeft, headerRight, @@ -80,7 +85,7 @@ } function handleKeyDown(event: KeyboardEvent) { - if (!isOpen) return + if (!isOpen || !closeOnEscape) return if (event.key === 'Escape') { event.preventDefault() event.stopPropagation() diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte index 430a53e66f..674c147c52 100644 --- a/frontend/src/lib/components/common/modal/PagedContent.svelte +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -26,6 +26,9 @@ * around this owns it, and a page component that swallowed it would stop the dialog from * closing. Without this prop the pages are still navigable, just not from the keyboard — * the caller owns `current` either way. + * + * A host that stays mounted while hidden must withhold it while hidden: the arrows are + * answered at `window`, so a parked instance would take the key off the visible one. */ onNavigate?: (key: string) => void /** @@ -80,6 +83,9 @@ function onKeydown(event: KeyboardEvent) { if (!onNavigate || !listening() || event.metaKey || event.ctrlKey || event.altKey) return + // A control on the page that already answered the key keeps it: the arrows move focus + // inside a toggle group, a menu, a slider, and those handlers run before this one. + if (event.defaultPrevented) return if (!ownsKeyboard(event.target)) return const step = event.key === 'ArrowRight' ? 1 : event.key === 'ArrowLeft' ? -1 : 0 if (step === 0) return diff --git a/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte b/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte index 260e368e2e..d18df4c655 100644 --- a/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte +++ b/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte @@ -2,6 +2,7 @@ import type { ComponentType } from 'svelte' import { twMerge } from 'tailwind-merge' import Button from '$lib/components/common/button/Button.svelte' + import { arrowTabNav } from '$lib/attachments/arrowTabNav' import EEOnly from '$lib/components/EEOnly.svelte' import { enterpriseLicense } from '$lib/stores' @@ -32,7 +33,10 @@ let { groups, selectedId, onNavigate, class: className = '' }: Props = $props() -
    + +
    {#each groups as group (group.title)}
    {#if group.title} diff --git a/frontend/src/lib/components/copilot/AIFormAssistant.svelte b/frontend/src/lib/components/copilot/AIFormAssistant.svelte index db6a341c58..9c7b3cb411 100644 --- a/frontend/src/lib/components/copilot/AIFormAssistant.svelte +++ b/frontend/src/lib/components/copilot/AIFormAssistant.svelte @@ -5,6 +5,7 @@ import OpenInSessionButton from '$lib/components/sessions/OpenInSessionButton.svelte' import { AIBtnClasses } from './chat/AIButtonStyle' import { workspaceStore } from '$lib/stores' + import { copilotInfo } from '$lib/aiStore' import { logFeatureUsage } from '$lib/utils/featureUsage' interface Props { @@ -49,45 +50,47 @@ ) -
    -
    - -

    AI can help with these inputs

    - - {#snippet fallback()} - - {/snippet} - +

    AI can help with these inputs

    + + {#snippet fallback()} + + {/snippet} + +
    +
    +

    + {instructions + ? 'Instructions: ' + instructions + : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} +

    +
    -
    -

    - {instructions - ? 'Instructions: ' + instructions - : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} -

    -
    -
    +{/if} diff --git a/frontend/src/lib/components/copilot/AIFormSettings.svelte b/frontend/src/lib/components/copilot/AIFormSettings.svelte index 44151a01b3..ef627ace4b 100644 --- a/frontend/src/lib/components/copilot/AIFormSettings.svelte +++ b/frontend/src/lib/components/copilot/AIFormSettings.svelte @@ -3,6 +3,7 @@ import Label from '../Label.svelte' import Toggle from '../Toggle.svelte' import Tooltip from '../Tooltip.svelte' + import { copilotInfo } from '$lib/aiStore' interface Props { prompt?: string | undefined @@ -12,35 +13,37 @@ let { prompt = $bindable(undefined), type = 'script' }: Props = $props() -
    - { - if (prompt !== undefined) { - prompt = undefined - } else { - prompt = '' - } - }} - options={{ right: `Enable filling ${type} inputs with AI` }} - /> - {#if prompt !== undefined} -
    - -
    - {/if} -
    +{#if !$copilotInfo.workspaceDisabled} +
    + { + if (prompt !== undefined) { + prompt = undefined + } else { + prompt = '' + } + }} + options={{ right: `Enable filling ${type} inputs with AI` }} + /> + {#if prompt !== undefined} +
    + +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index 6a8dc6892e..94715a0f18 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -79,66 +79,68 @@ }) - - {#snippet trigger()} -
    - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} -
    - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + /> +
    + {:else} +
    +

    Enable Windmill AI in the workspace settings

    +
    + {/if} +
    + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index e61ab5cbf2..f399726988 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,5 +1,5 @@ - - {#snippet trigger()} - +{#if !$copilotInfo.workspaceDisabled} + + {#snippet trigger()}
    - - {/snippet} - + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ResourceGen.svelte b/frontend/src/lib/components/copilot/ResourceGen.svelte index 34534285b4..2fc3cfa646 100644 --- a/frontend/src/lib/components/copilot/ResourceGen.svelte +++ b/frontend/src/lib/components/copilot/ResourceGen.svelte @@ -119,69 +119,71 @@ }) - - {#snippet trigger()} - -
    - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} -
    • - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + > + Generate + +
    + {:else} +
    +

    Enable Windmill AI in the workspace settings

    +
    + {/if} +
    + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ScriptFix.svelte b/frontend/src/lib/components/copilot/ScriptFix.svelte index d6ccb26ab8..f586ad1aaf 100644 --- a/frontend/src/lib/components/copilot/ScriptFix.svelte +++ b/frontend/src/lib/components/copilot/ScriptFix.svelte @@ -77,7 +77,7 @@ const sessionScopedManager = getContext('aiChatManager') -{#if SUPPORTED_LANGUAGES.has(lang)} +{#if SUPPORTED_LANGUAGES.has(lang) && !$copilotInfo.workspaceDisabled} {#if sessionScopedManager}
    diff --git a/frontend/src/lib/components/copilot/StepInputsGen.svelte b/frontend/src/lib/components/copilot/StepInputsGen.svelte index 5b1ddb741b..c9ef616aa5 100644 --- a/frontend/src/lib/components/copilot/StepInputsGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputsGen.svelte @@ -226,7 +226,7 @@ input_name2: expression2 Fill inputs {/if} - {:else} + {:else if !$copilotInfo.workspaceDisabled} togglePanel() })} -{:else} +{:else if !$copilotInfo.workspaceDisabled} {#snippet trigger()} {@render button({ onPress: () => togglePanel() })} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index e4dba770eb..fd95613f2e 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -44,8 +44,12 @@ const isAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) const hasCopilot = $derived($copilotInfo.enabled) + // Another tab is running a turn on this session: transcript stays readable, + // composer locks, and the chat re-reads the shared record when the turn ends. + const runHeldElsewhere = $derived(aiChatManager.runHeldElsewhere) const disabled = $derived( forceDisabled || + runHeldElsewhere || !hasCopilot || (aiChatManager.mode === AIMode.SCRIPT && aiChatManager.scriptEditorOptions?.lang && @@ -58,19 +62,25 @@ const disabledMessage = $derived( forceDisabled ? forceDisabledMessage - : freeTierExhausted - ? '' - : !hasCopilot - ? $aiUserDisabled - ? 'Windmill AI is disabled in your account settings' - : isAdmin - ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` - : 'Ask an admin to enable Windmill AI in this workspace to use this chat' - : aiChatManager.mode === AIMode.SCRIPT && - aiChatManager.scriptEditorOptions?.lang && - !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) - ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` - : '' + : runHeldElsewhere + ? // The typing indicator and the composer placeholder already carry + // this state; a footer note would say it a third time. + '' + : freeTierExhausted + ? '' + : !hasCopilot + ? $copilotInfo.workspaceDisabled + ? 'Windmill AI is hidden in this workspace' + : $aiUserDisabled + ? 'Windmill AI is disabled in your account settings' + : isAdmin + ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` + : 'Ask an admin to enable Windmill AI in this workspace to use this chat' + : aiChatManager.mode === AIMode.SCRIPT && + aiChatManager.scriptEditorOptions?.lang && + !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) + ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` + : '' ) const suggestions = [ diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 2ab29499ce..699098c116 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -7,6 +7,7 @@ AlertTriangle, ArrowDown, AtSign, + BookOpen, ChevronDown, ChevronsRight, CheckIcon, @@ -34,7 +35,9 @@ import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' - import McpConnections from './McpConnections.svelte' + import AssistantSettingsModal from './AssistantSettingsModal.svelte' + import { SkillsMenu } from './skills/skillsMenu.svelte' + import { McpMenu } from '$lib/components/mcp/mcpMenu.svelte' import ChatMode from './ChatMode.svelte' import DatatableCreationPolicy from './DatatableCreationPolicy.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' @@ -205,7 +208,11 @@ } = $props() let aiChatInput: AIChatInput | undefined = $state() - let mcpConnections: McpConnections | undefined = $state() + let assistantSettings: AssistantSettingsModal | undefined = $state() + // The "+" menu's skill and MCP rows: enough state to check and flip one, with + // everything else about them behind the assistant settings modal. + const skillsMenu = new SkillsMenu(aiChatManager, () => assistantSettings?.open('skills')) + const mcpMenu = new McpMenu(aiChatManager, () => assistantSettings?.open('mcp')) let plusMenuOpen = $state(false) let editingMessageIndex = $state(null) @@ -312,7 +319,10 @@ } }) - const showTypingIndicator = $derived(aiChatManager.loading) + // Also shown for a run held by another tab, labeled with where it is: the + // dots say a turn is in flight even before the reader reaches the footer + // note. Remote runs pause nothing and offer no Stop — this tab can't cancel. + const showTypingIndicator = $derived(aiChatManager.loading || aiChatManager.runHeldElsewhere) // The manual `@` context-picker button. Shown in SCRIPT/FLOW (workspace items + // code blocks) and APP (datatables, frontend files). Hidden in GLOBAL — there @@ -568,8 +578,14 @@ (aiChatManager.flowAiChatHelpers?.hasPendingChanges() ?? false) && !aiChatManager.autoAcceptEditsActive ) + // A disabled state with no message (a remote hold, a spent free grant) keeps + // the footer toolbar in place — swapping it for an empty strip would make + // the model/mode row flash out and back on every remote turn. A state with + // a real message (archived, AI off) still shows it, hold or not, matching + // the precedence disabledMessage itself encodes. + const footerMessageShown = $derived(disabled && disabledMessage !== '') const showFooterLeftControls = $derived( - !disabled && + !footerMessageShown && (showContextPicker || showAutonomyModeSelector || (aiChatManager.mode === AIMode.SCRIPT && hasDiff)) @@ -670,10 +686,14 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#each pastChats as chat (chat.id)}
    {/if} @@ -931,39 +956,60 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/if} {#if canAttachFiles} [ - { - displayName: 'Attach file or image', - icon: FileText, - action: () => { - plusMenuOpen = false - linkFiles() - } - }, - { - // A real (live) link needs the File System Access API; without it the - // folder is only snapshotted, so call it "Add folder", not "Link folder". - displayName: canUseFsAccess ? 'Link folder' : 'Add folder', - icon: Folder, - tooltip: canUseFsAccess - ? 'Linked live — the assistant reads the folder’s current files from disk and refreshes each turn.' - : 'Loaded as a snapshot — the folder’s files are copied into your browser (they won’t auto-update). For a live link that refreshes from disk, use a Chromium-based browser (Chrome, Edge).', - action: () => { - plusMenuOpen = false - linkFolder() - } - }, - ...(aiChatManager.mode === AIMode.GLOBAL && mcpConnections - ? [ - { - displayName: 'MCP connections', - icon: Plug, - separatorTop: true, - submenuItems: await mcpConnections.menuItems(() => (plusMenuOpen = false)) - } - ] - : []) - ]} + items={async () => { + // Both submenus fetch on the menu's first open, so they start + // together: awaited inline they queue, and the whole menu — + // attachments included — waits out two round trips. + const closeMenu = () => (plusMenuOpen = false) + const inGlobal = aiChatManager.mode === AIMode.GLOBAL + const [skillItems, mcpItems] = await Promise.all([ + inGlobal ? skillsMenu.items(closeMenu) : undefined, + inGlobal ? mcpMenu.items(closeMenu) : undefined + ]) + return [ + { + displayName: 'Attach file or image', + icon: FileText, + action: () => { + plusMenuOpen = false + linkFiles() + } + }, + { + // A real (live) link needs the File System Access API; without it the + // folder is only snapshotted, so call it "Add folder", not "Link folder". + displayName: canUseFsAccess ? 'Link folder' : 'Add folder', + icon: Folder, + tooltip: canUseFsAccess + ? 'Linked live — the assistant reads the folder’s current files from disk and refreshes each turn.' + : 'Loaded as a snapshot — the folder’s files are copied into your browser (they won’t auto-update). For a live link that refreshes from disk, use a Chromium-based browser (Chrome, Edge).', + action: () => { + plusMenuOpen = false + linkFolder() + } + }, + ...(skillItems + ? [ + { + displayName: 'Skills', + icon: BookOpen, + separatorTop: true, + submenuItems: skillItems + } + ] + : []), + ...(mcpItems + ? [ + { + displayName: 'MCP connections', + icon: Plug, + separatorTop: !skillItems, + submenuItems: mcpItems + } + ] + : []) + ] + }} placement="bottom-start" fixedHeight={false} closeOnItemClick={false} @@ -1080,12 +1126,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/snippet} {/if} - {#if aiChatManager.mode === AIMode.SCRIPT && hasDiff} + {#if aiChatManager.mode === AIMode.SCRIPT && hasDiff && !disabled} {/if}
    {/if} - {#if disabled} + {#if footerMessageShown}
    @@ -1101,9 +1147,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/if} - + + {#if aiChatManager.mode === AIMode.GLOBAL} - + {/if} {#if aiChatManager.mode === AIMode.APP && appContext && (appContext.inspectorElement || appContext.codeSelection)} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index e87c2afa57..7f4514cd41 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -129,6 +129,12 @@ // Generate mode-specific placeholder const modePlaceholder = $derived.by(() => { + // The composer unlocks by itself when the other tab's turn ends, so the + // placeholder names what it is waiting on (the typing indicator says + // where the run is). + if (aiChatManager.runHeldElsewhere) { + return 'Waiting for the turn in the other tab to finish' + } if (pendingQuestionToolCallId !== undefined) { return 'Answer the question above' } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 9bbc507931..43df13f78c 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -501,7 +501,24 @@ export class AIChatManager { openRunInPreview?: (a: { jobId: string; workspace: string; label: string }) => void openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void closeArtifact?: (artifactId: string) => void - loading = $state(false) + #loading = $state(false) + get loading(): boolean { + return this.#loading + } + // An accessor so every run bracket — the send turn, manual compaction, a + // rollback — reports its transitions through one place, synchronously: the + // rising edge posts the cross-tab "running here" signal the moment the + // bracket opens (after the send's preflight awaits; the post-preflight + // guard covers that gap), and `loading` falls only after the turn's last + // saveChat, making the falling edge the "safe to re-read the record" signal. + set loading(v: boolean) { + if (v === this.#loading) return + this.#loading = v + this.onRunningChanged?.(v) + } + /** Sessions wiring (see sessionRuntime); undefined for the global + * side-panel chat, whose transcript no other tab renders. */ + onRunningChanged: ((running: boolean) => void) | undefined = undefined currentReply = $state('') currentReasoning = $state('') currentReasoningActive = $state(false) @@ -677,6 +694,14 @@ export class AIChatManager { // sessions modules — and re-read on every system-message rebuild; the send // path rebuilds after beforeSend, so a fork committed there is picked up. sessionContextResolver: (() => SessionPromptContext | undefined) | undefined = undefined + // Whether another tab is running a turn on this session right now (sessions + // wiring, same seam as above). The composer locks on it, and sendRequest + // refuses on it — the refusal covers the send already in flight when the + // other tab's run signal arrives, which no disabled input can stop. + runHeldElsewhereResolver: (() => boolean) | undefined = undefined + get runHeldElsewhere(): boolean { + return this.runHeldElsewhereResolver?.() ?? false + } // The page the side panel shows, stamped on each user message. Same seam as above: // a page tab is an iframe in its own realm, so the tab model is the only place the // chat can learn it. Undefined for a live editor — ACTIVE EDITOR covers those. @@ -688,8 +713,8 @@ export class AIChatManager { workspaceResolver: (() => string | undefined) | undefined = undefined // The workspace every workspace-scoped chat action targets — skills, tool - // loop, logging, user-message context, and commit. Session-resolved when a - // resolver is set, else the globally-active workspace. + // loop, logging, user-message context, message rendering, and commit. + // Session-resolved when a resolver is set, else the globally-active workspace. get operatingWorkspace(): string | undefined { return this.workspaceResolver?.() ?? get(workspaceStore) } @@ -1056,6 +1081,16 @@ export class AIChatManager { * doesn't spawn a new job leaves nothing to re-trigger on. A turn that DOES * spawn another job resumes again when that one finishes, which is the point. */ + #autoResumeRetry: ReturnType | undefined + + #scheduleAutoResumeRetry() { + clearTimeout(this.#autoResumeRetry) + this.#autoResumeRetry = setTimeout(() => { + this.#autoResumeRetry = undefined + void this.#maybeAutoResumeFromJobs() + }, 5_000) + } + async #maybeAutoResumeFromJobs() { if (this.#autoResuming) return // Global/sessions chat only (the only mode with a jobs tray + preamble). @@ -1066,6 +1101,17 @@ export class AIChatManager { // Nothing to continue (empty chat), or the user is mid-compose — don't // clobber their draft or auto-send it. Their eventual send carries the notes. if (this.messages.length === 0 || this.instructions.trim()) return + // Another tab is driving: the synthetic send would only be refused, and + // the instructions staged below would then block every later auto-resume + // in this tab. The notes stay pending; re-checked shortly, because the + // hold can clear silently (staleness after a driver crash) with nothing + // else to fire this. When the driver instead ends its turn normally, its + // own resume carries the notes and this tab's catch-up clears the local + // copy — the re-check then finds nothing and stands down. + if (this.runHeldElsewhere) { + this.#scheduleAutoResumeRetry() + return + } this.#autoResuming = true try { const count = this.pendingJobNotes.length @@ -1104,6 +1150,8 @@ export class AIChatManager { // Invalidate any in-flight poll so its post-await continuation can't write // into the conversation we're switching to. this.#jobPollGeneration++ + clearTimeout(this.#autoResumeRetry) + this.#autoResumeRetry = undefined this.backgroundJobs = [] this.pendingJobNotes = [] } @@ -1135,9 +1183,10 @@ export class AIChatManager { } } - // Workspace AI skills (name + description) advertised in the GLOBAL system - // prompt and surfaced as slash commands in session chat. Loaded - // asynchronously when entering GLOBAL mode; the system message is rebuilt + // The `ai_skill` resources this user turned on for the operating workspace, + // advertised in the GLOBAL system prompt and surfaced as slash commands in + // session chat. Loaded asynchronously when entering GLOBAL mode and again + // whenever the picker changes the selection; the system message is rebuilt // once they resolve. globalSkills = $state([]) private globalSkillsRefreshId = 0 @@ -1173,9 +1222,10 @@ export class AIChatManager { ] // Built-ins followed by workspace skills, with any skill whose name collides - // with a built-in dropped: the picker keys leaves by name, so a duplicate - // would break its keyed list and ambiguous-resolve nav. Built-ins win — they - // already shadow same-named skills at execution (the submit interception). + // with a built-in dropped. Built-ins win — they already shadow same-named + // skills at execution (the submit interception), so listing both would offer + // a row that cannot run. Two skills may still share a name; the picker keys + // those by path and the submit path declines to guess between them. sessionCommands: ChatCommandItem[] = $derived([ ...this.sessionBuiltinCommands, ...this.globalSkills @@ -2073,7 +2123,10 @@ export class AIChatManager { // pipeline surface when a /pipeline editor has registered helpers. Centralized // so changeMode, refreshGlobalSkills, and setPipelineHelpers stay consistent — // each rebuild would otherwise drop the pipeline augmentation the others added. - private configureGlobalMode = () => { + // + // Public because it is purely local, unlike `changeMode(GLOBAL)`, which also + // fires the three network refreshes. + configureGlobalMode = () => { const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { previewTools: this.isSessionChat, user: this.globalIdentity, @@ -2136,7 +2189,11 @@ export class AIChatManager { if (refreshId !== this.globalSkillsRefreshId) { return } - this.globalSkills = skills + // Newest-wins is not enough: a refresh for the workspace just left can still + // hold the newest id, and installing it would advertise that workspace's + // skills to a chat now acting elsewhere. Same check the identity and MCP + // refreshes make. + this.globalSkills = workspace === (this.operatingWorkspace ?? '') ? skills : [] if (this.mode === AIMode.GLOBAL) { this.configureGlobalMode() } @@ -2224,16 +2281,25 @@ export class AIChatManager { if (!this.isSessionChat || this.mode !== AIMode.GLOBAL || !instructions.startsWith('/')) { return instructions } - const match = /^\/([a-z0-9-]+)(?:\s+([\s\S]*))?$/.exec(instructions) + // Accepts a bare name or a whole resource path: names are what people type, + // but the picker inserts the path when two folders answer to the same name. + // Unicode-aware rather than `\w`, which is ASCII-only — a resource path may + // hold any word character, and the picker can insert one the user must then + // be able to send (`f/équipe/deploy`). + const match = /^\/([\p{L}\p{N}_\-/]+)(?:\s+([\s\S]*))?$/u.exec(instructions) if (!match) { return instructions } - const skill = this.globalSkills.find((s) => s.name === match[1]) - if (!skill) { + // A path identifies one skill; a name shared by two would otherwise silently + // apply instructions the user did not choose, so it is left unexpanded. + const byPath = this.globalSkills.find((s) => s.path === match[1]) + const matches = byPath ? [byPath] : this.globalSkills.filter((s) => s.name === match[1]) + if (matches.length !== 1) { return instructions } const rest = match[2]?.trim() - return rest ? `Use the "${skill.name}" skill. ${rest}` : `Use the "${skill.name}" skill.` + const use = `Use the skill at "${matches[0].path}".` + return rest ? `${use} ${rest}` : use } canApplyCode = $derived(this.allowedModes.script && this.mode === AIMode.SCRIPT) @@ -2287,6 +2353,10 @@ export class AIChatManager { } openChat = () => { + // Nothing may open the docked pane in a workspace that hid the assistant. + if (get(copilotInfo).workspaceDisabled) { + return + } chatState.size = this.savedSize > 0 ? this.savedSize : DEFAULT_SIZE localStorage.setItem('ai-chat-open', 'true') } @@ -2298,6 +2368,9 @@ export class AIChatManager { } toggleOpen = () => { + if (chatState.size === 0 && get(copilotInfo).workspaceDisabled) { + return + } if (chatState.size > 0) { this.savedSize = chatState.size } @@ -2817,6 +2890,41 @@ export class AIChatManager { sendUserToast('This action needs the AI chat. Start an AI session to continue.', true) return } + // The workspace hid the assistant: every entry point is gone from the UI, so a turn + // reaching here comes from a path that missed the gate and would stream unseen. + if (!this.isSessionChat && get(copilotInfo).workspaceDisabled) { + sendUserToast('Windmill AI is hidden in this workspace.', true) + return + } + // Refused before anything mutates, so there is nothing to unwind: the + // draft (already taken by the composer) goes back where the user can see + // it, and the turn never starts. Only the message's own send restores it + // — a refused queued flush is re-queued by its caller (`accepted === + // false`), and a copy here would double it. Paste tokens are expanded + // into the text, as the queue does, because the restore lanes carry no + // pastes. + if (this.runHeldElsewhere) { + if (options.synthetic) { + // Client-authored prompt (a job auto-resume), not user input: nothing + // to hand back and no toast. Releasing the staged text un-blocks the + // next auto-resume attempt, scheduled for when the hold clears. + this.instructions = '' + this.#scheduleAutoResumeRetry() + } else { + if (!options.queued) { + // Programmatic prompts (askAi, fix) stage their text in + // `this.instructions` and pass no option — fall back to it so + // they are handed back too. + this.restoreToInput( + expanded(chatDraft(options.instructions ?? this.instructions, options.pastes ?? [])), + options.images, + options.files + ) + } + sendUserToast('This session is running in another tab. Your message was kept.', true) + } + return false + } this.#sendsInFlight++ try { return await this.sendRequestImpl(options) @@ -3036,6 +3144,28 @@ export class AIChatManager { ) } const images = modelIsBlind ? [] : requestedImages + // Re-checks the wrapper's remote-run guard: a run announced by another tab + // during the upkeep awaits above would otherwise interleave two turns into + // one chat id. Resends are exempt — restartGeneration already truncated + // the transcript, so they run as the documented advisory race instead. + if (this.runHeldElsewhere && !options.resendReservationKey) { + this.#releaseOutgoingReservation(reservationKey) + if (options.synthetic) { + // Same as the wrapper guard: an internal prompt is released, not + // restored as a draft the user never wrote. + this.instructions = '' + this.#scheduleAutoResumeRetry() + } else { + // restoreToInput, not restoreInstructions: a draft typed during the + // awaits above occupies the composer, and this restore must merge + // into it (or queue), never be refused by it. + if (!options.queued) { + this.restoreToInput(expanded(chatDraft(this.instructions, pastes)), images, files) + } + sendUserToast('This session is running in another tab. Your message was kept.', true) + } + return false + } const optimisticIndex = this.displayMessages.length this.loading = true // Create the abort controller before the (possibly slow) beforeSend pre-flight, @@ -3877,6 +4007,29 @@ export class AIChatManager { throw new Error('No user message found at the specified index') } + // Refused before anything mutates: past this point the transcript is + // sliced and resend bytes are reserved, and the sendRequest guard could + // only refuse AFTER that damage — restoring nothing, since this path + // carries its text in `this.instructions`, not the options. The retry and + // edit controls check only local `loading`, so a remote run reaches here. + // An edit (newContent defined, even '': attachment-only edits exist) is + // restored with its pastes expanded into the text; a bare retry mutates + // nothing yet, so there is nothing to restore. Un-submitted context-chip + // edits are the one loss — the chips re-seed from the untouched message + // on the next edit. + if (this.runHeldElsewhere) { + if (newContent !== undefined) { + this.restoreToInput( + expanded(chatDraft(newContent, pastes ?? [])), + images ?? [], + files ?? [] + ) + } + // "Text", not "message": chip edits are the part that does not survive. + sendUserToast('This session is running in another tab. Your text was kept.', true) + return + } + // Resolve the API restart point BEFORE reserving bytes or truncating: a // stale index must fail while nothing has been mutated, or the transcript // would be left truncated with the reservation leaked. A negative index @@ -4000,7 +4153,7 @@ export class AIChatManager { this.onChatRotated?.(this.historyManager.getCurrentChatId()) } - loadPastChat = async (id: string) => { + loadPastChat = async (id: string, { preserveQueue = false } = {}) => { // A turn commits into whatever transcript it finds when it ends, so swapping // one in underneath it misfiles the turn — or duplicates it, when the loaded // chat already carries the turn's own checkpoint. Gated on `sendInFlight` @@ -4010,7 +4163,10 @@ export class AIChatManager { if (chat) { // Drop any message queued in the current conversation so it doesn't // auto-send into the loaded one or linger as a card across the switch. - this.#clearQueue() + // `preserveQueue` is for reloads that are NOT a switch — a cross-tab + // catch-up re-reading the conversation on screen — where the queued + // draft is unsent user input the reload must not destroy. + if (!preserveQueue) this.#clearQueue() // Stop the poller for the conversation being left before swapping in the // loaded chat's jobs below. this.clearBackgroundJobs() diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 5372fec7ca..084dc2c040 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -7,6 +7,7 @@ import type { ChatCompletionMessageParam } from 'openai/resources/chat/completio import type { DisplayMessage } from './shared' import type { AttachedImage } from './imageUtils' import { AIChatManager, AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { makePasteToken } from './pasteTokens' import { chatState } from './sharedChatState.svelte' import { PLAN_MODE_MESSAGES } from './planModeMessages' import { runChatLoop } from './chatLoop' @@ -33,7 +34,7 @@ const mocks = vi.hoisted(() => ({ getAnthropicClient: vi.fn(), getNonStreamingCompletion: vi.fn(), runChatLoop: vi.fn(), - listAiSkills: vi.fn(), + listResource: vi.fn(), getJob: vi.fn(), whoami: vi.fn(), workspace: 'test_workspace' as string | undefined, @@ -49,8 +50,9 @@ vi.mock('monaco-editor', () => ({ vi.mock('$lib/utils/featureUsage', () => ({ logFeatureUsage: vi.fn() })) vi.mock('$lib/gen', () => ({ - WorkspaceService: { - listAiSkills: mocks.listAiSkills + WorkspaceService: {}, + ResourceService: { + listResource: mocks.listResource }, ScriptService: {}, FlowService: {}, @@ -169,7 +171,7 @@ beforeEach(() => { mocks.isWebSearchEnabledForProvider.mockReturnValue(true) mocks.getOpenaiClient.mockReturnValue({}) mocks.getAnthropicClient.mockReturnValue({}) - mocks.listAiSkills.mockResolvedValue([]) + mocks.listResource.mockResolvedValue([]) mocks.workspace = 'test_workspace' mocks.runChatLoop.mockResolvedValue({ addedMessages: [], @@ -330,17 +332,29 @@ describe('AIChatManager global skills', () => { mocks.tryGetCurrentModel.mockReturnValue(model) }) + // Only selected skills reach the prompt, and the selection is keyed by + // workspace and account (see skills/enabledSkills.ts). + function selectSkills(workspace: string, ...paths: string[]) { + const stored = JSON.parse(localStorage.getItem('wm_skills_enabled') ?? '{}') + stored[`${workspace}:${TEST_EMAIL}`] = paths + localStorage.setItem('wm_skills_enabled', JSON.stringify(stored)) + } + it('loads skills after beforeSend commits the session workspace', async () => { - let resolveParentSkills: ((skills: { name: string; description: string }[]) => void) | undefined - const parentSkills = new Promise<{ name: string; description: string }[]>((resolve) => { + let resolveParentSkills: ((skills: unknown[]) => void) | undefined + const parentSkills = new Promise((resolve) => { resolveParentSkills = resolve }) mocks.workspace = 'parent' - mocks.listAiSkills.mockImplementation(({ workspace }: { workspace: string }) => { + selectSkills('parent', 'f/skills/parent-skill') + selectSkills('child', 'f/skills/child-skill') + mocks.listResource.mockImplementation(({ workspace }: { workspace: string }) => { if (workspace === 'parent') { return parentSkills } - return Promise.resolve([{ name: 'child-skill', description: 'child workspace skill' }]) + return Promise.resolve([ + { path: 'f/skills/child-skill', description: 'child workspace skill' } + ]) }) mocks.runChatLoop.mockImplementation(async (config: any) => { expect(config.workspace).toBe('child') @@ -362,22 +376,45 @@ describe('AIChatManager global skills', () => { } await manager.sendRequest({ instructions: 'first', mode: AIMode.GLOBAL }) - resolveParentSkills?.([{ name: 'parent-skill', description: 'parent workspace skill' }]) + resolveParentSkills?.([ + { path: 'f/skills/parent-skill', description: 'parent workspace skill' } + ]) await Promise.resolve() - expect(mocks.listAiSkills).toHaveBeenCalledWith({ workspace: 'parent' }) - expect(mocks.listAiSkills).toHaveBeenCalledWith({ workspace: 'child' }) + expect(mocks.listResource).toHaveBeenCalledWith( + expect.objectContaining({ workspace: 'parent', resourceType: 'ai_skill' }) + ) + expect(mocks.listResource).toHaveBeenCalledWith( + expect.objectContaining({ workspace: 'child', resourceType: 'ai_skill' }) + ) expect(manager.systemMessage.content).toContain('child-skill') expect(manager.systemMessage.content).not.toContain('parent-skill') }) - it('expands a leading slash skill command for the model while preserving the displayed text', async () => { - mocks.listAiSkills.mockResolvedValue([ - { name: 'review-code', description: 'review code for bugs' } + it('leaves a readable but unselected skill out of the prompt', async () => { + mocks.listResource.mockResolvedValue([ + { path: 'f/skills/selected', description: 'the one turned on' }, + { path: 'f/skills/unselected', description: 'readable but never turned on' } ]) + selectSkills('test_workspace', 'f/skills/selected') + + const manager = new AIChatManager() + manager.isSessionChat = true + await manager.refreshGlobalSkills('test_workspace') + await manager.changeMode(AIMode.GLOBAL) + + expect(manager.systemMessage.content).toContain('f/skills/selected') + expect(manager.systemMessage.content).not.toContain('f/skills/unselected') + }) + + it('expands a leading slash skill command for the model while preserving the displayed text', async () => { + mocks.listResource.mockResolvedValue([ + { path: 'u/admin/review-code', description: 'review code for bugs' } + ]) + selectSkills('test_workspace', 'u/admin/review-code') mocks.runChatLoop.mockImplementation(async (config: any) => { const userMessage = config.messages[config.messages.length - 1] - expect(userMessage.content).toContain('Use the "review-code" skill. find bugs') + expect(userMessage.content).toContain('Use the skill at "u/admin/review-code". find bugs') expect(userMessage.content).not.toContain('/review-code find bugs') const message = { role: 'assistant' as const, content: 'done' } config.addedMessages?.push(message) @@ -395,6 +432,33 @@ describe('AIChatManager global skills', () => { expect(manager.displayMessages[0]?.content).toBe('/review-code find bugs') }) + + it('does not expand a slash command two folders both answer to', async () => { + mocks.listResource.mockResolvedValue([ + { path: 'u/admin/deploy', description: 'personal deploy steps' }, + { path: 'f/team/deploy', description: 'the team deploy steps' } + ]) + selectSkills('test_workspace', 'u/admin/deploy', 'f/team/deploy') + mocks.runChatLoop.mockImplementation(async (config: any) => { + // Picking either one would silently apply instructions the user did not + // choose, so the text is left alone for the model to ask about. + const userMessage = config.messages[config.messages.length - 1] + expect(userMessage.content).toContain('/deploy ship it') + expect(userMessage.content).not.toContain('Use the skill at') + const message = { role: 'assistant' as const, content: 'done' } + config.addedMessages?.push(message) + return { + addedMessages: [message], + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + hitMaxIterations: false + } + }) + + const manager = new AIChatManager() + manager.isSessionChat = true + + await manager.sendRequest({ instructions: '/deploy ship it', mode: AIMode.GLOBAL }) + }) }) describe('AIChatManager global prompt identity', () => { @@ -404,7 +468,7 @@ describe('AIChatManager global prompt identity', () => { localStorage.clear() mocks.getCurrentModel.mockReturnValue(model) mocks.tryGetCurrentModel.mockReturnValue(model) - mocks.listAiSkills.mockResolvedValue([]) + mocks.listResource.mockResolvedValue([]) }) afterEach(() => { @@ -3099,8 +3163,8 @@ describe('AIChatManager manual compaction', () => { vi.clearAllMocks() mocks.getCurrentModel.mockReturnValue(model) mocks.tryGetCurrentModel.mockReturnValue(model) - // changeMode(GLOBAL) refreshes workspace skills; keep it a no-op here. - mocks.listAiSkills.mockResolvedValue([]) + // changeMode(GLOBAL) refreshes the selected skills; keep it a no-op here. + mocks.listResource.mockResolvedValue([]) }) function seedExchange(manager: AIChatManager) { @@ -3315,15 +3379,19 @@ describe('AIChatManager manual compaction', () => { expect(mocks.getNonStreamingCompletion).not.toHaveBeenCalled() }) - it('shadows a workspace skill that collides with a built-in command', () => { + it('shadows a selected skill that collides with a built-in command', () => { const manager = new AIChatManager() manager.globalSkills = [ - { name: 'compact', description: 'a workspace skill that happens to be named compact' }, - { name: 'review-code', description: 'review code for bugs' } + { + path: 'u/admin/compact', + name: 'compact', + description: 'a skill that happens to be named compact' + }, + { path: 'u/admin/review-code', name: 'review-code', description: 'review code for bugs' } ] - // Built-ins come first and the colliding skill is dropped, so the picker - // never renders two leaves with the same `skill:compact` key. + // Built-ins come first and the colliding skill is dropped: the built-in + // wins at execution too, so listing both would offer a row that cannot run. const names = manager.sessionCommands.map((c) => c.name) expect(names).toEqual(['compact', 'clear', 'review-code']) expect(manager.sessionCommands[0].description).toBe( @@ -3930,3 +3998,135 @@ describe('AIChatManager reasoning duration', () => { expect(assistantDurations(manager)).toEqual([3_000, 7_000]) }) }) + +describe('AIChatManager cross-tab run seams', () => { + // The whole cross-tab feature hangs off these two seams: `loading`'s edges + // are the "running here" / "safe to re-read" signals, and the resolver is + // the advisory lock. Reverting `loading` to a plain $state field would + // silently disconnect every tab. + it('reports loading transitions, and only transitions, through onRunningChanged', () => { + const manager = new AIChatManager() + const seen: boolean[] = [] + manager.onRunningChanged = (running) => seen.push(running) + manager.loading = true + manager.loading = true + manager.loading = false + manager.loading = false + expect(seen).toEqual([true, false]) + }) + + it('refuses a send while another tab holds the run, keeping the draft', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.runHeldElsewhereResolver = () => true + + const accepted = await manager.sendRequest({ instructions: 'race loser' }) + + expect(accepted).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.loading).toBe(false) + // restoreToInput falls back to the queued draft when no composer is + // mounted, so the refused text must surface there rather than vanish. + expect(manager.queuedMessage).toBe('race loser') + }) + + // A synthetic (auto-resume) prompt is client-authored: a refusal must + // release it rather than hand it back as a draft the user never wrote — + // staged instructions would otherwise block every later auto-resume. + it('releases a refused synthetic send instead of restoring it as a draft', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.runHeldElsewhereResolver = () => true + manager.instructions = 'A background job just finished.' + + const accepted = await manager.sendRequest({ synthetic: true }) + + expect(accepted).toBe(false) + expect(manager.instructions).toBe('') + expect(manager.queuedMessage).toBe('') + }) + + // The restore lanes carry no pastes, so a refusal must expand the tokens + // into the text — dangling markers with the content gone otherwise. + it('expands paste tokens into the text a refusal hands back', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.runHeldElsewhereResolver = () => true + const paste = { id: 1, lines: 1, content: 'the pasted block' } + + await manager.sendRequest({ + instructions: `see ${makePasteToken(paste)}`, + pastes: [paste] + }) + + expect(manager.queuedMessage).toBe('see the pasted block') + }) + + // The wrapper's check runs before the attachment upkeep awaits; a run + // announced by another tab during that upkeep must still be refused before + // the turn takes visible effect. + it('refuses a run announced by another tab during the preflight awaits', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + let held = false + manager.runHeldElsewhereResolver = () => held + let releaseUpkeep: (() => void) | undefined + vi.spyOn(manager.attachedFiles, 'refreshFolders').mockImplementation( + () => new Promise((resolve) => (releaseUpkeep = resolve)) + ) + + const sending = manager.sendRequest({ instructions: 'racing turn' }) + await vi.waitFor(() => expect(manager.sendInFlight).toBe(true)) + held = true + releaseUpkeep?.() + + expect(await sending).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.loading).toBe(false) + }) + + it('refuses a retry/edit while another tab holds the run, before mutating the transcript', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.displayMessages = [ + { role: 'user', content: 'original prompt', index: 0 }, + { role: 'assistant', content: 'original reply' } + ] as DisplayMessage[] + manager.messages = [ + { role: 'user', content: 'original prompt' } + ] as ChatCompletionMessageParam[] + manager.runHeldElsewhereResolver = () => true + + await manager.restartGeneration(0, 'edited prompt') + + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.displayMessages).toHaveLength(2) + expect(manager.messages).toHaveLength(1) + // The edited text survives the refusal via restoreToInput's queued-draft + // fallback. + expect(manager.queuedMessage).toBe('edited prompt') + }) + + // A cross-tab catch-up re-reads the conversation on screen; the queued + // draft is unsent user input (possibly the refusal's kept message) that + // this non-switch reload must not destroy — while a real conversation + // switch still drops it. + it('keeps the queued draft when a catch-up reload preserves it', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + vi.spyOn(manager.historyManager, 'loadPastChat').mockResolvedValue({ + id: 'c1', + actualMessages: [], + displayMessages: [], + title: '', + lastModified: 1 + } as never) + + manager.queueMessage('kept across catch-up') + await manager.loadPastChat('c1', { preserveQueue: true }) + expect(manager.queuedMessage).toBe('kept across catch-up') + + await manager.loadPastChat('c1') + expect(manager.queuedMessage).toBe('') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte index 61d9a47966..12553be961 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte @@ -13,9 +13,21 @@ import { messageDraft, segments } from './chatDraft' import { lineCountLabel } from './pasteTokens' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' + import { workspaceStore } from '$lib/stores' const aiChatManager = getAiChatManager() + // Paths in a message name items the chat's tools reach, so they resolve against the + // operating workspace, never `workspaceStore`: a fork session leaves the store on the + // navigated workspace, where a fork-only item resolves to nothing and the rest resolve + // to a different copy. + const messageWorkspace = $derived.by(() => { + // Registers the dependency that `operatingWorkspace`'s own untracked + // `get(workspaceStore)` cannot. + void $workspaceStore + return aiChatManager.operatingWorkspace + }) + // Per-message expand/collapse state for paste chips shown in the bubble. let expandedPastes = $state>(new Set()) @@ -119,7 +131,7 @@ {:else}
    {#if message.role === 'assistant'} -
    +
    {:else if message.role === 'tool'}
    - + {#if promptSettings} + + {/if}
    Model
    @@ -409,21 +418,24 @@ {/snippet} - + +{#if promptSettings} + +{/if} diff --git a/frontend/src/lib/components/table/DataTable.svelte b/frontend/src/lib/components/table/DataTable.svelte index b6ab8d00e6..d86d165883 100644 --- a/frontend/src/lib/components/table/DataTable.svelte +++ b/frontend/src/lib/components/table/DataTable.svelte @@ -15,6 +15,24 @@ let tableHeight: number = $state(0) const dispatch = createEventDispatcher() let tableContainer: HTMLDivElement | undefined = $state() + let tableEl: HTMLTableElement | undefined = $state() + + // A pinned actions column only earns a seam once something can actually pass under it, + // so the overflow is measured rather than assumed: on a table that fits, the column + // should be indistinguishable from an ordinary one. + let xOverflowing = $state(false) + $effect(() => { + const container = tableContainer + const table = tableEl + if (!container || !table) return + // Sub-pixel widths make an exactly-fitting table read as 0.5px over. + const measure = () => (xOverflowing = container.scrollWidth > container.clientWidth + 1) + measure() + const observer = new ResizeObserver(measure) + observer.observe(container) + observer.observe(table) + return () => observer.disconnect() + }) interface Props { paginated?: boolean currentPage?: number @@ -130,11 +148,15 @@ >
    - +
    {@render children?.()}
    {@render emptyMessage?.()} diff --git a/frontend/src/lib/components/table/Row.svelte b/frontend/src/lib/components/table/Row.svelte index c3cbd68665..5854a88d3b 100644 --- a/frontend/src/lib/components/table/Row.svelte +++ b/frontend/src/lib/components/table/Row.svelte @@ -28,7 +28,9 @@ void ) => Promise>> create?: (workspace: string, requestBody: any) => Promise + /** + * Write back an existing trigger's config. Used to point an imported trigger at a + * resource the workspace already has; `schedule` has none because its own service + * takes a different body shape, handled by the caller. + */ + update?: (workspace: string, path: string, requestBody: any) => Promise } > = { http: { @@ -101,7 +107,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'authentication_resource_path', list: (workspace) => HttpTriggerService.listHttpTriggers({ workspace }), create: (workspace, requestBody) => - HttpTriggerService.createHttpTrigger({ workspace, requestBody }) + HttpTriggerService.createHttpTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + HttpTriggerService.updateHttpTrigger({ workspace, path, requestBody }) }, websocket: { configFields: [ @@ -119,7 +127,9 @@ export const TRIGGER_KINDS: Record< note: 'Reconnect WebSocket auth after import if external service requires it.', list: (workspace) => WebsocketTriggerService.listWebsocketTriggers({ workspace }), create: (workspace, requestBody) => - WebsocketTriggerService.createWebsocketTrigger({ workspace, requestBody }) + WebsocketTriggerService.createWebsocketTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + WebsocketTriggerService.updateWebsocketTrigger({ workspace, path, requestBody }) }, schedule: { configFields: [ @@ -180,7 +190,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => KafkaTriggerService.listKafkaTriggers({ workspace }), create: (workspace, requestBody) => - KafkaTriggerService.createKafkaTrigger({ workspace, requestBody }) + KafkaTriggerService.createKafkaTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + KafkaTriggerService.updateKafkaTrigger({ workspace, path, requestBody }) }, nats: { configFields: [ @@ -197,7 +209,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => NatsTriggerService.listNatsTriggers({ workspace }), create: (workspace, requestBody) => - NatsTriggerService.createNatsTrigger({ workspace, requestBody }) + NatsTriggerService.createNatsTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + NatsTriggerService.updateNatsTrigger({ workspace, path, requestBody }) }, sqs: { configFields: [ @@ -212,7 +226,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => SqsTriggerService.listSqsTriggers({ workspace }), create: (workspace, requestBody) => - SqsTriggerService.createSqsTrigger({ workspace, requestBody }) + SqsTriggerService.createSqsTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + SqsTriggerService.updateSqsTrigger({ workspace, path, requestBody }) }, mqtt: { configFields: [ @@ -228,7 +244,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'mqtt_resource_path', list: (workspace) => MqttTriggerService.listMqttTriggers({ workspace }), create: (workspace, requestBody) => - MqttTriggerService.createMqttTrigger({ workspace, requestBody }) + MqttTriggerService.createMqttTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + MqttTriggerService.updateMqttTrigger({ workspace, path, requestBody }) }, amqp: { configFields: ['amqp_resource_path', 'queue_name', 'exchange', 'options'], @@ -238,7 +256,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'amqp_resource_path', list: (workspace) => AmqpTriggerService.listAmqpTriggers({ workspace }), create: (workspace, requestBody) => - AmqpTriggerService.createAmqpTrigger({ workspace, requestBody }) + AmqpTriggerService.createAmqpTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + AmqpTriggerService.updateAmqpTrigger({ workspace, path, requestBody }) }, gcp: { configFields: [ @@ -256,7 +276,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => GcpTriggerService.listGcpTriggers({ workspace }), create: (workspace, requestBody) => - GcpTriggerService.createGcpTrigger({ workspace, requestBody }) + GcpTriggerService.createGcpTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + GcpTriggerService.updateGcpTrigger({ workspace, path, requestBody }) }, azure: { configFields: [ @@ -274,7 +296,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => AzureTriggerService.listAzureTriggers({ workspace }), create: (workspace, requestBody) => - AzureTriggerService.createAzureTrigger({ workspace, requestBody }) + AzureTriggerService.createAzureTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + AzureTriggerService.updateAzureTrigger({ workspace, path, requestBody }) }, postgres: { configFields: [ @@ -288,7 +312,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'postgres_resource_path', list: (workspace) => PostgresTriggerService.listPostgresTriggers({ workspace }), create: (workspace, requestBody) => - PostgresTriggerService.createPostgresTrigger({ workspace, requestBody }) + PostgresTriggerService.createPostgresTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + PostgresTriggerService.updatePostgresTrigger({ workspace, path, requestBody }) }, email: { configFields: ['local_part', 'workspaced_local_part'], @@ -297,7 +323,9 @@ export const TRIGGER_KINDS: Record< note: 'Email address regenerates on import.', list: (workspace) => EmailTriggerService.listEmailTriggers({ workspace }), create: (workspace, requestBody) => - EmailTriggerService.createEmailTrigger({ workspace, requestBody }) + EmailTriggerService.createEmailTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + EmailTriggerService.updateEmailTrigger({ workspace, path, requestBody }) } } diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 5979e8a535..65187d98c8 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -15,8 +15,6 @@ import { supportsAutocomplete } from '../copilot/utils' import TestAiKey from '../copilot/TestAIKey.svelte' import Label from '../Label.svelte' - import AiSkillsSettings from './AiSkillsSettings.svelte' - import { isGlobalAiEnabled } from '../copilot/chat/global/gate' import SettingsPageHeader from '../settings/SettingsPageHeader.svelte' import ResourcePicker from '../ResourcePicker.svelte' import Toggle from '../Toggle.svelte' @@ -80,6 +78,7 @@ let modelPricing: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) + let copilotDisabled = $state(false) // --- Initial state for dirty tracking --- let initialAiProviders: Exclude = $state({}) @@ -90,6 +89,7 @@ let initialMaxTokensPerModel: Record = $state({}) let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) + let initialCopilotDisabled = $state(false) let lastLoadedConfigKey = $state(undefined) function clone(v: T): T { @@ -117,6 +117,7 @@ customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) modelPricing = clone(config?.model_pricing ?? {}) + copilotDisabled = config?.copilot_disabled === true for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -133,6 +134,7 @@ initialMaxTokensPerModel = clone(maxTokensPerModel) initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) + initialCopilotDisabled = copilotDisabled } export function loadFromConfig(config: AIConfig | undefined) { @@ -148,6 +150,7 @@ customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) modelPricing = clone(initialModelPricing) + copilotDisabled = initialCopilotDisabled } $effect(() => { @@ -182,7 +185,8 @@ codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || - JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) + JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) || + copilotDisabled !== initialCopilotDisabled ) $effect(() => { @@ -287,6 +291,8 @@ .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) + // The flag is the one thing a workspace on instance defaults still stores of its own. + const copilot_disabled = copilotDisabled ? true : undefined return Object.keys(aiProviders ?? {}).length > 0 ? { providers: aiProviders, @@ -296,9 +302,10 @@ custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, - model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined + model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined, + copilot_disabled } - : {} + : { copilot_disabled } } function isSaveDisabled(): boolean { @@ -607,10 +614,6 @@
    {/if} - - {#if promptScope === 'workspace' && isGlobalAiEnabled()} - - {/if}
    {/if} -{#if showWorkspaceOverrideEditor} - +{#if promptScope === 'workspace'} + + { + copilotDisabled = e.detail + }} + options={{ right: 'Hide AI sessions in this workspace' }} + /> + {/if} + + + diff --git a/frontend/src/lib/components/workspaceSettings/AiSkillsSettings.svelte b/frontend/src/lib/components/workspaceSettings/AiSkillsSettings.svelte deleted file mode 100644 index 78e7989c3f..0000000000 --- a/frontend/src/lib/components/workspaceSettings/AiSkillsSettings.svelte +++ /dev/null @@ -1,658 +0,0 @@ - - - - -{#snippet pasteZone()} - - {#if editingOriginalName || pasteContent.trim()} -
    - {isDirty && pasteError ? pasteError : ''} - -
    - {/if} -{/snippet} - - - {#snippet headerAction()} -
    - {#if manageMode} - - - {:else} - {#if skills.length > 1} - - {/if} - - {/if} -
    - {#if $addMenuOpen} -
    - - -
    - {/if} - {/snippet} - -
    - {#if skills.length === 0} -
    - No custom skills yet -
    - {:else} -
    - {#if manageMode} -
    - 0 && !allSelected} - onChange={toggleSelectAll} - /> - - {selectedCount ? `${selectedCount} selected` : 'Select all'} - -
    - {/if} - {#each skills as skill (skill.name)} -
    - {#if manageMode} - - {:else} -
    -
    {skill.name}
    -
    {skill.description}
    - -
    - openSkill(skill.name, 'edit') - }, - { - displayName: 'Delete', - icon: Trash2, - type: 'delete', - action: () => (toDelete = skill.name) - } - ]} - /> - {/if} -
    - {/each} -
    - {/if} -
    -     - - - - - - {#snippet headerRight()} - {#if editingOriginalName} - - {#snippet children({ item })} - - - {/snippet} - - {/if} - {/snippet} -
    - {#if detailMode === 'view'} -
    - {#if viewParsed.description} -

    {viewParsed.description}

    - {/if} -
    - -
    -
    - {:else} - {@render pasteZone()} - {/if} -
    -     - - { - const toImport = [...pendingNew, ...pendingConflicts.filter((s) => overwriteChoices[s.name])] - const skipped = pendingSkipped - pendingImport = undefined - pendingSkipped = [] - overwriteChoices = {} - if (toImport.length) await uploadSkills(toImport, skipped) - else sendUserToast('No skills imported.') - }} - onCanceled={() => { - pendingImport = undefined - pendingSkipped = [] - overwriteChoices = {} - }} -> -
    - {#if pendingNew.length} -
    - Add {pendingNew.length} new skill(s): - {pendingNew.map((s) => s.name).join(', ')} -
    - {/if} - {#if pendingConflicts.length} -
    - - {pendingConflicts.length} skill(s) already exist — choose which to overwrite: - -
    - {#each pendingConflicts as conflict (conflict.name)} -
    - {conflict.name} - -
    - {/each} -
    -
    - {/if} - {#if pendingSkipped.length} - {pendingSkipped.length} file(s) will be skipped. - {/if} -
    -     - - { - const name = toDelete - toDelete = undefined - if (name) await deleteSkill(name) - }} - onCanceled={() => (toDelete = undefined)} -> - - Delete the skill {toDelete}? The AI chat will no longer be able to use it. - - - - { - confirmBatchDelete = false - await deleteSelected() - }} - onCanceled={() => (confirmBatchDelete = false)} -> - - Delete {selectedCount} selected skill(s)? The AI chat will no longer be able to use them. - - diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts index 669c0a43de..1833a33b33 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -16,6 +16,8 @@ import { collectExportVarPaths, extractTriggerConfigResourceRefs, extractVarRefsFromValue, + projectReferencesResource, + textHoldsBarePath, type ProjectExport, type FetchedItem, type ItemRef @@ -867,3 +869,46 @@ describe('flow_env and preprocessor_module', () => { expect(out.flow_env.PLAIN).toBe('not-a-ref') }) }) + +describe('projectReferencesResource', () => { + /** + * A project declares one resource per `resource-` input schema as well as one per + * `$res:` reference, so an app pinning `f/proj/google_calendar` for a script whose schema + * says `resource-gcal` ships both it and an unreferenced `f/proj/gcal`. Only the pinned + * one has to hold a credential for the project to work. + */ + const bundle = { + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [{ path: 'f/proj/send', content: 'const c = "$res:f/proj/google_calendar"' }], + flows: [], + apps: [], + resources: [ + { path: 'f/proj/gcal', resource_type: 'gcal' }, + { path: 'f/proj/google_calendar', resource_type: 'gcal' }, + { path: 'f/proj/db', resource_type: 'postgresql' } + ], + triggers: [{ path: 'f/proj/ingest', config: { postgres_resource_path: 'f/proj/db' } }] + } as unknown as ProjectExport + + it("sees a $res: token and a trigger's bare path, and not a stub nothing points at", () => { + expect(projectReferencesResource(bundle, 'f/proj/google_calendar')).toBe(true) + expect(projectReferencesResource(bundle, 'f/proj/db')).toBe(true) + // Declared only because a script's input schema names the type. + expect(projectReferencesResource(bundle, 'f/proj/gcal')).toBe(false) + }) +}) + +describe('textHoldsBarePath', () => { + // Gates three deletion decisions, and its two directions cost differently: a false yes + // keeps a stub nobody needed, a false no deletes one something still reads. + const P = 'f/proj/db' + it.each([ + ['a token only', `const c = "$res:${P}"`, false], + ['a path written in code', `await getResource("${P}")`, true], + ['both spellings', `"$res:${P}"; getResource("${P}")`, true], + ['a longer path that starts the same', `await getResource("${P}_prod")`, false], + ['the same path inside a longer token', `"$res:${P}_prod"`, false] + ])('%s', (_label, text, expected) => { + expect(textHoldsBarePath(text, P)).toBe(expected) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts index 1fd62498d5..8adedeb559 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -238,6 +238,89 @@ export function extractTriggerConfigResourceRefs(config: any): string[] { return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) } +/** + * Whether a value reaches a resource, in either spelling a rewrite has to handle: a + * `$res:`/`res://` token embedded in content, or the bare path standing alone as a string + * the way trigger configs hold it (`kafka_resource_path: "f/slug/db"`). + * + * Matching the parsed structure rather than its serialization is what keeps + * `f/slug/db` out of `$res:f/slug/db_prod`. + */ +export function referencesResourcePath(value: unknown, path: string): boolean { + const walk = (v: any): boolean => { + if (typeof v === 'string') { + if (v === path) return true + RES_TOKEN_RE.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = RES_TOKEN_RE.exec(v)) !== null) if (m[1] === path) return true + return false + } + if (Array.isArray(v)) return v.some(walk) + if (v && typeof v === 'object') return Object.values(v).some(walk) + return false + } + return walk(value) +} + +/** + * Whether a `$res:`/`res://` token for this path survives anywhere in the value — the one + * spelling the rewriters relocate, and so the only one whose survival means a rewrite did + * not take. A bare path is deliberately not matched: nothing here moves one, so its presence + * says nothing about whether the rewrite worked. + */ +export function holdsResourceToken(value: unknown, path: string): boolean { + const walk = (v: any): boolean => { + if (typeof v === 'string') { + RES_TOKEN_RE.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = RES_TOKEN_RE.exec(v)) !== null) if (m[1] === path) return true + return false + } + if (Array.isArray(v)) return v.some(walk) + if (v && typeof v === 'object') return Object.values(v).some(walk) + return false + } + return walk(value) +} + +/** + * Whether the text names the resource somewhere no rewriter reaches — a path written on its + * own rather than inside a `$res:` token, the way `getResource("f/proj/db")` does. The + * tokens are stripped first so the ones a rewrite would move do not count, and the match is + * bounded so `f/proj/db` is not found inside `f/proj/db_prod`. + */ +export function textHoldsBarePath(text: string, path: string): boolean { + const withoutTokens = text.replace(RES_TOKEN_RE, '') + const boundary = /[\w\-./]/ + for (let i = withoutTokens.indexOf(path); i !== -1; i = withoutTokens.indexOf(path, i + 1)) { + const before = withoutTokens[i - 1] ?? '' + const after = withoutTokens[i + path.length] ?? '' + if (!boundary.test(before) && !boundary.test(after)) return true + } + return false +} + +/** + * Whether anything the project ships points at one of its own resources. + * + * A project declares two kinds of resource. One is referenced — an app pins `$res:` for a + * script argument, a trigger names it — and the project does not work until it holds a + * credential. The other is minted from a `resource-` input schema: it names a type + * a script accepts, nothing points at it, and a standalone run picks a resource in the + * argument picker instead. Only the first kind is worth asking anyone to fill in. + * + * The `resources` list is excluded from the walk because a stub's own declaration carries + * its path, which would make every stub look referenced. + */ +export function projectReferencesResource(bundle: ProjectExport, path: string): boolean { + const { resources: _resources, ...rest } = bundle as any + if (referencesResourcePath(rest, path)) return true + // A script that reads the resource by name rather than through a `$res:` token still needs + // it filled in. Asked about is the safe side of this answer: the cost of a wrong yes is a + // row nobody had to act on, and of a wrong no a credential nobody was told to set up. + return textHoldsBarePath(JSON.stringify(rest ?? {}), path) +} + /** * Trigger configs reference resources as plain path strings (e.g. * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting diff --git a/frontend/src/lib/folderDraft.test.ts b/frontend/src/lib/folderDraft.test.ts new file mode 100644 index 0000000000..75b8fa6dd0 --- /dev/null +++ b/frontend/src/lib/folderDraft.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect } from 'vitest' +import { + folderPermissionDiff, + isFolderDraftDirty, + type FolderDraft, + type FolderMember, + type FolderRole +} from './folderDraft' + +function member(role: FolderRole): FolderMember { + return { owner_name: 'u/alice', role } +} + +function baseline(): FolderDraft { + return { + summary: 'Reporting jobs', + labels: ['prod'], + defaultPermissionedAs: [{ path_glob: '**', permissioned_as: 'u/admin' }], + perms: [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'viewer' } + ] + } +} + +describe('folderPermissionDiff', () => { + // The whole transition matrix: which endpoint each role change maps to. `admin` lives in + // `owners` and the other two in `extra_perms`, so leaving admin is the one transition that + // cannot go through the ACL endpoint. + const transitions: Array<[from: FolderRole | 'absent', to: FolderRole, expected: unknown]> = [ + ['absent', 'viewer', { kind: 'setAcl', owner: 'u/alice', write: false }], + ['absent', 'writer', { kind: 'setAcl', owner: 'u/alice', write: true }], + ['absent', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['viewer', 'writer', { kind: 'setAcl', owner: 'u/alice', write: true }], + ['viewer', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['writer', 'viewer', { kind: 'setAcl', owner: 'u/alice', write: false }], + ['writer', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['admin', 'viewer', { kind: 'demoteAdmin', owner: 'u/alice', write: false }], + ['admin', 'writer', { kind: 'demoteAdmin', owner: 'u/alice', write: true }] + ] + + it.each(transitions)('%s → %s', (from, to, expected) => { + const prev = from === 'absent' ? [] : [member(from)] + expect(folderPermissionDiff(prev, [member(to)])).toEqual([expected]) + }) + + it.each(['viewer', 'writer', 'admin'] as const)('%s → removed drops owner and acl', (role) => { + expect(folderPermissionDiff([member(role)], [])).toEqual([{ kind: 'remove', owner: 'u/alice' }]) + }) + + it.each(['viewer', 'writer', 'admin'] as const)('%s unchanged calls nothing', (role) => { + expect(folderPermissionDiff([member(role)], [member(role)])).toEqual([]) + }) + + // The caller is a folder admin only through `g/ops`, so that demotion is the one the write + // policy refuses. Sent first it takes the rest of the save down with it. + it('gives up the caller own admin last', () => { + const prev: FolderMember[] = [ + { owner_name: 'g/ops', role: 'admin' }, + { owner_name: 'u/bob', role: 'viewer' } + ] + const next: FolderMember[] = [ + { owner_name: 'g/ops', role: 'viewer' }, + { owner_name: 'u/bob', role: 'admin' } + ] + expect(folderPermissionDiff(prev, next, ['u/alice', 'g/ops'])).toEqual([ + { kind: 'grantAdmin', owner: 'u/bob' }, + { kind: 'demoteAdmin', owner: 'g/ops', write: false } + ]) + }) + + // `g/z` is a group the caller belongs to but holds no admin through, so removing it is an + // ordinary call — queued behind the refused one it would never run. + it('defers only the rows the caller is an admin through', () => { + const prev: FolderMember[] = [ + { owner_name: 'g/a', role: 'admin' }, + { owner_name: 'g/z', role: 'viewer' } + ] + expect(folderPermissionDiff(prev, [], ['u/alice', 'g/a', 'g/z'])).toEqual([ + { kind: 'remove', owner: 'g/z' }, + { kind: 'remove', owner: 'g/a' } + ]) + }) + + it('touches only the members that changed', () => { + const prev: FolderMember[] = [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'viewer' }, + { owner_name: 'g/ops', role: 'writer' } + ] + const next: FolderMember[] = [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'writer' } + ] + expect(folderPermissionDiff(prev, next)).toEqual([ + { kind: 'setAcl', owner: 'g/all', write: true }, + { kind: 'remove', owner: 'g/ops' } + ]) + }) +}) + +describe('isFolderDraftDirty', () => { + it('is clean against its own baseline', () => { + expect(isFolderDraftDirty(baseline(), baseline())).toBe(false) + }) + + it('is clean before anything has loaded', () => { + expect(isFolderDraftDirty(baseline(), undefined)).toBe(false) + }) + + // A reload rebuilds the members in the server's order, which is not the order they were + // added in. Order-sensitive, an applied change would keep Save lit with nothing to send. + it('ignores the order the members are held in', () => { + const reordered = baseline() + reordered.perms = [...reordered.perms].reverse() + expect(isFolderDraftDirty(reordered, baseline())).toBe(false) + }) + + // Enumerated from the value itself rather than a hand-written list: a field added to + // `FolderDraft` and to `baseline()` is covered here without anyone remembering to add a + // case. An edit this misses is one the drawer discards without asking. + it.each(Object.keys(baseline()) as Array)('notices a change to %s', (key) => { + const edited = baseline() + if (key === 'summary') edited.summary = 'Something else' + else if (key === 'labels') edited.labels = [...edited.labels, 'staging'] + else if (key === 'defaultPermissionedAs') edited.defaultPermissionedAs = [] + else if (key === 'perms') edited.perms[1].role = 'writer' + else throw new Error(`no edit defined for ${key} — add one so the field stays covered`) + + expect(isFolderDraftDirty(edited, baseline())).toBe(true) + }) + + it('notices a member added and a member removed', () => { + const added = baseline() + added.perms.push({ owner_name: 'g/ops', role: 'writer' }) + expect(isFolderDraftDirty(added, baseline())).toBe(true) + + const removed = baseline() + removed.perms.pop() + expect(isFolderDraftDirty(removed, baseline())).toBe(true) + }) + + it('is clean again once the baseline catches up', () => { + const saved = baseline() + saved.summary = 'Renamed' + expect(isFolderDraftDirty(saved, structuredClone(saved))).toBe(false) + }) +}) diff --git a/frontend/src/lib/folderDraft.ts b/frontend/src/lib/folderDraft.ts new file mode 100644 index 0000000000..ade1b43d71 --- /dev/null +++ b/frontend/src/lib/folderDraft.ts @@ -0,0 +1,96 @@ +import { deepEqual } from 'fast-equals' +import type { FolderDefaultPermissionedAs } from '$lib/gen' + +/** What a member may hold on a folder. `admin` is the `owners` array server-side; `writer` + * and `viewer` are the `true`/`false` entries of `extra_perms`. */ +export type FolderRole = 'viewer' | 'writer' | 'admin' + +export type FolderMember = { owner_name: string; role: FolderRole } + +/** Everything the folder editor can change, held as one value so the whole edit is one + * comparison against the loaded folder and one Save. */ +export type FolderDraft = { + summary: string + labels: string[] + defaultPermissionedAs: FolderDefaultPermissionedAs + perms: FolderMember[] +} + +/** Whether the draft still matches the folder it was loaded from. Every field of + * `FolderDraft` participates, so a field added to the type is covered by construction — + * which is what the discard guard depends on: an edit this misses is an edit the drawer + * throws away without asking. No baseline means nothing has loaded yet, so nothing to lose. */ +export function isFolderDraftDirty(draft: FolderDraft, baseline: FolderDraft | undefined): boolean { + return baseline != undefined && !deepEqual(sortedMembers(draft), sortedMembers(baseline)) +} + +/** Members are a set, but a reload rebuilds them in the server's `extra_perms` key order while + * the draft keeps the order they were added in. Compared as-is, a change that has already been + * applied still reads as dirty. Labels and rules keep their order, which is meaningful. */ +function sortedMembers(value: FolderDraft): FolderDraft { + return { + ...value, + perms: [...value.perms].sort((a, b) => a.owner_name.localeCompare(b.owner_name)) + } +} + +/** One backend call the folder's members need. Kept as data so the mapping from role + * transitions to endpoints can be read — and tested — without a server. */ +export type FolderPermissionCall = + /** `addowner`: appends to `owners` and sets `extra_perms[owner] = true`. */ + | { kind: 'grantAdmin'; owner: string } + /** `removeowner` with a write flag: takes the member out of `owners` and sets their + * level. The only way down from admin. */ + | { kind: 'demoteAdmin'; owner: string; write: boolean } + /** `acls/add`: sets `extra_perms[owner]`, for a member who is not an admin. */ + | { kind: 'setAcl'; owner: string; write: boolean } + /** Both removals. `removeowner` without a write only drops the member from `owners`, + * leaving their `extra_perms` entry — alone it demotes an admin rather than removing + * them, so the ACL delete is not optional. */ + | { kind: 'remove'; owner: string } + +/** The calls that turn `prev` into `next`. Members whose role is unchanged produce none. + * + * `callerOwners` is the caller's own `u/name` plus every group they belong to. Giving up the + * last of those that is in `owners` goes last: the write policy checks the row the update + * would produce, so that call is refused for anyone but a workspace admin, and sent early it + * takes the rest of the save with it. */ +export function folderPermissionDiff( + prev: FolderMember[], + next: FolderMember[], + callerOwners?: string[] +): FolderPermissionCall[] { + const previousRole = new Map(prev.map((p) => [p.owner_name, p.role])) + const calls: FolderPermissionCall[] = [] + + for (const member of next) { + const before = previousRole.get(member.owner_name) + if (before === member.role) continue + if (member.role === 'admin') { + calls.push({ kind: 'grantAdmin', owner: member.owner_name }) + } else if (before === 'admin') { + calls.push({ + kind: 'demoteAdmin', + owner: member.owner_name, + write: member.role === 'writer' + }) + } else { + calls.push({ kind: 'setAcl', owner: member.owner_name, write: member.role === 'writer' }) + } + } + + const kept = new Set(next.map((n) => n.owner_name)) + for (const member of prev) { + if (kept.has(member.owner_name)) continue + calls.push({ kind: 'remove', owner: member.owner_name }) + } + + // `previousRole === 'admin'` is what makes it a handle: `callerOwners` lists every group + // the caller belongs to, and one holding only a viewer or writer row is not in `owners`, + // so removing it is an ordinary call that should not queue behind the fatal one. + const revokesCaller = (call: FolderPermissionCall) => + (call.kind === 'demoteAdmin' || call.kind === 'remove') && + (callerOwners?.includes(call.owner) ?? false) && + previousRole.get(call.owner) === 'admin' + return [...calls.filter((c) => !revokesCaller(c)), ...calls.filter(revokesCaller)] +} diff --git a/frontend/src/lib/groupDraft.test.ts b/frontend/src/lib/groupDraft.test.ts new file mode 100644 index 0000000000..2b8ef8c180 --- /dev/null +++ b/frontend/src/lib/groupDraft.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest' +import { groupMemberDiff, isGroupDraftDirty, type GroupDraft, type GroupRole } from './groupDraft' + +function baseline(): GroupDraft { + return { + summary: 'On-call engineers', + members: [ + { member_name: 'admin', role: 'admin' }, + { member_name: 'alice', role: 'member' } + ] + } +} + +describe('groupMemberDiff', () => { + // The whole transition matrix: which endpoints each role change maps to. A role is a + // membership row plus an ACL entry, so only the halves that actually change are sent — + // an extra call would log a permission-history row for something that did not move. + const transitions: Array< + [from: GroupRole | 'absent', to: GroupRole | 'absent', expected: unknown[]] + > = [ + ['absent', 'member', [{ kind: 'addUser', username: 'bob' }]], + ['absent', 'manager', [{ kind: 'setAcl', username: 'bob' }]], + [ + 'absent', + 'admin', + [ + { kind: 'addUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' } + ] + ], + ['member', 'admin', [{ kind: 'setAcl', username: 'bob' }]], + [ + 'member', + 'manager', + [ + { kind: 'removeUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' } + ] + ], + ['manager', 'admin', [{ kind: 'addUser', username: 'bob' }]], + [ + 'manager', + 'member', + [ + { kind: 'addUser', username: 'bob' }, + { kind: 'removeAcl', username: 'bob' } + ] + ], + ['admin', 'member', [{ kind: 'removeAcl', username: 'bob' }]], + ['admin', 'manager', [{ kind: 'removeUser', username: 'bob' }]], + ['member', 'absent', [{ kind: 'removeUser', username: 'bob' }]], + ['manager', 'absent', [{ kind: 'removeAcl', username: 'bob' }]], + [ + 'admin', + 'absent', + [ + { kind: 'removeUser', username: 'bob' }, + { kind: 'removeAcl', username: 'bob' } + ] + ] + ] + + for (const [from, to, expected] of transitions) { + it(`${from} to ${to}`, () => { + const prev = from === 'absent' ? [] : [{ member_name: 'bob', role: from }] + const next = to === 'absent' ? [] : [{ member_name: 'bob', role: to }] + expect(groupMemberDiff(prev, next)).toEqual(expected) + }) + } + + it('sends nothing for an unchanged member', () => { + expect(groupMemberDiff(baseline().members, baseline().members)).toEqual([]) + }) + + it('revokes the caller last so the rest of the save stays authorized', () => { + const prev = [{ member_name: 'admin', role: 'admin' as GroupRole }] + const next = [ + { member_name: 'admin', role: 'member' as GroupRole }, + { member_name: 'bob', role: 'admin' as GroupRole } + ] + expect(groupMemberDiff(prev, next, 'admin')).toEqual([ + { kind: 'addUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' }, + { kind: 'removeAcl', username: 'admin' } + ]) + }) +}) + +describe('isGroupDraftDirty', () => { + it('is clean against an equal baseline and dirty on any field', () => { + expect(isGroupDraftDirty(baseline(), baseline())).toBe(false) + expect(isGroupDraftDirty({ ...baseline(), summary: 'Other' }, baseline())).toBe(true) + expect( + isGroupDraftDirty( + { ...baseline(), members: [{ member_name: 'admin', role: 'member' }] }, + baseline() + ) + ).toBe(true) + }) + + it('is clean while nothing has loaded', () => { + expect(isGroupDraftDirty(baseline(), undefined)).toBe(false) + }) + + // A reload rebuilds the members in the server's order, which is not the order they were + // added in. Order-sensitive, an applied change would keep Save lit with nothing to send. + it('ignores the order the members are held in', () => { + const reordered = baseline() + reordered.members = [...reordered.members].reverse() + expect(isGroupDraftDirty(reordered, baseline())).toBe(false) + }) +}) diff --git a/frontend/src/lib/groupDraft.ts b/frontend/src/lib/groupDraft.ts new file mode 100644 index 0000000000..dfd1767c30 --- /dev/null +++ b/frontend/src/lib/groupDraft.ts @@ -0,0 +1,99 @@ +import { deepEqual } from 'fast-equals' + +/** What a member may hold on a group. `member` is the `usr_to_group` row server-side and + * `manager` is the `true` entry in `extra_perms`; `admin` is both at once. */ +export type GroupRole = 'member' | 'manager' | 'admin' + +export type GroupMember = { member_name: string; role: GroupRole } + +/** Everything the group editor can change, held as one value so the whole edit is one + * comparison against the loaded group and one Save. */ +export type GroupDraft = { + summary: string + members: GroupMember[] +} + +/** Whether the draft still matches the group it was loaded from. Every field of `GroupDraft` + * participates, so a field added to the type is covered by construction — which is what the + * discard guard depends on: an edit this misses is an edit the drawer throws away without + * asking. No baseline means nothing has loaded yet, so nothing to lose. */ +export function isGroupDraftDirty(draft: GroupDraft, baseline: GroupDraft | undefined): boolean { + return baseline != undefined && !deepEqual(sortedMembers(draft), sortedMembers(baseline)) +} + +/** Members are a set, but a reload rebuilds them in the server's order while the draft keeps + * the order they were added in. Compared as-is, a change that has already been applied still + * reads as dirty. */ +function sortedMembers(value: GroupDraft): GroupDraft { + return { + ...value, + members: [...value.members].sort((a, b) => a.member_name.localeCompare(b.member_name)) + } +} + +/** One backend call a group's members need. Kept as data so the mapping from role + * transitions to endpoints can be read — and tested — without a server. */ +export type GroupMemberCall = + /** `addUserToGroup` / `removeUserToGroup`: the `usr_to_group` row. */ + | { kind: 'addUser'; username: string } + | { kind: 'removeUser'; username: string } + /** `acls/add` / `acls/remove` on kind `group_`: the write entry that lets someone + * manage the group. */ + | { kind: 'setAcl'; username: string } + | { kind: 'removeAcl'; username: string } + +/** The two independent things a role is made of: belonging to the group, and holding the + * write entry that lets you manage it. Every role is one combination of the two, which is + * why a transition needs at most one call per flag. */ +function flagsOf(role: GroupRole | undefined): { belongs: boolean; manages: boolean } { + return { + belongs: role === 'member' || role === 'admin', + manages: role === 'manager' || role === 'admin' + } +} + +/** The calls that turn `prev` into `next`. Members whose role is unchanged produce none, and + * a member dropped from `next` is treated as holding neither flag — which is what removing + * one means. + * + * `require_is_owner` authorizes each of these against `extra_perms['u/']`, so the + * caller's own revocation goes last: in row order it lands first and the rest 403s. */ +export function groupMemberDiff( + prev: GroupMember[], + next: GroupMember[], + caller?: string +): GroupMemberCall[] { + const previousRole = new Map(prev.map((p) => [p.member_name, p.role])) + const calls: GroupMemberCall[] = [] + + const transition = ( + username: string, + before: GroupRole | undefined, + after: GroupRole | undefined + ) => { + const from = flagsOf(before) + const to = flagsOf(after) + if (to.belongs !== from.belongs) { + calls.push({ kind: to.belongs ? 'addUser' : 'removeUser', username }) + } + if (to.manages !== from.manages) { + calls.push({ kind: to.manages ? 'setAcl' : 'removeAcl', username }) + } + } + + for (const member of next) { + const before = previousRole.get(member.member_name) + if (before === member.role) continue + transition(member.member_name, before, member.role) + } + + const kept = new Set(next.map((n) => n.member_name)) + for (const member of prev) { + if (kept.has(member.member_name)) continue + transition(member.member_name, member.role, undefined) + } + + const revokesCaller = (call: GroupMemberCall) => + call.kind === 'removeAcl' && call.username === caller + return [...calls.filter((c) => !revokesCaller(c)), ...calls.filter(revokesCaller)] +} diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 2862fc7768..ee60298db4 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28184/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28910/git-sync-init-repository-windmill", + "gitInitRepo": "hub/28930/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", diff --git a/frontend/src/lib/importWizard/execution.svelte.ts b/frontend/src/lib/importWizard/execution.svelte.ts index 06c33f3222..6a2c39b0b6 100644 --- a/frontend/src/lib/importWizard/execution.svelte.ts +++ b/frontend/src/lib/importWizard/execution.svelte.ts @@ -8,9 +8,10 @@ import { installProject, type InstallResult } from '$lib/components/workspaceSettings/projectInstall' -import type { - ProjectExport, - ProjectMigration +import { + projectReferencesResource, + type ProjectExport, + type ProjectMigration } from '$lib/components/workspaceSettings/projectBundle' import { planWorkspaceId, type ImportPlan } from './plan' import { probeImportedPaths, probeWorkspace } from './probe' @@ -110,12 +111,16 @@ export class ImportExecution { } /** - * How many resources the project shipped. Every one arrives as an empty stub — - * the hub never publishes resource values — so a non-zero count means the setup - * step has something to offer. + * How many of the project's resources the setup step will ask about. Every resource + * arrives as an empty stub — the hub never publishes resource values — but only the ones + * something in the project points at have to hold a credential for it to work, and those + * are the ones the step lists. Counting all of them here would offer a fourth step that + * then has nothing on it. */ get resourceCount(): number { - return this.#export?.resources?.length ?? 0 + const e = this.#export + if (!e) return 0 + return (e.resources ?? []).filter((r) => projectReferencesResource(e, String(r.path))).length } get extraCounts(): { triggers: number; migrations: number } | undefined { @@ -453,9 +458,7 @@ export class ImportExecution { const problems: string[] = [] if (failed > 0) problems.push(`${failed} item${failed === 1 ? '' : 's'} failed to import`) if (badMigrations > 0) { - problems.push( - `${badMigrations} data table migration${badMigrations === 1 ? '' : 's'} failed` - ) + problems.push(`${badMigrations} data table migration${badMigrations === 1 ? '' : 's'} failed`) } if (problems.length) this.error = `${problems.join(', ')}.` } diff --git a/frontend/src/lib/importWizard/retargetDeployed.test.ts b/frontend/src/lib/importWizard/retargetDeployed.test.ts new file mode 100644 index 0000000000..adf7a4c795 --- /dev/null +++ b/frontend/src/lib/importWizard/retargetDeployed.test.ts @@ -0,0 +1,393 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * The safety property: the stub survives anything the scan cannot account for. Rewriting an + * item onto the chosen resource is safe on its own, so it always happens; deleting the stub + * while an item still reads it is what breaks the imported project, and that item is exactly + * the one nobody looks at afterwards. + */ + +const state = vi.hoisted(() => ({ + apps: [] as any[], + scripts: [] as any[], + triggers: [] as any[], + scheduleListError: undefined as any, + failingTriggerPath: undefined as string | undefined, + deployedJs: 'COMPILED', + deletedResources: [] as string[], + updatedRawApps: [] as any[], + updatedApps: [] as any[], + updatedFlows: [] as any[], + flows: [] as any[], + updatedTriggers: [] as any[] +})) + +vi.mock('$lib/gen', () => ({ + ScriptService: { + listSearchScript: vi.fn(async () => state.scripts), + getScriptByPath: vi.fn(), + createScript: vi.fn() + }, + FlowService: { + listSearchFlow: vi.fn(async () => state.flows), + getFlowByPath: vi.fn(async ({ path }: any) => state.flows.find((f: any) => f.path === path)), + updateFlow: vi.fn(async (p: any) => state.updatedFlows.push(p)) + }, + AppService: { + listSearchApp: vi.fn(async () => state.apps), + getAppByPath: vi.fn(async ({ path }: any) => state.apps.find((a) => a.path === path)), + getPublicSecretOfLatestVersionOfApp: vi.fn(async () => 'secret'), + updateApp: vi.fn(async (p: any) => state.updatedApps.push(p)), + updateAppRaw: vi.fn(async (p: any) => state.updatedRawApps.push(p)) + }, + ScheduleService: { updateSchedule: vi.fn() }, + ResourceService: { + deleteResource: vi.fn(async ({ path }: any) => state.deletedResources.push(path)) + } +})) + +vi.mock('$lib/components/triggers/workspaceTriggersList', () => ({ + TRIGGER_KINDS: { + schedule: { + badge: 'Schedule', + list: vi.fn(async () => { + if (state.scheduleListError) throw state.scheduleListError + return [] + }) + }, + postgres: { + badge: 'Postgres', + list: vi.fn(async () => state.triggers), + update: vi.fn(async (_w: string, path: string, body: any) => { + if (path === state.failingTriggerPath) throw new Error('the update was rejected') + state.updatedTriggers.push({ path, body }) + }) + } + }, + WORKSPACE_TRIGGER_KINDS: ['schedule', 'postgres'], + createWorkspaceTriggerDisabled: vi.fn(), + triggerHandlerRefs: () => [] +})) + +vi.mock('$lib/components/apps/editor/appPolicy', () => ({ updatePolicy: vi.fn(async () => ({})) })) +vi.mock('$lib/sharedUtils', () => ({ updateRawAppPolicy: vi.fn(async () => ({})) })) + +// The deployed bundle is served by secret, not through the generated client. +vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => ({ + ok: true, + status: 200, + text: async () => (url.endsWith('.js') ? state.deployedJs : 'STYLES') + })) +) + +import { applyRetarget, seesWholeWorkspace } from './retargetDeployed' + +const FROM = 'f/proj/smtp' +const TO = 'f/shared/company_smtp' + +/** A deployed raw app: sources plus runnables, with the bundle stored out of the value. */ +const rawApp = { + path: 'f/proj/dash', + raw_app: true, + value: { files: { '/App.tsx': 'v1' }, runnables: { send: { fields: { smtp: `$res:${FROM}` } } } } +} + +async function run() { + return applyRetarget({ + workspace: 'w', + folder: 'proj', + from: FROM, + to: TO, + seesWholeWorkspace: true + }) +} + +describe('applyRetarget', () => { + beforeEach(() => { + state.apps = [rawApp] + state.scripts = [] + state.triggers = [] + state.deployedJs = 'COMPILED' + state.scheduleListError = undefined + state.failingTriggerPath = undefined + state.deletedResources = [] + state.updatedRawApps = [] + state.updatedApps = [] + state.flows = [] + state.updatedFlows = [] + state.updatedTriggers = [] + }) + + // `updateAppRaw` refuses without a bundle, and the browser cannot rebuild one. The bundle + // that goes back is the deployed one, read back by secret. + it('sends the deployed bundle back with the rewritten value, then drops the stub', async () => { + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.gaps).toEqual([]) + expect(outcome.stubDeleted).toBe(true) + expect(state.deletedResources).toEqual([FROM]) + const sent = state.updatedRawApps[0] + expect(sent.formData.js).toBe('COMPILED') + expect(sent.formData.css).toBe('STYLES') + // Rewritten, and the bundle entries stay out of the value the way the import leaves them. + expect(JSON.stringify(sent.formData.app.value)).toContain(`$res:${TO}`) + expect(Object.keys(sent.formData.app.value.files)).toEqual(['/App.tsx']) + }) + + // The bundle is compiled from the sources, so a `$res:` a source spells out is baked into + // it. `retargetProjectExport` rewrites that copy on import, while /bundle.js is still one + // of `files` — sending the deployed bundle back untouched would undo exactly that. + it("rewrites the deployed bundle's own tokens before sending it back", async () => { + state.deployedJs = `const cfg = "$res:${FROM}"; export default cfg` + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(true) + const sent = state.updatedRawApps[0] + expect(sent.formData.js).toContain(`$res:${TO}`) + expect(sent.formData.js).not.toContain(`$res:${FROM}`) + }) + + // A path the bundle names any other way is one nothing here can move, so the app is left + // alone and the stub it still reads has to survive. + it('keeps the stub when the bundle names the resource outside a $res: token', async () => { + state.deployedJs = `const cfg = await getResource("${FROM}")` + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(state.updatedRawApps).toEqual([]) + }) + + // The path written inside a source file is a reference too, and no rewriter reaches it. + // + it('keeps the stub when a source file names the resource in code', async () => { + state.apps = [ + { + path: 'f/proj/dash', + raw_app: true, + value: { + files: { '/App.tsx': `const c = await getResource("${FROM}")` }, + runnables: {} + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(state.updatedRawApps).toEqual([]) + }) + + // A trigger holds its resource as a bare path in its own column, not as a `$res:` token. + it('finds a trigger that holds the resource as a bare path', async () => { + state.apps = [] + state.triggers = [ + { + path: 'f/proj/ingest', + script_path: 'f/proj/run', + postgres_resource_path: FROM, + permissioned_as: 'u/service_account', + enabled: true + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + // The trigger keeps its own path even though it was the string being remapped. + expect(state.updatedTriggers[0].path).toBe('f/proj/ingest') + // Pointing a trigger at a credential must not also start it: `enabled` is left out so + // the backend keeps whatever the trigger is set to. + expect(state.updatedTriggers[0].body).not.toHaveProperty('enabled') + // Nor run it as whoever picked the credential: a trigger states its identity as + // `permissioned_as`, which the backend keeps only when told to preserve it. + expect(state.updatedTriggers[0].body.permissioned_as).toBe('u/service_account') + expect(state.updatedTriggers[0].body.preserve_permissioned_as).toBe(true) + expect(state.deletedResources).toEqual([FROM]) + }) + + // The stub is what a reference this run did not move still resolves through, so a write + // that fails partway must not take it: the moved items and the rest both keep working. + it('keeps the stub when a write fails, and reports what had already moved', async () => { + state.apps = [] + state.triggers = [ + { path: 'f/proj/first', postgres_resource_path: FROM }, + { path: 'f/proj/second', postgres_resource_path: FROM } + ] + state.failingTriggerPath = 'f/proj/second' + const outcome = await run() + expect(outcome.error).toContain('the update was rejected') + expect(outcome.rewritten.map((r) => r.path)).toEqual(['f/proj/first']) + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + }) + + // Most trigger kinds are cargo features an instance may not compile in, and their routes + // then 404. Reading that as a failed listing keeps the stub on every stock build. + it('drops the stub when a trigger kind is not compiled in, keeps it when one truly fails', async () => { + state.scheduleListError = { status: 404 } + expect((await run()).gaps).toEqual([]) + expect(state.deletedResources).toEqual([FROM]) + + state.deletedResources = [] + state.scheduleListError = { status: 500 } + expect((await run()).gaps.map((g) => g.path)).toContain('Schedule triggers') + expect(state.deletedResources).toEqual([]) + }) + + // The referrer scan matches a bare path anywhere in an app's value, while the rewriter only + // relocates `$res:` tokens and runnable paths. Rewriting such an app would claim a move that + // did not happen, and the stub it still reads would go. + it('leaves an app holding the resource path as a bare string alone, and keeps the stub', async () => { + state.apps = [ + { + path: 'f/proj/page', + value: { grid: [{ data: { input: { type: 'static', value: FROM } } }] } + } + ] + const outcome = await run() + expect(outcome.rewritten).toEqual([]) + expect(outcome.gaps.map((g) => g.path)).toEqual(['f/proj/page']) + expect(state.deletedResources).toEqual([]) + }) + + // Both spellings at once. The token is moved so the item stops depending on the stub for + // what it could, and the mention it spells out still keeps the stub alive. + it('rewrites the token of an app that also names the resource in code, and keeps the stub', async () => { + state.apps = [ + { + path: 'f/proj/dash', + raw_app: true, + value: { + files: { '/App.tsx': `// the credential lives at ${FROM}` }, + runnables: { send: { fields: { smtp: `$res:${FROM}` } } } + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.rewritten.map((r) => r.path)).toEqual(['f/proj/dash']) + expect(outcome.gaps.map((g) => g.path)).toEqual(['f/proj/dash']) + expect(outcome.stubDeleted).toBe(false) + expect(JSON.stringify(state.updatedRawApps[0].formData.app.value)).toContain(`$res:${TO}`) + }) + + // Scripts, flows and resources share a path namespace, and the map holds a resource path. + // The import's rewriters remap a runnable's own `path` on an exact match, which here would + // repoint the step at the credential. + it("moves an app's tokens without repointing a runnable that shares the path", async () => { + state.apps = [ + { + path: 'f/proj/page', + value: { + grid: [{ data: { type: 'runnableByPath', runType: 'script', path: FROM } }], + inline: `$res:${FROM}` + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + const sent = state.updatedApps[0] + expect(sent.requestBody.value.inline).toBe(`$res:${TO}`) + expect(sent.requestBody.value.grid[0].data.path).toBe(FROM) + }) + + // A token moves wherever it sits in the value, and a step's own path is left alone. + it('moves a flow token outside the fields the import rewriter reached', async () => { + state.flows = [ + { + path: 'f/proj/pipeline', + value: { + modules: [ + { id: 'a', summary: `reads $res:${FROM}`, value: { type: 'script', path: FROM } } + ] + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + const sent = state.updatedFlows[0] + expect(sent.requestBody.value.modules[0].summary).toBe(`reads $res:${TO}`) + expect(sent.requestBody.value.modules[0].value.path).toBe(FROM) + }) + + // The listings run as the caller and row-level security filters them inside the query, so + // an item a member cannot read is invisible rather than counted. The stub has to outlive + // a scan that cannot see the whole workspace. + it('keeps the stub when the caller is not shown the whole workspace', async () => { + const outcome = await applyRetarget({ + workspace: 'w', + folder: 'proj', + from: FROM, + to: TO, + seesWholeWorkspace: false + }) + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(outcome.gaps.map((g) => g.path)).toContain('This workspace') + }) + + // A handler field names a runnable, and the map holds a resource path. A script sharing + // that path is not the reference being moved. + it("moves a trigger's resource field without repointing its error handler", async () => { + state.apps = [] + state.triggers = [ + { + path: 'f/proj/ingest', + script_path: 'f/proj/run', + postgres_resource_path: FROM, + on_failure: `script/${FROM}`, + error_handler_path: FROM, + permissioned_as: 'u/service_account' + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + expect(state.updatedTriggers[0].body.on_failure).toBe(`script/${FROM}`) + // The bare spelling too: a handler path is a path and never a `$res:` token. + expect(state.updatedTriggers[0].body.error_handler_path).toBe(FROM) + }) + + // `listSearchApp` caps at 1000 rows server-side, unordered and unpaginated, so a full page + // may not hold the project's own app — and the stub would go anyway. + it('keeps the stub when the app listing comes back at its server-side cap', async () => { + state.apps = Array.from({ length: 1000 }, (_, i) => ({ path: `f/other/a${i}`, value: {} })) + const outcome = await run() + expect(outcome.gaps.map((g) => g.path)).toContain('Apps') + expect(state.deletedResources).toEqual([]) + }) + + // The listings are workspace-wide. An item outside the project's folder is the user's own, + // so it is not rewritten — and it is exactly why the stub it reads has to stay. + it('rewrites what it owns and keeps the stub for a reference outside the project', async () => { + state.apps = [] + state.scripts = [{ path: 'u/alice/report', content: `$res:${FROM}` }] + state.triggers = [ + { path: 'f/proj/ingest', script_path: 'f/proj/run', postgres_resource_path: FROM } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + expect(outcome.gaps.map((g) => g.path)).toEqual(['u/alice/report']) + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + }) +}) + +describe('seesWholeWorkspace', () => { + // The user record is per-workspace and survives a workspace change, so reading `is_admin` + // without checking which workspace it describes answers for the wrong one — and a wrong + // yes here is what lets an RLS-filtered scan clear the stub for deletion. + it.each([ + ['admin of this workspace', { workspace_id: 'w', is_admin: true }, false, true], + ['admin of another workspace', { workspace_id: 'other', is_admin: true }, false, false], + ['member of this workspace', { workspace_id: 'w', is_admin: false }, false, false], + ['superadmin, record stale', { workspace_id: 'other', is_admin: false }, true, true], + ['no user record', undefined, false, false] + ])('%s', (_label, user, isSuperadmin, expected) => { + expect(seesWholeWorkspace(user as any, isSuperadmin, 'w')).toBe(expected) + }) +}) diff --git a/frontend/src/lib/importWizard/retargetDeployed.ts b/frontend/src/lib/importWizard/retargetDeployed.ts new file mode 100644 index 0000000000..b33a69a667 --- /dev/null +++ b/frontend/src/lib/importWizard/retargetDeployed.ts @@ -0,0 +1,650 @@ +/** + * Point an already-imported project at a resource the workspace already has. + * + * The import writes `$res:f//` into every item that uses the project's + * resource — except a trigger, which holds the bare path in its own `*_resource_path` + * field. This rewrites those references to an existing resource, so the project reads the + * workspace's own credential — the same end state the import would have produced, reached + * after the fact. + * + * Two rules make that safe to run over deployed items: + * + * - Only items inside the project's folder are rewritten. The import wrote nothing outside + * it, so a reference from elsewhere is the user's own and not ours to move. + * - The stub is deleted only when the scan can prove it saw every reference to it. + * Listings come back capped, a trigger kind can fail to list, a reference can sit where no + * rewriter reaches, and the listings themselves are row-level-security filtered so a + * caller who is not a workspace admin is not shown every item — each of those is a gap, + * and any gap keeps the stub. + * + * Rewriting is separable from deleting, and only the delete is destructive. An item moved + * onto the chosen resource resolves whether or not the stub survives; an item the scan never + * saw resolves only while the stub is there. So an incomplete scan downgrades the run to + * "rewritten, stub kept" rather than refusing it — the outcome names the gaps so the caller + * can say the placeholder is still around. + */ + +import { AppService, FlowService, ResourceService, ScheduleService, ScriptService } from '$lib/gen' +import { + rewriteContent, + rewriteTriggerConfig, + referencesResourcePath, + holdsResourceToken, + textHoldsBarePath +} from '$lib/components/workspaceSettings/projectBundle' +import { + TRIGGER_KINDS, + WORKSPACE_TRIGGER_KINDS, + type WorkspaceTriggerKind +} from '$lib/components/triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { apiErrorMessage as errorMessage } from '$lib/utils' + +export type ReferrerKind = 'script' | 'flow' | 'app' | 'trigger' + +export interface Referrer { + kind: ReferrerKind + path: string + /** Present for triggers: which kind's table it lives in. */ + triggerKind?: WorkspaceTriggerKind + /** + * The plan already recorded this item as naming the path somewhere no rewrite reaches, so + * the stub survives whatever the write does. Its tokens are still worth moving, and the + * write's own staleness check has nothing left to add. + */ + gapped?: true + /** + * Present for triggers: the row the scan read, which is also what the write sends back. + * Re-reading it costs another listing of the whole kind per trigger — and for schedules a + * listing plus a detail fetch per row, since `list` resolves each one. + */ + row?: Record +} + +/** Something the scan could not account for. `path` names an item, or a listing standing in + * for every item it failed to return. */ +export interface Gap { + path: string + reason: string +} + +export interface RetargetPlan { + /** Items whose reference this run will move. */ + referrers: Referrer[] + /** + * Why the scan cannot claim it saw every reference to the stub. Empty is the only state + * in which deleting the stub is provably safe. + */ + gaps: Gap[] +} + +export interface RetargetOutcome { + /** Items now reading the chosen resource. */ + rewritten: Referrer[] + /** Why the stub was kept, when it was. */ + gaps: Gap[] + stubDeleted: boolean + /** Set when a write failed. The run stopped there and the stub stays, so what was + * rewritten before it and what was not both resolve. */ + error?: string +} + +/** + * Whether this caller's listings are the whole workspace, and so whether a clean scan proves + * anything. Row-level security filters the listings inside the query for everyone else. + * + * `UserExt` is per-workspace and outlives a workspace change, so the role is only this + * workspace's role when the record says it is. On the setup step's reload path nothing + * re-fetches it, and it still describes the workspace the user came from — reading + * `is_admin` alone there answers for the wrong workspace. An instance superadmin bypasses + * the policies everywhere, which is why it is asked separately. + */ +export function seesWholeWorkspace( + user: { workspace_id?: string; is_admin?: boolean } | undefined, + isSuperadmin: boolean, + workspace: string +): boolean { + if (isSuperadmin) return true + return !!user?.is_admin && user.workspace_id === workspace +} + +/** Everything under the project's folder, which is all this rewrites. */ +function inFolder(path: unknown, folder: string): boolean { + return typeof path === 'string' && path.startsWith(`f/${folder}/`) +} + +/** + * Whether the item names the resource path anywhere a rewriter would not reach it. + * + * A script, flow or app spells a reference the rewriters move as a `$res:` token and nothing + * else. The path appearing any other way is either a reference beyond them — a static string + * value in a component, an argument inline code assembles itself — or not the resource at + * all, such as a step running a script that happens to share the path. Neither is rewritable: + * moving the first is beyond the rewriters, and moving the second would repoint a runnable at + * a credential. Both are reasons the stub has to outlive the run. + * + * The whole serialized item is searched, because inline code is a string inside it and a path + * written in code is no less a reference for being surrounded by other characters. Triggers + * are the exception and hold the bare path by design, so they never come here. + */ +function namesPathUnreachably(value: unknown, path: string): boolean { + return textHoldsBarePath(JSON.stringify(value ?? null), path) +} + +/** + * What each `listSearch*` endpoint caps its answer at, server-side. The queries carry no + * `ORDER BY` and the routes take no pagination, so a full page is an arbitrary subset with no + * page two to ask for. + * + * A full page is a sound truncation test only for an unscoped caller, which a wizard session + * is. The server applies its scope-path predicate to the rows the `LIMIT` already returned, + * so a scoped token can be handed a short page cut from a truncated query — reuse this scan + * under one and the cap goes undetected. + */ +const SEARCH_LIMITS = { script: 10000, flow: 1000, app: 1000 } + +/** + * What a trigger listing caps at. The kinds' list routes take pagination this table does not + * pass, so each answers with the server's `DEFAULT_PER_PAGE`. A full page is read the same + * way as a full `listSearch*` page: as a listing that cannot account for the rest. + */ +const TRIGGER_LIST_LIMIT = 1000 + +/** + * Where a trigger names a runnable, taken from what `triggerHandlerRefs` reads. + * + * `rewriteTriggerConfig` remaps a runnable reference on an exact path match, which is right + * for the folder-wide map the import hands it and wrong for a map holding one resource path: + * a script sharing that path is not the reference being moved, and remapping it makes the + * trigger run a resource. So these are put back from the row after the rewrite. + * + * Split by spelling, because only one of the two can also hold a `$res:` token. A prefixed + * field is restored only when it holds the runnable spelling, so a token in it still moves; + * a bare field is a path and nothing else, so it is always restored. + */ +const PREFIXED_RUNNABLE_FIELDS = ['on_failure', 'on_recovery', 'on_success', 'url'] +const PREFIXED_RUNNABLE_RE = /^(?:\$(?:script|flow):|(?:script|flow)\/)/ +const BARE_RUNNABLE_FIELDS = ['dynamic_skip', 'error_handler_path', 'script_path'] + +/** The rewritten config with every runnable reference put back as the row holds it. */ +function restoreRunnableRefs(rewritten: any, row: any): any { + const out = { ...rewritten } + for (const k of PREFIXED_RUNNABLE_FIELDS) { + if (typeof row?.[k] === 'string' && PREFIXED_RUNNABLE_RE.test(row[k])) out[k] = row[k] + } + for (const k of BARE_RUNNABLE_FIELDS) { + if (typeof row?.[k] === 'string') out[k] = row[k] + } + // A websocket's initial messages can each carry a runnable result, whose `path` is bare. + // Only that field is put back; the rest of the message is rewritten like any other value. + if (Array.isArray(out.initial_messages)) { + out.initial_messages = out.initial_messages.map((m: any, i: number) => { + const was = row?.initial_messages?.[i]?.runnable_result?.path + return m?.runnable_result && typeof was === 'string' + ? { ...m, runnable_result: { ...m.runnable_result, path: was } } + : m + }) + } + return out +} + +/** A reference this run will not move, recorded so the stub outlives it. */ +const OUTSIDE_PROJECT = 'reads this resource from outside the project' +const UNSEEN_BY_CALLER = 'holds items this account is not shown' +const UNREACHABLE_REFERENCE = 'names the resource path outside a $res: reference' + +/** + * Which deployed items reference the stub, which of them this run can move, and what it + * could not account for. + * + * The `listSearch*` endpoints return each item's content in one call per kind, so this is + * three calls plus one per trigger kind rather than one per item. + */ +export async function planRetarget( + workspace: string, + folder: string, + from: string, + opts: { seesWholeWorkspace: boolean } +): Promise { + const referrers: Referrer[] = [] + const gaps: Gap[] = [] + + // The listings run as the caller. Row-level security filters the rows out inside the + // query, so an item this account cannot read is not merely absent from the answer — it + // does not count towards the full-page test either, and nothing downstream can notice it. + // A colleague's private script reading this stub is exactly that shape. + if (!opts.seesWholeWorkspace) gaps.push({ path: 'This workspace', reason: UNSEEN_BY_CALLER }) + + const [scripts, flows, apps] = await Promise.all([ + ScriptService.listSearchScript({ workspace }), + FlowService.listSearchFlow({ workspace }), + AppService.listSearchApp({ workspace }) + ]) + + for (const [label, rows, limit] of [ + ['Scripts', scripts, SEARCH_LIMITS.script], + ['Flows', flows, SEARCH_LIMITS.flow], + ['Apps', apps, SEARCH_LIMITS.app] + ] as const) { + if ((rows?.length ?? 0) >= limit) gaps.push({ path: label, reason: 'could not all be listed' }) + } + + // The listings are workspace-wide, so an item outside the folder is seen for free. It is + // the user's own and stays on the stub, which is the whole reason the stub stays too. + for (const s of scripts ?? []) { + const content = String(s.content ?? '') + const reads = referencesResourcePath(content, from) + // `rewriteContent` moves the `$res:` tokens and nothing else, so a path the code spells + // out is a reference this run leaves behind. + const bare = textHoldsBarePath(content, from) + if (!reads && !bare) continue + if (!inFolder(s.path, folder)) { + gaps.push({ path: s.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: s.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + referrers.push({ kind: 'script', path: s.path!, ...(bare ? { gapped: true as const } : {}) }) + } + for (const f of flows ?? []) { + const value: any = f.value ?? {} + // A token is the only spelling a rewriter moves. `referencesResourcePath` would also + // count a whole string equal to the path, which is the unreachable case `bare` covers. + const reads = holdsResourceToken(value, from) + const bare = namesPathUnreachably(value, from) + if (!reads && !bare) continue + if (!inFolder(f.path, folder)) { + gaps.push({ path: f.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: f.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + const gapped = bare ? ({ gapped: true } as const) : undefined + referrers.push({ kind: 'flow', path: f.path!, ...gapped }) + } + for (const a of apps ?? []) { + const value: any = a.value ?? {} + // A token is the only spelling a rewriter moves. `referencesResourcePath` would also + // count a whole string equal to the path, which is the unreachable case `bare` covers. + const reads = holdsResourceToken(value, from) + const bare = namesPathUnreachably(value, from) + if (!reads && !bare) continue + if (!inFolder(a.path, folder)) { + gaps.push({ path: a.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: a.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + const gapped = bare ? ({ gapped: true } as const) : undefined + referrers.push({ kind: 'app', path: a.path!, ...gapped }) + } + + for (const kind of WORKSPACE_TRIGGER_KINDS) { + const def = TRIGGER_KINDS[kind] + // No `eeOnly` skip. That reads a client-side store, which is empty on an EE instance + // whose licence is unset or whose fetch failed — while the rows are still in the + // database and the routes still answer. A kind skipped that way leaves no gap, so the + // stub would go while an EE trigger still points at it. On CE the routes are not + // registered and the 404 below says so, from the server. + let rows: Array> = [] + // A 404 is not an incomplete listing — the instance has that trigger feature compiled + // out, so there is no trigger of the kind to have missed. Anything else means triggers + // of this kind may reference the stub without this ever seeing them. + let incomplete = false + try { + rows = await def.list(workspace, () => (incomplete = true)) + } catch (e: any) { + if (e?.status === 404) continue + incomplete = true + } + if (incomplete) { + gaps.push({ path: `${def.badge} triggers`, reason: 'could not be listed' }) + continue + } + if (rows.length >= TRIGGER_LIST_LIMIT) { + gaps.push({ path: `${def.badge} triggers`, reason: 'could not all be listed' }) + continue + } + for (const t of rows) { + if (!referencesResourcePath(t, from)) continue + if (!inFolder(t.path, folder)) { + gaps.push({ path: String(t.path), reason: OUTSIDE_PROJECT }) + continue + } + // `schedule` has no `update` in the table because its service takes a different body + // shape; `rewriteTrigger` handles it directly, the way the import's create does. + if (kind !== 'schedule' && !def.update) { + gaps.push({ + path: String(t.path), + reason: `${def.badge} triggers cannot be updated from here` + }) + continue + } + referrers.push({ kind: 'trigger', path: String(t.path), triggerKind: kind, row: t }) + } + } + + return { referrers, gaps } +} + +/** + * Rewrite every referrer the plan found, then delete the stub if the plan came back clean. + * + * A gap never stops the rewriting — moving an item onto the chosen resource is safe on its + * own. It stops only the delete, which is the one step that can strand a reference nobody + * looked at. + */ +export async function applyRetarget(args: { + workspace: string + folder: string + from: string + to: string + /** Whether the listings the scan reads are the whole workspace. See `planRetarget`. */ + seesWholeWorkspace: boolean +}): Promise { + const { workspace, folder, from, to, seesWholeWorkspace } = args + const map = new Map([[from, to]]) + const rewritten: Referrer[] = [] + + const plan = await planRetarget(workspace, folder, from, { seesWholeWorkspace }) + const gaps = [...plan.gaps] + + for (const r of plan.referrers) { + let moved: true | string + try { + if (r.kind === 'script') moved = await rewriteScript(workspace, r, map) + else if (r.kind === 'flow') moved = await rewriteFlow(workspace, r, map) + else if (r.kind === 'app') moved = await rewriteApp(workspace, r, map) + else moved = await rewriteTrigger(workspace, r, map) + } catch (e: any) { + return { rewritten, gaps, stubDeleted: false, error: errorMessage(e) } + } + if (moved === true) rewritten.push(r) + else gaps.push({ path: r.path, reason: moved }) + } + + if (gaps.length > 0) return { rewritten, gaps, stubDeleted: false } + + try { + await ResourceService.deleteResource({ workspace, path: from }) + } catch (e: any) { + return { rewritten, gaps, stubDeleted: false, error: errorMessage(e) } + } + return { rewritten, gaps, stubDeleted: true } +} + +/** + * The rewrite this module needs: `$res:` tokens and nothing else. + * + * `rewriteFlowValue` and `rewriteAppValue` also remap a runnable's own `path` on an exact + * match. That is right for the folder-wide map the import hands them, where every path is + * moving. Here the map holds one entry, a resource path — and scripts, flows and resources + * share a namespace, so a project shipping both a script and a resource named `smtp` would + * have the step calling `f//smtp` repointed at the credential. Rewriting the + * serialized value moves the tokens and leaves every path alone. + */ +function rewriteTokens(value: T, map: Map): T { + return JSON.parse(rewriteContent(JSON.stringify(value ?? null), map)) +} + +/** + * Whether the item as just read names a path where no rewrite reaches it. + * + * The plan classifies items from the `listSearch*` rows; every rewriter then re-reads its + * item by path. A value that has gained an unreachable mention in between is one the plan + * cleared and the write must not: rewriting its tokens is fine, but the stub it still names + * has to survive. Checking the read the write is about to send is the only place that can be + * seen. + */ +function readsPathUnreachably(value: unknown, map: Map): boolean { + for (const from of map.keys()) if (namesPathUnreachably(value, from)) return true + return false +} + +/** + * Every write here is an in-place edit of a deployed item, not a redeployment by whoever + * opened the wizard, so each one has to say so. + * + * `preserve_on_behalf_of` is the flag that says it. Without it the backend replaces the + * item's stored run identity with the caller's — `resolve_on_behalf_of` for scripts and + * flows, the `should_preserve` branch of `update_app` for apps — and an imported item that + * ran as a service account would silently start running as the person who picked a + * credential. The backend still gates it on `wm_deployers` membership, so a caller who + * cannot preserve gets what they would have got anyway. + * + * For apps the policy is the other half: it carries the run identity, the execution mode and + * the sandbox rules, so it is read from the deployed app and handed back to the recompute + * rather than rebuilt from nothing. + */ +const PRESERVE_DEPLOYED_IDENTITY = { preserve_on_behalf_of: true } + +/** + * A new script version, the way the editor saves one. Spread rather than field-by-field: + * `Script` and `NewScript` share their names, and listing them here would silently drop + * whichever field someone adds next. + */ +async function rewriteScript( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const s: any = await ScriptService.getScriptByPath({ workspace, path }) + if (!r.gapped) + for (const stub of map.keys()) + if (textHoldsBarePath(String(s.content ?? ''), stub)) return UNREACHABLE_REFERENCE + const content = rewriteContent(s.content ?? '', map) + if (content === s.content) return true + await ScriptService.createScript({ + workspace, + requestBody: { + ...s, + ...PRESERVE_DEPLOYED_IDENTITY, + content, + parent_hash: s.hash, + deployment_message: undefined + } + }) + return true +} + +async function rewriteFlow( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const f: any = await FlowService.getFlowByPath({ workspace, path }) + if (!r.gapped && readsPathUnreachably(f.value ?? {}, map)) return UNREACHABLE_REFERENCE + const value = rewriteTokens(f.value ?? {}, map) + // The tokens the plan saw are gone from the deployed item, so there is nothing to write. + if (JSON.stringify(value) === JSON.stringify(f.value ?? {})) return true + await FlowService.updateFlow({ + workspace, + path, + requestBody: { ...f, ...PRESERVE_DEPLOYED_IDENTITY, path, value } + }) + return true +} + +/** + * The deployed policy is recomputed, not rebuilt. Recomputing is required: `triggerables_v2` + * is keyed by `:rawscript/`, and rewriting an inline + * runnable's content changes that key, so a policy copied verbatim would leave the component + * "forbidden by policy". Handing the deployed policy to the recompute is what keeps the run + * identity, the sandbox rules and everything else it does not touch. + * + * No execution mode is defaulted. The backend keeps the deployed mode when the submitted + * policy states none, and stating one here would put a `viewer` app on the publisher's + * identity. + */ +async function rewriteApp( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const a: any = await AppService.getAppByPath({ workspace, path }) + // The deployed record says which kind this is. `list_search_apps` returns only the path and + // the value, so the scan could only have guessed from the value's shape — and a guess wrong + // in either direction is a deploy the backend refuses for changing an app's kind. + if (a.raw_app) return rewriteRawApp(workspace, r, map, a) + if (!r.gapped && readsPathUnreachably(a.value ?? {}, map)) return UNREACHABLE_REFERENCE + const next = rewriteTokens(a.value ?? {}, map) + // The tokens the plan saw are gone from the deployed item, so there is nothing to write. + if (JSON.stringify(next) === JSON.stringify(a.value ?? {})) return true + const policy = (await updatePolicy(next as App, a.policy)) as any + await AppService.updateApp({ + workspace, + path, + requestBody: { ...PRESERVE_DEPLOYED_IDENTITY, path, value: next, policy } + }) + return true +} + +/** + * One half of a deployed raw app's compiled bundle, read back the way the Hub publish reads + * it: `/apps/get_data/v/{secret}.{ext}` serves it to anyone holding the secret, and the + * secret is minted for the caller against a plain `apps:read:` check. + * + * A missing `.css` is an app that ships no styles. A missing `.js` is a broken deployment, + * and uploading an empty one in its place would break it further. + */ +async function fetchBundlePart( + workspace: string, + secret: string, + ext: 'js' | 'css' +): Promise { + const res = await fetch( + `/api/w/${encodeURIComponent(workspace)}/apps/get_data/v/${secret}.${ext}`, + { credentials: 'include' } + ) + if (res.ok) return await res.text() + if (ext === 'css' && res.status === 404) return '' + throw new Error(`the compiled bundle could not be read (${res.status})`) +} + +/** + * A raw app: its deployed value carries the sources and the runnables, and `updateAppRaw` + * refuses without a bundle. The bundle is the deployed one, read back, rewritten and sent + * again — rebuilding it is not possible from the browser and is not needed, but re-uploading + * it untouched is not an option either. + * + * The bundle is compiled from the sources, so a `$res:` a source file spells out is baked + * into it. The import rewrites that copy — `retargetProjectExport` runs while `/bundle.js` + * is still one of `files`, and only `installProject` splits it out afterwards. Sending the + * deployed bundle back unrewritten would undo on reuse what the import got right, and the + * app would keep reading a resource this run is about to delete. + */ +async function rewriteRawApp( + workspace: string, + r: Referrer, + map: Map, + a: any +): Promise { + const path = r.path + const value: any = a.value ?? {} + // One walk over the whole value: `$res:` tokens live in the runnables and can appear in + // the sources too, and both are plain text inside this JSON. + if (!r.gapped && readsPathUnreachably(value, map)) return UNREACHABLE_REFERENCE + const next = rewriteTokens(value, map) + const runnables = next.runnables ?? {} + const policy = (await updateRawAppPolicy(runnables, a.policy)) as any + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ workspace, path }) + const [deployedJs, deployedCss] = await Promise.all([ + fetchBundlePart(workspace, secret, 'js'), + fetchBundlePart(workspace, secret, 'css') + ]) + const js = rewriteContent(deployedJs, map) + const css = rewriteContent(deployedCss, map) + // `rewriteContent` moves the `$res:` tokens. A path the bundle spells out any other way + // is one nothing here can move, and uploading it would leave the app reading a resource + // about to be deleted. + for (const stub of map.keys()) if (textHoldsBarePath(js, stub)) return UNREACHABLE_REFERENCE + // The tokens the plan saw are gone from both the value and the bundle, so there is nothing + // to write, and uploading would cut a version differing from the last in nothing. + if (js === deployedJs && css === deployedCss && JSON.stringify(next) === JSON.stringify(value)) + return true + const files = { ...(next.files ?? {}) } + delete files['/bundle.js'] + delete files['/bundle.css'] + await AppService.updateAppRaw({ + workspace, + path, + formData: { + app: { + ...PRESERVE_DEPLOYED_IDENTITY, + path, + summary: a.summary ?? '', + value: { + files, + runnables, + ...(next.data !== undefined ? { data: next.data } : {}), + ...(next.datatables !== undefined ? { datatables: next.datatables } : {}) + }, + policy + }, + js, + css + } + }) + return true +} + +/** + * The trigger's own row, rewritten and written back. `enabled` is deliberately not sent: + * imported triggers are created disabled and re-enabling one is the user's decision, not a + * side effect of pointing it at a credential. + * + * `path` is put back from the row afterwards, and so is every runnable reference — see + * `restoreRunnableRefs`. The rewrite remaps any string equal to the stub's path, so a trigger + * sitting at the path the resource used to hold would otherwise be renamed along with the + * reference. + * + * A trigger states its run identity as `permissioned_as`, not the `on_behalf_of` the other + * kinds use, and `resolve_permissioned_as` keeps the row's value only when + * `preserve_permissioned_as` says so. Without the pair a trigger created under a folder's + * `default_permissioned_as` would start running as whoever picked the credential. + */ +async function rewriteTrigger( + workspace: string, + r: Referrer, + map: Map +): Promise { + const def = TRIGGER_KINDS[r.triggerKind!] + const row: any = r.row ?? {} + const { enabled: _enabled, ...rest } = { + ...restoreRunnableRefs(rewriteTriggerConfig(row, map), row), + path: r.path, + ...(typeof row.permissioned_as === 'string' + ? { permissioned_as: row.permissioned_as, preserve_permissioned_as: true } + : {}) + } + // No unreachable-mention check: a trigger holds the bare path by design, in its own + // resource field, and `rewriteTriggerConfig` remaps every bare match at every depth. What + // can still read `from` afterwards is a restored identity field, not a reference. + if (r.triggerKind === 'schedule') { + // `EditSchedule` needs these three; everything else on the row carries over by name. + await ScheduleService.updateSchedule({ + workspace, + path: r.path, + requestBody: { + ...rest, + schedule: rest.schedule ?? '0 0 * * * *', + timezone: rest.timezone ?? 'UTC', + args: rest.args ?? {} + } + }) + return true + } + await def.update!(workspace, r.path, rest) + return true +} diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 3e8ae4cba0..ce4adf82be 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -885,7 +885,7 @@ export const mcpEndpointTools: EndpointTool[] = [ { name: "runScriptByPath", description: "run script by path", - instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected.", + instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`.", path: "/w/{workspace}/jobs/run/p/{path}", method: "POST", pathParamsSchema: { @@ -1426,7 +1426,7 @@ export const mcpEndpointTools: EndpointTool[] = [ { name: "runFlowByPath", description: "run flow by path", - instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected.", + instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`.", path: "/w/{workspace}/jobs/run/f/{path}", method: "POST", pathParamsSchema: { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 04e21c41f2..e82d328c76 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -84,6 +84,7 @@ import SessionPicker from '$lib/components/sessions/SessionPicker.svelte' import SessionModeSwitch from '$lib/components/sessions/SessionModeSwitch.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + import { copilotInfo } from '$lib/aiStore' import { parsePreviewItemRoute } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' @@ -251,6 +252,10 @@ // so it follows the gate; opted-out users get the legacy Ask-AI pane instead. // The /sessions page has its own gate for direct navigation. const globalAiEnabled = isGlobalAiEnabled() + // A workspace that hid the assistant (`ai_config.copilot_disabled`) loses both entry + // points: the Workspace ⇄ Sessions switch and the legacy Ask-AI button. + const sessionsSwitchShown = $derived(globalAiEnabled && !$copilotInfo.workspaceDisabled) + const askAiShown = $derived(!globalAiEnabled && !$copilotInfo.workspaceDisabled) if (page.status == 404) { goto('/user/login') @@ -1004,7 +1009,7 @@
    - {#if !embedded && globalAiEnabled} + {#if !embedded && sessionsSwitchShown}
    -
    +
    {/if} @@ -1052,7 +1057,7 @@ class="!text-xs" shortcut={`${getModifierKey()}k`} /> - {#if !globalAiEnabled} + {#if askAiShown}
    - {#if !embedded && globalAiEnabled} + {#if !embedded && sessionsSwitchShown}
    @@ -1150,7 +1155,7 @@ -
    +
    {/if} @@ -1189,7 +1194,7 @@ class="!text-xs" shortcut={`${getModifierKey()}k`} /> - {#if !globalAiEnabled} + {#if askAiShown} - aiChatManager.toggleOpen()} - {isCollapsed} - icon={WandSparkles} - iconProps={{ - forceDarkMode: true - }} - label="Ask AI" - class="!text-xs" - iconClasses="!text-ai" - shortcut={`${getModifierKey()}L`} - /> + {#if !$copilotInfo.workspaceDisabled} + aiChatManager.toggleOpen()} + {isCollapsed} + icon={WandSparkles} + iconProps={{ + forceDarkMode: true + }} + label="Ask AI" + class="!text-xs" + iconClasses="!text-ai" + shortcut={`${getModifierKey()}L`} + /> + {/if}
    + + + status and on the workspace inside the component, which owns its own vertical spacing + because the hero and the bare connect row want different amounts of it. --> {#if isGlobalAiEnabled()} -
    - -
    + {/if} {#if $workspaceStore == 'admins'} @@ -280,8 +281,6 @@
    {/if} - - (showCreateButtons = v)} /> {#if tab == 'hub'} diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index e35c4a7381..b184e4aa4a 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -4,11 +4,10 @@ import CenteredPage from '$lib/components/CenteredPage.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import FolderEditor from '$lib/components/FolderEditor.svelte' + import FolderEditorDrawer from '$lib/components/FolderEditorDrawer.svelte' import PageHeader from '$lib/components/PageHeader.svelte' import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Button, Drawer, DrawerContent, EmptyState, Skeleton } from '$lib/components/common' - import Popover from '$lib/components/meltComponents/Popover.svelte' import FolderInfo from '$lib/components/FolderInfo.svelte' import FolderUsageInfo from '$lib/components/FolderUsageInfo.svelte' import { sendUserToast } from '$lib/utils' @@ -28,9 +27,8 @@ isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) ) - let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) - let folderDrawer: Drawer | undefined = $state() + let folderEditorDrawer: FolderEditorDrawer | undefined = $state() let hubDrawer: Drawer | undefined = $state() let publishFolderName: string = $state('') @@ -47,23 +45,11 @@ }) } - function handleKeyUp(event: KeyboardEvent, close: () => void) { - const key = event.key - if (key === 'Enter') { - event.preventDefault() - addFolder() - close() - } - } - async function addFolder() { - await FolderService.createFolder({ - workspace: $workspaceStore ?? '', - requestBody: { name: newFolderName } - }) - $userStore?.folders.push(newFolderName) - loadFolders() - editFolderName = newFolderName - folderDrawer?.openDrawer() + function onFolderSaved(name: string, created: boolean) { + if (created) $userStore?.folders.push(name) + // Returned, not fired: the drawer reports a failed reload, and it can only see one + // through the promise this hands back. + return loadFolders() } $effect(() => { @@ -74,8 +60,6 @@ } }) - let editFolderName: string = $state('') - function computeMembers(owners: string[], extra_perms: Record) { const members = new Set(owners) for (const [user, _] of Object.entries(extra_perms)) { @@ -85,48 +69,18 @@ } -{#snippet newFolderPopover( - label: string, - placement: 'bottom' | 'bottom-end', - variant: 'accent' | 'default' -)} - folderEditorDrawer?.initNew()} > - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} - handleKeyUp(e, () => close())} - placeholder="New folder name" - bind:value={newFolderName} - /> - -
    - -
    - {/snippet} -     + {label} + {/snippet} - - - - - + {:else} - {@render newFolderPopover('New folder', 'bottom-end', 'accent')} + {@render newFolderButton('New folder', 'accent')} {/if}
    @@ -181,7 +135,7 @@ description="Folders are how you grant permissions: make a user or group viewer, writer or admin on a folder and that access applies to every script, flow, app, resource and schedule inside it." > {#if !restricted} - {@render newFolderPopover('Add a folder', 'bottom', 'default')} + {@render newFolderButton('Add a folder', 'default')} {/if} {:else} @@ -196,8 +150,8 @@ Schedules Variables Resources - Participants - + Members + Actions @@ -211,13 +165,7 @@ {/each} {:else} {#each folders as { name, extra_perms, owners, canWrite, summary, labels } (name)} - { - editFolderName = name - folderDrawer?.openDrawer() - }} - > + folderEditorDrawer?.initEdit(name)}> {name} {#if summary} @@ -250,17 +198,14 @@ - + { - editFolderName = name - folderDrawer?.openDrawer() - } + action: () => folderEditorDrawer?.initEdit(name) }, { displayName: 'Publish to Hub', diff --git a/frontend/src/routes/(root)/(logged)/groups/+page.svelte b/frontend/src/routes/(root)/(logged)/groups/+page.svelte index 85b4b536fe..ce8c6a8db0 100644 --- a/frontend/src/routes/(root)/(logged)/groups/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/groups/+page.svelte @@ -5,9 +5,8 @@ import CenteredPage from '$lib/components/CenteredPage.svelte' import { Button, Drawer, DrawerContent, Skeleton } from '$lib/components/common' - import Popover from '$lib/components/meltComponents/Popover.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import GroupEditor from '$lib/components/GroupEditor.svelte' + import GroupEditorDrawer from '$lib/components/GroupEditorDrawer.svelte' import InstanceGroupEditor from '$lib/components/InstanceGroupEditor.svelte' import GroupInfo from '$lib/components/GroupInfo.svelte' import PageHeader from '$lib/components/PageHeader.svelte' @@ -20,7 +19,6 @@ import Cell from '$lib/components/table/Cell.svelte' import Row from '$lib/components/table/Row.svelte' import { untrack } from 'svelte' - import TextInput from '$lib/components/text_input/TextInput.svelte' import { Tooltip } from '$lib/components/meltComponents' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' @@ -30,10 +28,9 @@ isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) ) - let newGroupName: string = $state('') let groups: GroupW[] | undefined = $state(undefined) let instanceGroups: InstanceGroupWithWorkspaces[] | undefined = $state(undefined) - let groupDrawer: Drawer | undefined = $state() + let groupEditorDrawer: GroupEditorDrawer | undefined = $state() async function loadGroups(): Promise { groups = (await GroupService.listGroups({ workspace: $workspaceStore! })).map((x) => { @@ -49,24 +46,6 @@ } } - function handleKeyUp(event: KeyboardEvent, close: () => void) { - const key = event.key - if (key === 'Enter') { - event.preventDefault() - addGroup() - close() - } - } - async function addGroup() { - await GroupService.createGroup({ - workspace: $workspaceStore ?? '', - requestBody: { name: newGroupName } - }) - loadGroups() - editGroupName = newGroupName - groupDrawer?.openDrawer() - } - $effect(() => { untrack(() => loadInstanceGroups()) if ($workspaceStore && $userStore) { @@ -74,16 +53,11 @@ } }) - let editGroupName: string = $state('') let instanceGroupDrawer: Drawer | undefined = $state() let editInstanceGroupName: string = $state('') - - - - - + {:else} - - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} -
    - handleKeyUp(e, close) - }} - bind:value={newGroupName} - /> - -
    - {/snippet} -     + {/if}
    @@ -161,7 +112,7 @@ Name Members - + Actions @@ -175,13 +126,7 @@ {/each} {:else} {#each groups as { name, summary, extra_perms, canWrite } (name)} - { - editGroupName = name - groupDrawer?.openDrawer() - }} - > + groupEditorDrawer?.initEdit(name)}>
    @@ -197,7 +142,7 @@ - + { e?.stopPropagation() - editGroupName = name - groupDrawer?.openDrawer() + groupEditorDrawer?.initEdit(name) } }, { diff --git a/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte index a1bc2ab970..a6f4632526 100644 --- a/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte @@ -261,9 +261,10 @@ setupUndecided = false return } - // Every resource the project ships arrives as an empty stub, so any project with - // resources has something to fill in. The step itself re-checks and shows only - // what is genuinely outstanding, which is what makes a re-import quiet. + // `resourceCount` is the referenced subset — the resources something in the project + // points at — and each one arrives as an empty stub, so any project that has them has + // something to fill in. The step itself re-checks and shows only what is genuinely + // outstanding, which is what makes a re-import quiet. if (execution.resourceCount > 0) { setupNeeded = true setupUndecided = false diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 6f3fd1a6dc..4932355654 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -1086,8 +1086,8 @@ Path Resource type Description - - + Status + Actions @@ -1254,8 +1254,8 @@ {/if}
    - -
    + +
    {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} Name Description - + Actions @@ -1418,7 +1418,7 @@
    - + {#if !canWrite} diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index fbf045fe19..da8fc31c65 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -21,6 +21,8 @@ type Scope } from '$lib/components/sessions/PreviewRouterPicker.svelte' import { goto } from '$lib/navigation' + import { resource } from 'runed' + import { WorkspaceService } from '$lib/gen' import SessionWrapper from '$lib/components/sessions/SessionWrapper.svelte' import PreviewTabHost from '$lib/components/sessions/PreviewTabHost.svelte' import { useIsDarkMode } from '$lib/components/DarkModeObserver.svelte' @@ -278,6 +280,36 @@ (activeSession ? getEffectiveWorkspaceId(activeSession) : undefined) ?? $workspaceStore ) + // Whether that workspace hid the AI assistant (`ai_config.copilot_disabled`). Read + // directly rather than from `copilotInfo`: that store holds whichever workspace loaded + // last, and the gate below unmounts the session wrapper that would refresh it, so a + // hidden verdict would stick across session and workspace switches. Tagged with its + // workspace and guarded against a superseded response like the protection-rules + // resource: runed keeps the previous `current` while a new source loads, so a switch + // would otherwise be judged on the previous workspace's verdict. A failed read leaves + // the page usable, as an unloaded config does everywhere else. + const workspaceAiHidden = resource( + () => previewWorkspace, + async (workspace, _prev, { signal }) => { + if (!workspace) return { workspace, hidden: false } + let hidden = false + try { + hidden = (await WorkspaceService.getCopilotInfo({ workspace })).copilot_disabled === true + } catch (e) { + console.error(`Failed to read the AI config of workspace ${workspace}:`, e) + } + // The generated client can't take an abort signal, so drop a superseded response here. + if (signal.aborted) throw new DOMException('superseded', 'AbortError') + return { workspace, hidden } + } + ) + // Only a verdict for the workspace currently judged counts; `undefined` means it has not + // landed yet. + const aiHiddenVerdict = $derived.by(() => { + const current = workspaceAiHidden.current + return current && current.workspace === previewWorkspace ? current.hidden : undefined + }) + // Lazy-mount gate: a tab's content only renders once its key lands here (on // first activation) — so restoring a session with N saved tabs boots just // the active one instead of N full Windmill apps at once. Pure "has this @@ -559,9 +591,7 @@ } }) - // Preview cards on create/update tool calls dispatch here. Open - // (or focus, if already shown) the item's preview in the active session's panel — - // the visible chat is always the active session, so `owner` is its panel. Read + // The visible chat is always the active session, so `owner` is its panel. Read // `owner` lazily inside the handler (not in the effect body) so this registers // once, not on every session switch. $effect(() => { @@ -836,284 +866,309 @@
    {:else} -
    - +
    +
    - {#if !fullscreen} - - -
    - {#each warmSessions as s (s.id)} -
    - -
    - {/each} -
    -     - {/if} + +
    + {#each warmSessions as s (s.id)} +
    + +
    + {/each} +
    +     + {/if} - - -
    -
    - {#if !fullscreen} - - - {/if} + + {/if} - -
    - {#if !activeTabIsArtifact} - + {#if !activeTabIsArtifact} + + + + {/if} + -
    + {#if fullscreen} + + {:else} + + {/if} + +
    - - (activeTabPickerOpen = !activeTabPickerOpen)} - onClose={closeTab} - onReorder={reorderTabs} - class="session-preview-tab-strip h-8 border-b border-light bg-surface-secondary/50 {fullscreen - ? 'pl-1.5' - : 'pl-9'} pr-16" - > - {#snippet tabAccessory(_tab, isActive)} - {#if isActive} - - - (e.currentTarget as HTMLElement) - .closest('[role="tab"]') - ?.focus() - }} - > - {#snippet content()} - - {#key activePickerScope?.dir ?? ''} - { - activeTabPickerOpen = false - navigatePreviewTo(t) - }} - /> - {/key} - {/snippet} - - - {/if} - {/snippet} - {#snippet afterTabs()} - - {#snippet trigger()} - - {/snippet} - {#snippet content()} - { - newTabOpen = false - openInNewTab(t) - }} + {#key activePickerScope?.dir ?? ''} + { + activeTabPickerOpen = false + navigatePreviewTo(t) + }} + /> + {/key} + {/snippet} + + - {/snippet} - - {/snippet} - - - -
    - {#each warmSessions as s (s.id)} - {@const rt = getRuntime(s.id)} - {@const tabs = rt?.previewTabs} - {#each tabs?.tabs ?? [] as tab (tab.id)} - - - tabs && onTabLoad(tabs, tab, frame)} - /> - {/each} - {/each} - {#if (owner?.tabs.length ?? 0) === 0} - -
    - -
    - No preview open - Open a page, flow, script or app to preview it alongside the chat. -
    + {/if} + {/snippet} + {#snippet afterTabs()} {#snippet trigger()} - - Open a preview - + {/snippet} {#snippet content()} { - emptyStateNewTabOpen = false + newTabOpen = false openInNewTab(t) }} /> {/snippet} -
    - {/if} + {/snippet} + + + +
    + {#each warmSessions as s (s.id)} + {@const rt = getRuntime(s.id)} + {@const tabs = rt?.previewTabs} + {#each tabs?.tabs ?? [] as tab (tab.id)} + + + tabs && onTabLoad(tabs, tab, frame)} + /> + {/each} + {/each} + {#if (owner?.tabs.length ?? 0) === 0} + +
    + +
    + No preview open + Open a page, flow, script or app to preview it alongside the chat. +
    + + {#snippet trigger()} + + Open a preview + + {/snippet} + {#snippet content()} + { + emptyStateNewTabOpen = false + openInNewTab(t) + }} + /> + {/snippet} + +
    + {/if} +
    -
    - - - {#if previewCollapsed && !fullscreen} - -
    - +
    + +
    + {/if} +
    + {#if aiHiddenVerdict === undefined} +
    + +
    + {:else if aiHiddenVerdict} + +
    +

    AI Sessions are hidden in this workspace

    +

    A workspace admin hid AI sessions in the workspace settings.

    +
    {/if}
    diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 0a23656514..016841f1ae 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -391,8 +391,8 @@ Path Value Description - - + Status + Actions @@ -494,7 +494,7 @@ {#if refresh_error} + so it can't paint over anything that scrolls past it -->
    @@ -546,7 +546,7 @@ {/if}
    - + { let owner = isOwner(path, $userStore, $workspaceStore) diff --git a/frontend/src/routes/kitchen_sink/+page.svelte b/frontend/src/routes/kitchen_sink/+page.svelte index 8681fb78da..a001da222f 100644 --- a/frontend/src/routes/kitchen_sink/+page.svelte +++ b/frontend/src/routes/kitchen_sink/+page.svelte @@ -9,6 +9,7 @@ import type { DisplayMessage } from '$lib/components/copilot/chat/shared' import DraggableTabs, { type TabItem } from '$lib/components/common/tabs/DraggableTabs.svelte' import { Globe } from 'lucide-svelte' + import { workspaceStore } from '$lib/stores' let tab = $state('button') @@ -195,7 +196,7 @@ That's the full round-trip.` CodeDisplay → HighlightCode), constrained to the chat panel width.
    - +
    diff --git a/integration_tests/test/git_sync_test.py b/integration_tests/test/git_sync_test.py index 3cc54f2796..47ac02351e 100644 --- a/integration_tests/test/git_sync_test.py +++ b/integration_tests/test/git_sync_test.py @@ -1,9 +1,12 @@ +import json import os +import re import shutil import tempfile import time import unittest import uuid +from pathlib import Path import git as gitpython @@ -19,6 +22,21 @@ def unique_name(prefix: str = "git-sync-test") -> str: return f"{prefix}-{uuid.uuid4().hex[:8]}" +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def ui_pull_script_path() -> str: + """The hub script the git-sync UI runs for a pull (git → workspace).""" + with open(REPO_ROOT / "frontend/src/lib/hubPaths.json") as f: + return json.load(f)["gitInitRepo"] + + +def backend_pull_script_path() -> str: + """The hub script the backend runs for an auto-pull: `GIT_SYNC_PULL_SCRIPT_PATH`.""" + source = (REPO_ROOT / "backend/windmill-common/src/workspaces.rs").read_text() + return re.search(r'GIT_SYNC_PULL_SCRIPT_PATH: &str = "([^"]+)"', source).group(1) + + class GitSyncTestBase(unittest.TestCase): """Shared fixture + helpers for git sync e2e tests (no tests of its own). @@ -221,6 +239,84 @@ class GitSyncTestBase(unittest.TestCase): ) return matching[0] + def _seed_wmill_yaml( + self, repo_name: str, branch: str = "main", include_schedules: bool = False + ): + """Commit a minimal wmill.yaml: the pull CLI requires one in the repo. + Real setups get it from the init/settings-push flow; pushes alone + don't write it.""" + self._gitea.create_file( + repo_name, + "wmill.yaml", + "defaultTs: bun\n" + "includes:\n" + ' - "**"\n' + "excludes: []\n" + "codebases: []\n" + "skipVariables: true\n" + "skipResources: true\n" + "skipResourceTypes: true\n" + "skipSecrets: true\n" + f"includeSchedules: {'true' if include_schedules else 'false'}\n" + "includeTriggers: false\n", + branch=branch, + ) + + def _create_fork(self, client: WindmillClient) -> tuple: + """Fork `client`'s workspace the way the UI does (branch job first, then + the workspace). Returns (fork_id, fork_branch).""" + fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" + self._fork_workspaces_to_cleanup.append(fork_id) + job_ids = client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") + if job_ids: + client.wait_for_jobs_by_ids(job_ids, timeout=90) + time.sleep(3) + client.create_workspace_fork(fork_id, f"Fork {fork_id}") + return fork_id, f"wm-fork/main/{fork_id[len('wm-fork-'):]}" + + def _run_ui_pull( + self, + client: WindmillClient, + resource_path: str, + include_type: list, + clone_ref: str = None, + dry_run: bool = False, + timeout: int = 180, + ) -> dict: + """Run the pull the git-sync UI runs (git → `client`'s workspace) and + return the job's result. `dry_run` is the UI's preview.""" + payload = { + "workspace_id": client._workspace, + "repo_url_resource_path": resource_path, + "dry_run": dry_run, + "pull": True, + "only_wmill_yaml": False, + "settings_json": json.dumps({ + "include_path": ["**"], + "exclude_path": [], + "extra_include_path": [], + "include_type": include_type, + }), + "use_promotion_overrides": False, + **({"clone_ref": clone_ref} if clone_ref else {}), + } + response = client._client.post( + f"/api/w/{client._workspace}/jobs/run/p/{ui_pull_script_path()}", + params={"skip_preprocessor": "true"}, + json=payload, + ) + self.assertEqual( + response.status_code // 100, 2, f"UI pull failed to start: {response.content.decode()}" + ) + job_id = response.content.decode() + client.wait_for_jobs_by_ids([job_id], timeout=timeout) + job = client._client.get(f"/api/w/{client._workspace}/jobs_u/get/{job_id}").json() + self.assertTrue( + job.get("success"), + f"UI pull job {job_id} failed: {job.get('result')}\n{job.get('logs')}", + ) + return job.get("result") or {} + class TestGitSync(GitSyncTestBase): # ────────────────────────────────────────────────── @@ -802,27 +898,6 @@ class TestGitSyncAutoPull(GitSyncTestBase): # Two poll cycles + job execution, with slack for a loaded CI runner. PULL_TIMEOUT = 240 - def _seed_wmill_yaml(self, repo_name: str, branch: str = "main"): - """Commit a minimal wmill.yaml: the pull CLI requires one in the repo. - Real setups get it from the init/settings-push flow; pushes alone - don't write it.""" - self._gitea.create_file( - repo_name, - "wmill.yaml", - "defaultTs: bun\n" - "includes:\n" - ' - "**"\n' - "excludes: []\n" - "codebases: []\n" - "skipVariables: true\n" - "skipResources: true\n" - "skipResourceTypes: true\n" - "skipSecrets: true\n" - "includeSchedules: false\n" - "includeTriggers: false\n", - branch=branch, - ) - def _configure_auto_pull(self, resource_path: str, sync_forks: bool = False): """Single sync repo with auto-pull enabled in polling mode.""" auto_pull = {"enabled": True, "mode": "polling"} @@ -914,16 +989,7 @@ class TestGitSyncAutoPull(GitSyncTestBase): self._configure_auto_pull(resource_path, sync_forks=True) - # Create the fork (branch first, then workspace), like the UI does. - fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" - self._fork_workspaces_to_cleanup.append(fork_id) - job_ids = self._client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") - if job_ids: - self._client.wait_for_jobs_by_ids(job_ids, timeout=90) - time.sleep(3) - self._client.create_workspace_fork(fork_id, f"Fork {fork_id}") - - fork_branch = f"wm-fork/main/{fork_id[len('wm-fork-'):]}" + fork_id, fork_branch = self._create_fork(self._client) self._gitea.create_file( repo_name, script_file, ts_script("return 'fork only'"), branch=fork_branch, @@ -1012,19 +1078,12 @@ class TestGitSyncAutoPull(GitSyncTestBase): f"attach_dev_workspace failed: {attach.content.decode()}", ) - # Fork the dev workspace (branch first, then workspace) — its parent is - # the dev, so this is a fork OF a dev workspace. - fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" - self._fork_workspaces_to_cleanup.append(fork_id) - job_ids = dev_client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") - if job_ids: - dev_client.wait_for_jobs_by_ids(job_ids, timeout=90) - time.sleep(3) - dev_client.create_workspace_fork(fork_id, f"Fork {fork_id}") + # Fork the dev workspace — its parent is the dev, so this is a fork OF + # a dev workspace. + fork_id, fork_branch = self._create_fork(dev_client) # The fork branch is named after the tracked branch, not the dev label. fork_suffix = fork_id[len("wm-fork-"):] - fork_branch = f"wm-fork/main/{fork_suffix}" branches = self._get_branches(self._clone_repo_all_branches(repo_name)) self.assertTrue( any(fork_branch in b for b in branches), @@ -1195,3 +1254,73 @@ class TestGitSyncAutoPull(GitSyncTestBase): initial_count, "Unknown webhook delivery enqueued a job", ) + + +class TestGitSyncUiPull(GitSyncTestBase): + """The pull the git-sync UI runs (hub init script, git → workspace).""" + + def test_ui_pull_script_is_the_backend_pull_script(self): + """The UI's pull and the backend's auto-pull are the same hub script, + so the two pins must move together.""" + self.assertEqual(ui_pull_script_path(), backend_pull_script_path()) + + def test_fork_pull_does_not_report_parent_owned_schedule_enabled(self): + """In a fork, a schedule the parent also has takes its `enabled` from + the parent, so a fork-branch file that disagrees on that flag can never + be made to agree: a pull that treated it as a change would list the + same row on every run. Pull into the fork, then preview: the schedule + must not be reported, while an ordinary fork-branch edit does land.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + include_type = ["script", "schedule"] + self._configure_single_repo_sync(resource_path, include_type=include_type) + + script_path = self._deploy_seed_script("forkuipull") + schedule_path = f"u/admin/{unique_name('forkuipull_sched')}" + initial_count = self._client.count_deployment_callback_jobs() + self._client.create_schedule(schedule_path, script_path, schedule="0 0 0 1 1 *") + self.addCleanup(self._client.delete_schedule, schedule_path) + self._client.wait_for_sync_jobs(initial_count, min_new=1) + time.sleep(3) + self._seed_wmill_yaml(repo_name, include_schedules=True) + + # Fork after the schedule reached git so the fork branch inherits it + # with the parent's `enabled: true`; the fork's own copy lands disabled. + fork_id, fork_branch = self._create_fork(self._client) + + schedule_file = f"{schedule_path}.schedule.yaml" + fork_dir = self._clone_repo(repo_name, branch=fork_branch) + content = self._read_file_content(fork_dir, schedule_file) + self.assertIn( + "enabled: true", content, f"expected the parent's enabled schedule in git:\n{content}" + ) + self._gitea.create_file( + repo_name, + schedule_file, + content.replace("enabled: true", "enabled: false"), + branch=fork_branch, + ) + # A real change alongside it proves the pull ran against the fork branch. + script_file = self._repo_script_file(repo_name, script_path, branch=fork_branch) + self._gitea.create_file( + repo_name, script_file, ts_script("return 'fork ui pull'"), branch=fork_branch + ) + + fork_client = WindmillClient(workspace=fork_id) + self._run_ui_pull(fork_client, resource_path, include_type, clone_ref=fork_branch) + self.assertIn( + "fork ui pull", + fork_client.get_script_content(script_path), + "the fork-branch script edit was not applied by the pull", + ) + preview = self._run_ui_pull( + fork_client, resource_path, include_type, clone_ref=fork_branch, dry_run=True + ) + self.assertIn("changes", preview, f"preview result has no changes list: {preview}") + changes = preview["changes"] + self.assertIsInstance(changes, list, f"preview changes is not a list: {preview}") + self.assertEqual( + [c for c in changes if c.get("path", "").endswith(schedule_file)], + [], + f"a pull into the fork keeps reporting the parent-owned schedule flag: {changes}", + ) diff --git a/lsp/Pipfile b/lsp/Pipfile index cf1bb1ec17..0bc31bcb41 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.800.1" +wmill = ">=1.803.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 0350536df4..bdf8ecce08 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.800.1 + version: 1.803.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index df980c46a0..089f7ff05a 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.800.1' + ModuleVersion = '1.803.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 00582eecd2..35a2b8d857 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.800.1" +version = "1.803.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index e51c8371dc..acf6622dc9 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -160,11 +160,11 @@ flow related commands - `flow push ` - push a local flow spec. This overrides any remote versions. - `--message ` - Deployment message - `flow run ` - run a flow by path. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files. - `--step ` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -560,11 +560,11 @@ script related commands - `--json` - Output as JSON (for piping to jq) - `script show ` - show a script's content (alias for get) - `script run ` - run a script by path - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script preview ` - preview a local script without deploying it. Supports both regular and codebase scripts. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other than the final output. Useful for scripting. - `--tag ` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script new ` - create a new script diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 918b88cb21..1feeadc162 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -498,6 +498,41 @@ Reference other resources: } \`\`\` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema \`format: resource-\`) is passed as +the **bare string** \`$res:\` — the whole argument value. Same for a variable, with +\`$var:\`. This applies everywhere job arguments are supplied: \`wmill script run/preview\`, +\`wmill flow run/preview\`, the \`runScriptByPath\` / \`runFlowByPath\` API, a schedule's \`args\`, a +trigger's configured static args. + +\`\`\`json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +\`\`\` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with \`$res:\` / \`$var:\`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +\`\`\`json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +\`\`\` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(\`{ "gh_auth": { "token": "$var:g/all/gh_token" } }\`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL @@ -3226,11 +3261,11 @@ flow related commands - \`flow push \` - push a local flow spec. This overrides any remote versions. - \`--message \` - Deployment message - \`flow run \` - run a flow by path. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`--step \` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -3626,11 +3661,11 @@ script related commands - \`--json\` - Output as JSON (for piping to jq) - \`script show \` - show a script's content (alias for get) - \`script run \` - run a script by path - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script preview \` - preview a local script without deploying it. Supports both regular and codebase scripts. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other than the final output. Useful for scripting. - \`--tag \` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script new \` - create a new script diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 1f3c54ea26..fef40d3341 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -165,11 +165,11 @@ flow related commands - `flow push ` - push a local flow spec. This overrides any remote versions. - `--message ` - Deployment message - `flow run ` - run a flow by path. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files. - `--step ` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -565,11 +565,11 @@ script related commands - `--json` - Output as JSON (for piping to jq) - `script show ` - show a script's content (alias for get) - `script run ` - run a script by path - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. - `--tag ` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script preview ` - preview a local script without deploying it. Supports both regular and codebase scripts. - - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - `-s --silent` - Do not output anything other than the final output. Useful for scripting. - `--tag ` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - `script new ` - create a new script diff --git a/system_prompts/auto-generated/skills/resources/SKILL.md b/system_prompts/auto-generated/skills/resources/SKILL.md index 19b8c9f420..396b3b9d8e 100644 --- a/system_prompts/auto-generated/skills/resources/SKILL.md +++ b/system_prompts/auto-generated/skills/resources/SKILL.md @@ -64,6 +64,41 @@ Reference other resources: } ``` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema `format: resource-`) is passed as +the **bare string** `$res:` — the whole argument value. Same for a variable, with +`$var:`. This applies everywhere job arguments are supplied: `wmill script run/preview`, +`wmill flow run/preview`, the `runScriptByPath` / `runFlowByPath` API, a schedule's `args`, a +trigger's configured static args. + +```json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +``` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with `$res:` / `$var:`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +```json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +``` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(`{ "gh_auth": { "token": "$var:g/all/gh_token" } }`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 6a5f36e9b4..925ffbe6ec 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -77,6 +77,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute `wmill flow preview -d ''` directly — pick plausible args from the flow's input schema. +An input typed as a resource (`format: resource-` in the schema) takes the bare string `"$res:"` as its whole value — `-d '{"db": "$res:f/databases/postgres_prod"}'`, not `{"db": {"$res": "..."}}` and not a plain path. Same for a variable, with `"$var:"`. See the `resources` skill. + `wmill flow preview` is safe to run yourself (it does not deploy). `wmill generate-metadata` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's `AGENTS.md` opts into automatic metadata. After running it, check the regenerated `.lock` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only `wmill sync push` deploys; run it only when the user explicitly asks. ### Visual preview diff --git a/system_prompts/base/flow-cli.md b/system_prompts/base/flow-cli.md index d8a2b41b13..fa4ec9cfcc 100644 --- a/system_prompts/base/flow-cli.md +++ b/system_prompts/base/flow-cli.md @@ -72,6 +72,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute `wmill flow preview -d ''` directly — pick plausible args from the flow's input schema. +An input typed as a resource (`format: resource-` in the schema) takes the bare string `"$res:"` as its whole value — `-d '{"db": "$res:f/databases/postgres_prod"}'`, not `{"db": {"$res": "..."}}` and not a plain path. Same for a variable, with `"$var:"`. See the `resources` skill. + `wmill flow preview` is safe to run yourself (it does not deploy). `wmill generate-metadata` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's `AGENTS.md` opts into automatic metadata. After running it, check the regenerated `.lock` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only `wmill sync push` deploys; run it only when the user explicitly asks. ### Visual preview diff --git a/system_prompts/base/resources.md b/system_prompts/base/resources.md index 1c548a2484..5252683a81 100644 --- a/system_prompts/base/resources.md +++ b/system_prompts/base/resources.md @@ -59,6 +59,41 @@ Reference other resources: } ``` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema `format: resource-`) is passed as +the **bare string** `$res:` — the whole argument value. Same for a variable, with +`$var:`. This applies everywhere job arguments are supplied: `wmill script run/preview`, +`wmill flow run/preview`, the `runScriptByPath` / `runFlowByPath` API, a schedule's `args`, a +trigger's configured static args. + +```json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +``` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with `$res:` / `$var:`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +```json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +``` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(`{ "gh_auth": { "token": "$var:g/all/gh_token" } }`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 122221df80..f148f1f03c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.800.1", + "version": "1.803.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 802d99056a..4970364ccd 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.800.1", + "version": "1.803.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 16cbd6c8ab..e5952c62ae 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.800.1 +1.803.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 197da5a8a8..d8c4b1e0fd 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.800.1", + "version": "1.803.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.800.1", + "version": "1.803.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index eb1eded0cc..8245569685 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.800.1", + "version": "1.803.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",