From 3ce158ad58286b49fd056dbefed2aa2d7bb439c6 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 18 Sep 2026 14:47:44 -0400 Subject: [PATCH 01/53] chore(frontend): upgrade quill to 2.0.3 (#11231) * chore(frontend): upgrade quill to 2.0.3 Co-Authored-By: Claude Fable 5.1 * fix(apps): load rich text editor HTML through Quill's converter so stored lists survive Quill 2 Co-Authored-By: Claude Fable 5.1 * fix(apps): emit standard ul/ol from the rich text editor output under Quill 2 Co-Authored-By: Claude Fable 5.1 * fix(apps): keep checklist state when normalising the rich text editor output Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- frontend/package-lock.json | 287 +++--------------- frontend/package.json | 2 +- .../components/inputs/AppQuillEditor.svelte | 57 +++- 3 files changed, 91 insertions(+), 255 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 82e2f46d5c..f0c3ca61c5 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -67,7 +67,7 @@ "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", "quicktype-core": "^23.2.6", - "quill": "^1.3.7", + "quill": "^2.0.3", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", @@ -3746,28 +3746,11 @@ "node": ">=8.10.0" } }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -3781,6 +3764,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -5222,26 +5206,6 @@ "dev": true, "license": "MIT" }, - "node_modules/deep-equal": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.2.tgz", - "integrity": "sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg==", - "license": "MIT", - "dependencies": { - "is-arguments": "^1.1.1", - "is-date-object": "^1.0.5", - "is-regex": "^1.1.4", - "object-is": "^1.1.5", - "object-keys": "^1.1.1", - "regexp.prototype.flags": "^1.5.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -5284,40 +5248,6 @@ "node": ">=6" } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/defu": { "version": "6.1.7", "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", @@ -5570,6 +5500,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -5684,6 +5615,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5693,6 +5625,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5709,6 +5642,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -6176,9 +6110,9 @@ } }, "node_modules/eventemitter3": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-2.0.3.tgz", - "integrity": "sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, "node_modules/events": { @@ -6233,9 +6167,9 @@ "license": "MIT" }, "node_modules/fast-diff": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.1.2.tgz", - "integrity": "sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", "license": "Apache-2.0" }, "node_modules/fast-equals": { @@ -6522,15 +6456,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/functions-have-names": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -6569,6 +6495,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -6593,6 +6520,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -6776,6 +6704,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6868,22 +6797,11 @@ "node": ">=8" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6896,6 +6814,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -6917,6 +6836,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -7315,22 +7235,6 @@ "node": ">=12" } }, - "node_modules/is-arguments": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", - "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -7368,22 +7272,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-date-object": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", - "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -7474,24 +7362,6 @@ "@types/estree": "^1.0.6" } }, - "node_modules/is-regex": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", - "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "gopd": "^1.2.0", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -8686,6 +8556,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -10139,31 +10010,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-is": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.6.tgz", - "integrity": "sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.7", - "define-properties": "^1.2.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -10356,9 +10202,9 @@ } }, "node_modules/parchment": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/parchment/-/parchment-1.1.4.tgz", - "integrity": "sha512-J5FBQt/pM2inLzg4hEWmzQx/8h8D0CiDxaG3vyp9rKrQRSDgBlhjdP5jQGgosEajXPSQouXGHOmVdgo7QmJuOg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parchment/-/parchment-3.0.0.tgz", + "integrity": "sha512-HUrJFQ/StvgmXRcQ1ftY6VEZUq3jA2t9ncFN4F84J/vN0/FPpQF+8FKXb3l6fLces6q0uOHj6NJn+2xvZnxO6A==", "license": "BSD-3-Clause" }, "node_modules/parent-module": { @@ -11677,31 +11523,32 @@ } }, "node_modules/quill": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/quill/-/quill-1.3.7.tgz", - "integrity": "sha512-hG/DVzh/TiknWtE6QmWAF/pxoZKYxfe3J/d/+ShUWkDvvkZQVTPeVmUJVu1uE6DDooC4fWTiCLh84ul89oNz5g==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/quill/-/quill-2.0.3.tgz", + "integrity": "sha512-xEYQBqfYx/sfb33VJiKnSJp8ehloavImQ2A6564GAbqG55PGw1dAWUn1MUbQB62t0azawUS2CZZhWCjO8gRvTw==", "license": "BSD-3-Clause", "dependencies": { - "clone": "^2.1.1", - "deep-equal": "^1.0.1", - "eventemitter3": "^2.0.3", - "extend": "^3.0.2", - "parchment": "^1.1.4", - "quill-delta": "^3.6.2" + "eventemitter3": "^5.0.1", + "lodash-es": "^4.17.21", + "parchment": "^3.0.0", + "quill-delta": "^5.1.0" + }, + "engines": { + "npm": ">=8.2.3" } }, "node_modules/quill-delta": { - "version": "3.6.3", - "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-3.6.3.tgz", - "integrity": "sha512-wdIGBlcX13tCHOXGMVnnTVFtGRLoP0imqxM696fIPwIf5ODIYUHIvHbZcyvGlZFiFhK5XzDC2lpjbxRhnM05Tg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-5.1.0.tgz", + "integrity": "sha512-X74oCeRI4/p0ucjb5Ma8adTXd9Scumz367kkMK5V/IatcX6A0vlgLgKbzXWy5nZmCGeNJm2oQX0d2Eqj+ZIlCA==", "license": "MIT", "dependencies": { - "deep-equal": "^1.0.1", - "extend": "^3.0.2", - "fast-diff": "1.1.2" + "fast-diff": "^1.3.0", + "lodash.clonedeep": "^4.5.0", + "lodash.isequal": "^4.5.0" }, "engines": { - "node": ">=0.10" + "node": ">= 12.0.0" } }, "node_modules/rbush": { @@ -11874,26 +11721,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/regexp.prototype.flags": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", - "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "define-properties": "^1.2.1", - "es-errors": "^1.3.0", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "set-function-name": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/rehype-github-alerts": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/rehype-github-alerts/-/rehype-github-alerts-3.0.0.tgz", @@ -12290,38 +12117,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/set-function-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", - "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "functions-have-names": "^1.2.3", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", diff --git a/frontend/package.json b/frontend/package.json index fa8da23774..c81465a19a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -145,7 +145,7 @@ "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", "quicktype-core": "^23.2.6", - "quill": "^1.3.7", + "quill": "^2.0.3", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", diff --git a/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte b/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte index 9d1302233f..17ac339e60 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte @@ -8,7 +8,7 @@ import InputValue from '../helpers/InputValue.svelte' import InitializeComponent from '../helpers/InitializeComponent.svelte' - let editor = $state() + let editor: HTMLElement | undefined = $state() let quill: any = $state() interface Props { id: string @@ -42,6 +42,7 @@ async function loadQuill() { const { default: Quill } = await import('quill') + if (!editor) return quill = new Quill(editor, { modules: { @@ -51,33 +52,73 @@ placeholder: placeholder }) - if (defaultValue) { - quill.root.innerHTML = defaultValue - } - quill.on('text-change', function (delta, oldDelta, source) { setOutput() }) + + if (defaultValue) { + setHtml(defaultValue) + } + } + + // Load HTML through Quill's clipboard converter rather than assigning `root.innerHTML`: + // Quill only keeps markup it has a blot for, and writes to the DOM bypass that mapping. + // Quill 2 renders every list as `
    `, so a `
      ` from stored content would be dropped + // outright instead of becoming a bullet list. + function setHtml(html: string | undefined) { + quill.setContents(quill.clipboard.convert({ html: html ?? '', text: '' })) } $componentControl[untrack(() => id)] = { setValue(nvalue: string) { if (quill) { - quill.root.innerHTML = nvalue + setHtml(nvalue) setOutput() } } } + // Quill 2 keeps every list in an `
        ` and marks the kind with `data-list` on each + // item (`bullet`, `ordered`, `checked`, `unchecked`), plus a `span.ql-ui` that only its + // stylesheet renders. Consumers of the output (an HTML component, stored content) render + // it outside that stylesheet, where that markup reads as a numbered list. Emit the + // standard shape instead: `
          `/`
            ` runs, checklists as `
              `, which + // is what the clipboard converter reads back on load (`matchList`). + function toStandardHtml(root: HTMLElement): string { + const clone = root.cloneNode(true) as HTMLElement + clone.querySelectorAll('span.ql-ui').forEach((s) => s.remove()) + clone.querySelectorAll('ol').forEach((ol) => { + const lists: HTMLElement[] = [] + for (const li of Array.from(ol.children)) { + const kind = li.getAttribute('data-list') + li.removeAttribute('data-list') + const tag = kind === 'ordered' ? 'ol' : 'ul' + const checked = kind === 'checked' ? 'true' : kind === 'unchecked' ? 'false' : null + const last = lists.at(-1) + if ( + last?.tagName.toLowerCase() !== tag || + last.getAttribute('data-checked') !== checked + ) { + const list = document.createElement(tag) + if (checked !== null) list.setAttribute('data-checked', checked) + lists.push(list) + } + lists.at(-1)!.appendChild(li) + } + ol.replaceWith(...lists) + }) + return clone.innerHTML + } + function setOutput() { if (quill) { - outputs?.result.set(quill.root.innerHTML) + outputs?.result.set(toStandardHtml(quill.root)) } } function handleDefault(defaultValue: string | undefined) { if (quill) { - quill.root.innerHTML = defaultValue + setHtml(defaultValue) setOutput() } } From 8f9a94328c832931f5bd11fd66865fb76eebb1ba Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 18 Sep 2026 14:48:07 -0400 Subject: [PATCH 02/53] give the rich text editor toolbar a bullet list button instead of a second ordered one (#11233) Co-authored-by: Claude Fable 5.1 --- .../lib/components/apps/components/inputs/AppQuillEditor.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte b/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte index 17ac339e60..83976a3ecd 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte @@ -24,7 +24,7 @@ toolbarOptions = [ [{ header: 1 }, { header: 2 }, 'blockquote', 'link', 'image', 'video'], ['bold', 'italic', 'underline', 'strike'], - [{ list: 'ordered' }, { list: 'ordered' }], + [{ list: 'ordered' }, { list: 'bullet' }], [{ align: [] }], ['clean'] ] From 974b0265f85f9cfac65bbd2f6ebbcb0e966e3301 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 18 Sep 2026 20:50:23 +0200 Subject: [PATCH 03/53] fix(cli): read wmill.yaml from the branch a git-sync deploy writes to (#11236) * fix(cli): read wmill.yaml from the branch a git-sync deploy writes to Co-Authored-By: Claude Opus 5 (1M context) * fix(cli): create the stateful dir once the deploy branch's config is read Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/commands/sync/sync.ts | 31 +++++-- cli/test/gitsync_deploy_branch_config.test.ts | 91 +++++++++++++++++++ 2 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 cli/test/gitsync_deploy_branch_config.test.ts diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6040afc747..32cff4a3cb 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -3594,12 +3594,15 @@ export async function pull( ) { if ((opts as any).jsonOutput) log.setSilent(true); const originalCliOpts = { ...opts }; - opts = await mergeConfigWithConfigFile(opts); - - // --include-secrets overrides skipSecrets from wmill.yaml - if ((originalCliOpts as any).includeSecrets) { - opts.skipSecrets = false; - } + const withConfigFile = async () => { + const merged = await mergeConfigWithConfigFile({ ...originalCliOpts }); + // --include-secrets overrides skipSecrets from wmill.yaml + if ((originalCliOpts as any).includeSecrets) { + merged.skipSecrets = false; + } + return merged; + }; + opts = await withConfigFile(); // Resolve workspace name for config lookups. // --branch resolves git branch → workspace name (deprecated but still supported). @@ -3634,10 +3637,6 @@ export async function pull( throw error; } - if (opts.stateful) { - await mkdir(path.join(process.cwd(), ".wmill"), { recursive: true }); - } - const workspace = await resolveWorkspace(opts, wsNameForConfig); await requireLogin(opts); @@ -3729,6 +3728,14 @@ export async function pull( }); return; } + + // The pull writes into the branch now checked out, so its wmill.yaml + // applies, not the cloned branch's: a fork branch that turned on + // `dedupeLockfiles` would otherwise get one lockfile per script back. + if (getCurrentGitBranch() !== clonedBranchName) { + opts = await withConfigFile(); + wsNameForConfig = resolveWsNameForConfigFromFlags(opts); + } } // If wsNameForConfig wasn't set from flags, infer from the resolved profile @@ -3763,6 +3770,10 @@ export async function pull( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + if (opts.stateful) { + await mkdir(path.join(process.cwd(), ".wmill"), { recursive: true }); + } + const codebases = await listSyncCodebases(opts); log.info( diff --git a/cli/test/gitsync_deploy_branch_config.test.ts b/cli/test/gitsync_deploy_branch_config.test.ts new file mode 100644 index 0000000000..39c435d872 --- /dev/null +++ b/cli/test/gitsync_deploy_branch_config.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createRemoteWorkspaceDeps, withTestBackend } from "./test_backend.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// The git-sync deploy callback starts in a clone of the tracked branch and +// switches to the fork's branch before pulling. What it writes there must follow +// that branch's wmill.yaml: a fork branch that turned on `dedupeLockfiles` keeps +// its shared lockfile instead of getting a `.script.lock` per script back. +test("git-sync fork deploy follows the fork branch's wmill.yaml", async () => { + await withTestBackend(async (backend) => { + const post = (path: string, body: unknown) => + backend.apiRequest!(`/api/w/${backend.workspace}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + await createRemoteWorkspaceDeps(backend, "python3", "wmill\n"); + await post("/folders/create", { name: "dedupe" }); + const script = await post("/scripts/create", { + path: "f/dedupe/a", + summary: "", + description: "", + content: "def main():\n pass\n", + language: "python3", + lock: "wmill==1.0.0\n", + }); + expect(script.ok).toBe(true); + + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_bare_")); + const seed = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_seed_")); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_work_")); + try { + execFileSync("git", ["init", "--bare", "--initial-branch=main", bare]); + git(seed, "init", "--initial-branch=main"); + git(seed, "config", "user.email", "seed@windmill.dev"); + git(seed, "config", "user.name", "seed"); + git(seed, "remote", "add", "origin", `file://${bare}`); + const wmillYaml = "defaultTs: bun\nincludes:\n - f/dedupe/**\nexcludes: []\n"; + await writeFile(join(seed, "wmill.yaml"), wmillYaml); + git(seed, "add", "-A"); + git(seed, "commit", "-m", "main"); + git(seed, "push", "origin", "main"); + + // Any parent id makes the callback deploy as a fork, to this branch. + const forkBranch = `wm-fork/main/${backend.workspace}`; + git(seed, "checkout", "-b", forkBranch); + await writeFile( + join(seed, "wmill.yaml"), + wmillYaml + "dedupeLockfiles: true\n", + ); + const pulled = await backend.runCLICommand(["sync", "pull", "--yes"], seed); + expect(pulled.code).toBe(0); + expect(existsSync(join(seed, "locks/requirements.in.lock"))).toBe(true); + git(seed, "add", "-A"); + git(seed, "commit", "-m", "dedupe"); + git(seed, "push", "origin", forkBranch); + + git(work, "clone", `file://${bare}`, "."); + const deployed = await backend.runCLICommand( + [ + "sync", + "git-deploy", + "--repository", + "u/test/unused", + "--git-deploy-items", + JSON.stringify([ + { path_type: "script", path: "f/dedupe/a", commit_msg: "deploy" }, + ]), + "--parent-workspace-id", + "parent", + ], + work, + ); + expect(deployed.code).toBe(0); + expect(git(work, "rev-parse", "--abbrev-ref", "HEAD")).toBe(forkBranch); + expect(git(work, "status", "--porcelain")).toBe(""); + } finally { + await rm(bare, { recursive: true, force: true }); + await rm(seed, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); + } + }); +}); From f00b2fcb1e65cb8f8e95d55f7cc58b816ac7d89d Mon Sep 17 00:00:00 2001 From: Guilhem Date: Fri, 18 Sep 2026 20:54:23 +0200 Subject: [PATCH 04/53] =?UTF-8?q?feat:=20drafts=20follow=20their=20item=20?= =?UTF-8?q?through=20a=20move;=20behind=20means=20base=20=E2=89=A0=20head?= =?UTF-8?q?=20(#10577)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: give home multi-select a reserved gutter and a menu entry Co-Authored-By: Claude Opus 5 (1M context) * fix: keep checkbox theming and reserve the gutter on non-selectable rows Co-Authored-By: Claude Opus 5 (1M context) * feat: carry every draft with an item when it moves Co-Authored-By: Claude Opus 5 (1M context) * feat: move draft-only items and warn editors when an item moves Co-Authored-By: Claude Opus 5 (1M context) * refactor: put the home selection checkbox back in the kind icon slot Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014FaEacdxR6M6VDej6C9r39 * feat: animate the home bulk bar and exit selection at zero Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014FaEacdxR6M6VDej6C9r39 * fix: keep dialog icon badges round and the panel inside narrow viewports Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014FaEacdxR6M6VDej6C9r39 * fix: address review findings on the draft-carry path Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014FaEacdxR6M6VDej6C9r39 * fix: keep a staged rename when a move carries the draft Co-Authored-By: Claude Opus 5 (1M context) * fix: restamp only the deployer's own carried draft Co-Authored-By: Claude Opus 5 (1M context) * fix: scope the moved-save restamp to the mover as well Co-Authored-By: Claude Opus 5 (1M context) * fix: read the app move's author from the head version, not the draft's base Co-Authored-By: Claude Opus 5 (1M context) * fix: carry a flow draft's baseline path so deploying it cannot un-move the flow Co-Authored-By: Claude Opus 5 (1M context) * fix: reject unsupported kinds in move_draft, survive NUL-poisoned draft rows Co-Authored-By: Claude Opus 5 (1M context) * fix: skip NUL-poisoned rows in every draft-value rewrite, not just the first Co-Authored-By: Claude Opus 5 (1M context) * fix: report a NUL-poisoned draft on move instead of 500ing Co-Authored-By: Claude Opus 5 (1M context) * fix: name the attempted operation in the NUL rejection message Co-Authored-By: Claude Opus 5 (1M context) * refactor: drop dead selection code and comments that outlived their state Co-Authored-By: Claude Opus 5 (1M context) * docs: describe script staleness as head-pinned, which is what the loader does Co-Authored-By: Claude Opus 5 (1M context) * docs: correct the third staleness comment left claiming a stable fork base Co-Authored-By: Claude Opus 5 (1M context) * fix: address CI review — auth order, save race, carry failure, path validation Co-Authored-By: Claude Opus 5 (1M context) * fix: gate operators earlier, skip the write tx without lineage, unblock a chained move Co-Authored-By: Claude Opus 5 (1M context) * fix: run the post-write moved re-assert under RLS, not the raw pool Co-Authored-By: Claude Opus 5 (1M context) * test: pin the moved answer to what the saver can see The post-write re-assert names a path and a username, and nothing at any layer stopped it reading them off a raw pool connection. Swapping the transaction back to `db.begin()` compiles and passes everything else, so the guard has to be a test: a non-admin saving at a path whose item moved into a folder they cannot see gets `saved`, while the admin gets `moved`. Also drops two doc comments still arguing that clearing the write gate at the old path removes the need for an RLS envelope. It does not — the gate resolves the old path and the re-assert asks about the new one. Co-Authored-By: Claude Opus 5 (1M context) * docs: name the real deploy path and stop restating the RLS constraint `update_path` is not a symbol in this repo; a script move goes through `create_script`. The re-assert's comment re-derived the disclosure argument that already sits on `resolve_moved_to_in`, where a caller would break it. Co-Authored-By: Claude Opus 5 (1M context) * docs: state the RLS and restamp constraints once each The RLS envelope was argued at three sites in drafts.rs; it now sits only on `resolve_moved_to_in`, whose signature is what a caller would break. The restamp scoping was copy-pasted at all three deploy call sites while already documented in full on `move_drafts_for_path`. Co-Authored-By: Claude Opus 5 (1M context) * fix: carry both path keys on a move, and grant the draft sequence The upsert now runs as `windmill_user`, so it calls nextval on `draft_id_seq` as that role. The only thing granting that is the ALTER DEFAULT PRIVILEGES in 20250205131523, whose DO block swallows failures — so an instance where it errored would fail every autosave with `permission denied for sequence`. A draft value carries two path keys: the typed one and a mirror the editors keep in step with it while it differs from the row's path. Rewriting only the typed one left the mirror naming the old location, and the loaders prefer the mirror — reopening a moved session script restored the old path and the next save un-did the move. Both keys now follow, in the move endpoint and in the passive carry, under the same tri-state rule. `typed_path_field` answered `draft_path` for every non-script kind, including resources, variables and triggers, which have no such key. It returns `None` for them now, and `move_draft` reads its guard off that mapping so the movable set and the field mapping cannot drift apart. Also documents that `move_drafts_for_path` mutates every owner's row and enforces nothing itself, and parses the draft payload once per save instead of three times. Co-Authored-By: Claude Opus 5 (1M context) * test: pin the two-key move, and stop the down migration breaking instances Revoking the sequence grant would strip a privilege a healthy instance had before this migration ran — the grant it adds is indistinguishable in the catalog from the one ALTER DEFAULT PRIVILEGES gives at creation time — so the down is a comment, matching the other grant-only migrations. The mirror rewrite is spread over three sites that have to agree and fails silently when they don't, so it gets a test: a draft carrying both path keys has both moved, and one carrying neither mirror does not gain one. It reads the value back over HTTP rather than with `sqlx::query!`, which would need an offline cache entry of its own. Also drops twelve `.sqlx` entries this branch added and then superseded, and corrects the doc and openapi text that still described only the typed path being rewritten. Co-Authored-By: Claude Opus 5 (1M context) * docs: point the empty down at the grant it is declining to revoke Co-Authored-By: Claude Opus 5 (1M context) * feat: drop the restamp and tri-state; a move relocates the draft row only A deploy that renames an item is a deploy like any other: every draft on the item goes stale, and the stale prompt with its diff is the single mechanism to catch up. move_drafts_for_path now touches only the row's path column, so the value keeps the base version the draft actually forked from, and the "moved" patch carries no version restamp. DraftBaseVersion shrinks to the three per-kind lineage fields. Co-Authored-By: Claude Fable 5.1 * feat: stale prompt links to a diff that names and lets you pick the deployed version The stale-draft prompt gains "See what changed", which opens the diff drawer. The drawer resolves the deployed side by the draft row's own path (not the typed path, which after a rename still names the archived row), labels which version the left pane is, and offers a picker over the item's deployed history for scripts, flows and raw apps. The history endpoints return created_by (and created_at for apps) so each entry can name its deployer. "Restore to deployed" moves to the header actions. Co-Authored-By: Claude Fable 5.1 * test: move_to asserts the response status Co-Authored-By: Claude Fable 5.1 * fix: keep a script draft's base at the version it forked from The script editor seeded the draft's parent_hash from the deployed head on every load, and the next autosave persisted it, so a draft behind the deploy read as up to date after being opened once. The base now comes from the draft when one exists; the head is only used for a fresh checkout or an explicit topHash. Deploy already fetches the live head and confirms on mismatch, so the base is what makes that check meaningful. The webhook "run this version" URL uses the deployed hash rather than the draft's base. Co-Authored-By: Claude Fable 5.1 * feat: store the version a draft forked from in one draft.base column Every kind kept its fork base under a different name and type inside the value: parent_hash (hex) for scripts, version_id for flows, parent_version for apps. draft.base holds it as one text id, derived on save from the value so every writer fills it the same way, backfilled by the migration (rows holding a NUL are skipped, since ->> raises on them). The get-by-path overlay exposes it as draft_base and the drafts list as base; the editors and the compare page read that one field and compare it to the head as text. Co-Authored-By: Claude Fable 5.1 * feat: raw-app drafts carry a fork base, so behind means base != head for them too The raw-app bundle never carried the version it forked from, which left raw apps on the timestamp check that self-heals as you type, and the header's deploy guard read a version prop nothing set, so deploying over a newer version never asked. The route now stamps parent_version into the bundle (the draft's own base when it has one, else the head), the server derives draft.base from it, the stale prompt compares it to the head and links to the diff, and the editor threads it to the header so the deploy guard confirms. A deploy re-pins the base to the version it wrote. Co-Authored-By: Claude Fable 5.1 * feat: refuse a rename onto a path that already holds a draft A draft occupies its path the way a deployed item does: a never-deployed item, or a draft left on an archived script. Renaming onto it would either merge two items or leave the losing row stranded at a path its item has left. The move now refuses with a BadRequest inside the deploy's transaction, so the rename itself fails and the source stays deployed. Every draft on the item then moves; there is no longer a left-behind count to report. Co-Authored-By: Claude Fable 5.1 * feat: save drafts by row id, so an open editor follows its draft through a move A rename carries every draft on the item to the new path. An editor left open across it was still saving by the path it opened on, which the server had to refuse and answer with where the item went (the "moved" handshake and its modal). The draft row has an id: the get-by-path overlay now returns it as draft_id, every later save sends it, and the server writes the row wherever it is and answers with that path. The editor then follows: it flushes what it holds, tells the user, and navigates to the item's new path, where the stale prompt says what changed. The lineage-based move resolvers, the moved status and the moved modal are gone. Co-Authored-By: Claude Fable 5.1 * feat: the out-of-date prompt names both versions and can take the latest as the new base The prompt now says which version the draft forked from and which is deployed (and by whom), instead of two timestamps, and gains "Take latest, keep my edits": the draft's base moves to the head and its content stays, so the user can acknowledge a newer version without discarding their work. Each route sets its kind's base field on the draft value and persists it; the raw-app bundle carries it already, so setting the state is enough there. Co-Authored-By: Claude Fable 5.1 * feat: two-action out-of-date prompt; taking the latest moves into the diff drawer Four buttons made the prompt hard to read. It keeps "See what changed" and a red "Use latest" (it replaces the draft); closing it is keeping the draft. "Take latest, keep my edits" moves to the diff drawer's header, offered only while the draft is behind, so the user takes the latest with the diff in front of them. Scripts, flows and raw apps pass the action through their diff drawer; the classic app editor has no drawer wired to the prompt and loses it. Co-Authored-By: Claude Fable 5.1 * chore: drop the draft_id_seq grant; the draft upsert runs on the raw pool Co-Authored-By: Claude Opus 5 (1M context) * fix: a moved draft's path keys follow it, and a refused rename names the draft's owner Co-Authored-By: Claude Opus 5 (1M context) * fix: follow a moved draft on tab close, and deploy a followed flow at its new path Co-Authored-By: Claude Opus 5 (1M context) * fix: write a followed draft by id against the row's own path keys; keep base on assign and clone Co-Authored-By: Claude Opus 5 (1M context) * fix: look up a script's head at its row path, and show flow and app version ids bare Co-Authored-By: Claude Opus 5 (1M context) * fix: session editors save by draft id; raw apps keep a legacy draft's base unknown Also advance the raw-app base on deploy, relocate once per move, drop the hoisted operator check and the unread base on drafts/list. Co-Authored-By: Claude Opus 5 (1M context) * fix: guard a base-unknown raw-app deploy against the head at load; keep the base in session hydration Co-Authored-By: Claude Opus 5 (1M context) * refactor: the server follows a moved draft through a move record, not client-sent row ids A move writes old path -> new path (per workspace and kind, per owner for a draft-only move) in its transaction; a draft save or discard addressed to a path the caller has no draft at resolves through it and keeps the moved draft's path keys. Creating an item at a path drops the records leaving it. Every writer (edit routes, sessions, chat, CLI, the tab-close flush) follows without passing an id, so the id plumbing is gone. Co-Authored-By: Claude Opus 5 (1M context) * fix: session loaders keep a draft's base, and a failed relocation flush stays put Co-Authored-By: Claude Opus 5 (1M context) * fix: a draft-only app move refuses the other app kind; a session keeps an unknown base unknown Co-Authored-By: Claude Opus 5 (1M context) * test: pin a teammate's carried draft; name the kind that refuses a draft move Co-Authored-By: Claude Opus 5 (1M context) * fix: an unknown base stays unknown in every loader, and an owner move extends an item move Co-Authored-By: Claude Opus 5 (1M context) * fix: a clone keeps only a base it can resolve; a base-unknown script deploys without a false guard Co-Authored-By: Claude Opus 5 (1M context) * fix: a workspace clone sanitizes a NUL-bearing draft instead of copying it unstripped Co-Authored-By: Claude Opus 5 (1M context) * fix: move records follow an account rename and deletion; a legacy draft says why it cannot move Co-Authored-By: Claude Opus 5 (1M context) * fix: an owner move extends only the item's own route, not another user's Co-Authored-By: Claude Opus 5 (1M context) * fix: a redeploy ends a route off its path, take-latest persists on raw apps, stale picker loads are dropped Co-Authored-By: Claude Opus 5 (1M context) * fix: a poisoned draft's path keys follow a move, legacy only bypasses routing on a delete, picker loads are generation-guarded Co-Authored-By: Claude Opus 5 (1M context) * fix: count picker load generations, and report a skipped legacy upsert as a conflict Co-Authored-By: Claude Opus 5 (1M context) * fix: a legacy discard follows the item's move record too Co-Authored-By: Claude Opus 5 (1M context) * fix: a failed version load keeps the picker on what the diff shows; one spelling for a legacy delete Co-Authored-By: Claude Opus 5 (1M context) * fix: the picker marks the version on display as head, restore compares the head, relocation follows the last move Co-Authored-By: Claude Opus 5 (1M context) * fix: say so when a version fails to load in the diff picker Co-Authored-By: Claude Opus 5 (1M context) * fix: take latest re-reads the head at click time; type the kept head as prepared diff data Co-Authored-By: Claude Opus 5 (1M context) * fix: taking the latest moves the head each editor knows, not just the base Co-Authored-By: Claude Opus 5 (1M context) * feat: take latest adopts the head the diff shows, and is offered while the drawer sees the draft behind Co-Authored-By: Claude Opus 5 (1M context) * fix: a head nobody could name is not behind, so take latest is not offered without one Co-Authored-By: Claude Opus 5 (1M context) * fix: the flow drawer's head is the version its payload came from, and its callback type says so Co-Authored-By: Claude Opus 5 (1M context) * fix: a NUL in a move's summary is dropped, and take latest simply adopts the head it was handed Co-Authored-By: Claude Opus 5 (1M context) * fix: a loaded raw-app draft keeps its own fork base, and an unknown head is refused Co-Authored-By: Claude Opus 5 (1M context) * fix: a routed discard names where it landed, a superseded drawer opening is dropped, and a loaded draft keeps its base in every editor Co-Authored-By: Claude Opus 5 (1M context) * fix: a legacy draft occupies its destination, a superseded opening writes nothing, and a loaded flow draft keeps no base it lacks Co-Authored-By: Claude Opus 5 (1M context) * fix: the drawer owns its opening, a loaded script draft keeps no base it lacks, and a legacy occupant says who can clear it Co-Authored-By: Claude Opus 5 (1M context) * fix: taking the diff drawer without a token claims it, and the classic app editor takes one Co-Authored-By: Claude Opus 5 (1M context) * fix: a retried routed discard still names the destination, and filling the drawer takes the opening too Co-Authored-By: Claude Opus 5 (1M context) * fix: a no-op routed discard names the destination only to someone who could write there Co-Authored-By: Claude Opus 5 (1M context) * fix: the no-op routed discard gates its answer on reading the destination, and a session draft keeps its unknown base Co-Authored-By: Claude Opus 5 (1M context) * fix: abandoning an opening clears the drawer it still owns Co-Authored-By: Claude Opus 5 (1M context) * fix: an app deploy pins only a version it wrote as the next draft's base Co-Authored-By: Claude Opus 5 (1M context) * fix: the deploy-override diff takes an opening its editor can hand back Co-Authored-By: Claude Opus 5 (1M context) * refactor: pin the version this deploy wrote even when one landed on top, and tighten three comment blocks Co-Authored-By: Claude Opus 5 (1M context) * fix: a deploy claims only the version it appended to the head it read, and names the head separately Co-Authored-By: Claude Opus 5 (1M context) * fix: a deploy always names the head it left behind, and pins a base only when it can claim one Co-Authored-By: Claude Opus 5 (1M context) * fix: the derived base is read after the sanitizer, and a deploy that claims nothing leaves no base to compare Co-Authored-By: Claude Opus 5 (1M context) * fix: the route's lineage follows an in-place deploy, and the raw-app editor's event type carries the head Co-Authored-By: Claude Opus 5 (1M context) * docs: the raw-app deploy comment says what that editor actually does with version Co-Authored-By: Claude Opus 5 (1M context) * fix: a group member can be told where their item went, and a deploy names the head's author Co-Authored-By: Claude Opus 5 (1M context) * fix: an emptied selection is no shift anchor, and a deploy leaves no draft for the prompt to compare Co-Authored-By: Claude Opus 5 (1M context) * fix: session tabs compare the same base pair, and a consumed draft is not out of date Co-Authored-By: Claude Opus 5 (1M context) * fix: a failed anchor read is not a raced deploy, and take latest closes only its own drawer Co-Authored-By: Claude Opus 5 (1M context) * fix: an unclaimed deploy always confirms, and the prompt keeps warning a loaded teammate draft Co-Authored-By: Claude Opus 5 (1M context) * docs: the base-unknown confirmation says what it knows, and two comments match the guard Co-Authored-By: Claude Opus 5 (1M context) * fix: the other app kind collides whoever owns it, and session tabs get a head to compare Co-Authored-By: Claude Opus 5 (1M context) * fix: the cross-kind refusal reads properly, and a session flow keeps its own response's head Co-Authored-By: Claude Opus 5 (1M context) * fix: a fresh session checkout takes the head its payload came from, and a deploy keeps the base it pinned Co-Authored-By: Claude Opus 5 (1M context) * fix: the move endpoint validates its source path, and two comments say what their branch does Co-Authored-By: Claude Opus 5 (1M context) * fix: an unanswered head read confirms rather than assuming the app editor is current Co-Authored-By: Claude Opus 5 (1M context) * fix: a deploy is not blocked by the draft a move carried to its destination Co-Authored-By: Claude Opus 5 (1M context) * fix: an unread head confirms with the copy for caution, not for an observed deploy Co-Authored-By: Claude Opus 5 (1M context) * fix: the move record alone excuses a carried draft at the destination, whoever owns it Co-Authored-By: Claude Opus 5 (1M context) * feat: an app deploy answers with the version it wrote, so the editor stops inferring it Co-Authored-By: Claude Opus 5 (1M context) * test: the rename assertion reads the deploy's json answer Co-Authored-By: Claude Opus 5 (1M context) * fix: the unread-head warning reads as caution in the deploy drawer too, and the cross-kind refusal names a remedy Co-Authored-By: Claude Opus 5 (1M context) * fix: a reused destination retires the routes pointing at it, and draft_base stays out of diffs Co-Authored-By: Claude Opus 5 (1M context) * fix: the app head is the tail of app.versions, not the newest timestamp Co-Authored-By: Claude Opus 5 (1M context) * fix: app history lists in deployed order, so the picker numbers it right Co-Authored-By: Claude Opus 5 (1M context) * fix: the ordering test's setup sql compiles offline, and the head join names its app Co-Authored-By: Claude Opus 5 (1M context) * fix: kinds that cannot move skip the move lookup, and the move wording needs read Co-Authored-By: Claude Opus 5 (1M context) * perf: a deploy history comes a page at a time, so the diff drawer opens at once Co-Authored-By: Claude Opus 5 (1M context) * fix: a history stays whole unless asked to page, and pages inside the version array Co-Authored-By: Claude Opus 5 (1M context) * fix: an asked-for history page is bounded, and a failed one is not the end of the list Co-Authored-By: Claude Opus 5 (1M context) * fix: an unasked history is whole again, and an absurd page is empty not an error Co-Authored-By: Claude Opus 5 (1M context) * fix: naming only a page still asks for one, and a stray version stays reachable Co-Authored-By: Claude Opus 5 (1M context) * fix: a fork's nul-poisoned draft arrives clean, so its dangling identity repoints too Co-Authored-By: Claude Opus 5 (1M context) * fix: a raw app names its deployed version even when the history will not load Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Ruben Fiszel --- ...f29c87ce48ffd89fac070c8314556f884ce55.json | 43 ++ ...fa00ab137c70b73f381452cadfae6e1cda934.json | 17 + ...94b329c4ec7978b016af4c964415e0c38396c.json | 71 +++ ...e6f80b2a41db4f1d352920d0c295ea3a018b1.json | 64 ++ ...d5fb3b2120835158f22d715b73460f6a55108.json | 14 + ...fc8a1db451d31dfddbb5be945b3551c8becb.json} | 10 +- ...7e47481877f3ab717682f3b84e665c95a5214.json | 49 ++ ...a575bfcd4469524f0dc8812470737759f02b.json} | 4 +- ...9698dbc44a18c0f016aae94a1705f06fd1e48.json | 18 + ...155e85370d4c2b7ed14fbb510919e7624ca1c.json | 18 + ...6122e78f0c76a93a1d9860e8dff05d9a02c09.json | 18 + ...4627cb1fd3d8c70210bf7f38cb222dd15d88f.json | 15 + ...b4e84a61008dbec0995ae192f1b3a1c261579.json | 72 +++ ...7e0055d742e44119bfb31ae7ff37a89b34d14.json | 18 + ...53ccaac72871eb68822d8e296c0d747e55f35.json | 43 ++ ...c66df3d19217059f62903be3c74eadb9c15d5.json | 16 - ...88fa7d0b650b5cfd80c099f785f5e411062e5.json | 17 + ...d46af4419dda9410ffc46f19bc38e56efe609.json | 18 + ...42c94445536e201b322eda6f7a90bdf38f36a.json | 35 -- ...bc06dc95d60ec92469523935f5a18a4d3f907.json | 85 +++ ...a1ff36620099c6b87fd82a59eae1e35303c07.json | 15 + ...340efd453512c1464a413af2393ab8bfea9ee.json | 47 ++ ...f30ed81e8c26a9f1e65f423078947153ff9dd.json | 15 + ...9fdb239ef507e23003db03aff7d35b02c540b.json | 16 + ...17510fd05d0b7c5311b61e7257ccd07b16da2.json | 59 ++ ...a895cce1e8ae010307431e6787d20c7b25780.json | 41 ++ ...db558549b2356fc6ac8f0557aa40e1345ff46.json | 31 + ...20260914133558_draft_base_version.down.sql | 1 + .../20260914133558_draft_base_version.up.sql | 14 + .../20260914154128_draft_move.down.sql | 1 + .../20260914154128_draft_move.up.sql | 17 + backend/tests/app_version_order.rs | 193 ++++++ backend/tests/dependency_map.rs | 6 +- backend/tests/drafts_move_mirror.rs | 74 +++ backend/tests/drafts_move_taken.rs | 250 ++++++++ backend/tests/drafts_nul.rs | 43 ++ backend/tests/drafts_save_follows_move.rs | 587 ++++++++++++++++++ backend/tests/fixtures/drafts_move_mirror.sql | 16 + backend/tests/fixtures/drafts_move_taken.sql | 37 ++ .../fixtures/drafts_save_follows_move.sql | 29 + backend/windmill-api-flows/src/flows.rs | 48 +- .../tests/fork_clone_on_behalf_of.rs | 11 +- backend/windmill-api-scripts/src/scripts.rs | 68 +- .../windmill-api-workspaces/src/workspaces.rs | 32 +- .../src/workspaces_extra.rs | 7 + backend/windmill-api/openapi.yaml | 92 ++- backend/windmill-api/src/apps.rs | 108 +++- backend/windmill-api/src/drafts.rs | 490 ++++++++++++++- backend/windmill-common/src/user_drafts.rs | 321 +++++++++- backend/windmill-common/src/utils.rs | 47 ++ .../src/windmill-client.js | 2 +- backend/windmill-types/src/scripts.rs | 4 + frontend/src/lib/components/DiffDrawer.svelte | 289 ++++++++- .../src/lib/components/FlowBuilder.svelte | 152 ++++- frontend/src/lib/components/MoveDrawer.svelte | 37 +- .../src/lib/components/ScriptBuilder.svelte | 160 ++++- .../components/apps/editor/AppEditor.svelte | 4 +- .../apps/editor/AppEditorHeader.svelte | 145 +++-- .../apps/editor/AppEditorHeaderDeploy.svelte | 15 +- .../apps/editor/appDeploy.svelte.ts | 8 +- frontend/src/lib/components/apps/types.ts | 5 + .../lib/components/common/alert/Alert.svelte | 4 +- .../ConfirmationModal.svelte | 17 +- .../DeployOverrideConfirmationModal.svelte | 47 +- .../DraftEditorModals.svelte | 86 ++- .../confirmationModal/StaleDraftModal.svelte | 102 ++- .../lib/components/common/table/AppRow.svelte | 18 +- .../components/common/table/FlowRow.svelte | 17 +- .../components/common/table/ScriptRow.svelte | 17 +- .../components/common/table/rowSelection.ts | 19 + frontend/src/lib/components/diff_drawer.ts | 60 +- frontend/src/lib/components/flow_builder.ts | 15 +- .../lib/components/home/BulkActionsBar.svelte | 182 +++--- .../src/lib/components/home/ItemsList.svelte | 17 +- .../lib/components/home/bulkActions.test.ts | 15 +- .../src/lib/components/home/bulkActions.ts | 46 +- .../components/home/homeSelection.dom.test.ts | 59 ++ .../components/home/homeSelection.svelte.ts | 22 +- .../src/lib/components/moveRenameManager.ts | 14 +- .../components/raw_apps/RawAppEditor.svelte | 38 +- .../raw_apps/RawAppEditorHeader.svelte | 215 ++++++- frontend/src/lib/components/script_builder.ts | 7 + .../components/sessions/FlowEditorView.svelte | 7 + .../sessions/RawAppEditorView.svelte | 7 + .../components/sessions/appDraftCodec.test.ts | 22 + .../lib/components/sessions/appDraftCodec.ts | 13 +- .../sessions/sessionRuntime.svelte.ts | 75 ++- frontend/src/lib/userDraftDbSyncer.svelte.ts | 36 ++ frontend/src/lib/utils.ts | 1 + frontend/src/lib/utils_draft_deploy.test.ts | 30 +- frontend/src/lib/utils_draft_deploy.ts | 36 +- .../(logged)/apps/edit/[...path]/+page.svelte | 45 +- .../apps_raw/edit/[...path]/+page.svelte | 94 ++- .../flows/edit/[...path]/+page.svelte | 49 +- .../scripts/edit/[...path]/+page.svelte | 80 ++- 95 files changed, 5147 insertions(+), 547 deletions(-) create mode 100644 backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json create mode 100644 backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json create mode 100644 backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json create mode 100644 backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json create mode 100644 backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json rename backend/.sqlx/{query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json => query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json} (57%) create mode 100644 backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json rename backend/.sqlx/{query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json => query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json} (74%) create mode 100644 backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json create mode 100644 backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json create mode 100644 backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json create mode 100644 backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json create mode 100644 backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json create mode 100644 backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json create mode 100644 backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json delete mode 100644 backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json create mode 100644 backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json create mode 100644 backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json delete mode 100644 backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json create mode 100644 backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json create mode 100644 backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json create mode 100644 backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json create mode 100644 backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json create mode 100644 backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json create mode 100644 backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json create mode 100644 backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json create mode 100644 backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json create mode 100644 backend/migrations/20260914133558_draft_base_version.down.sql create mode 100644 backend/migrations/20260914133558_draft_base_version.up.sql create mode 100644 backend/migrations/20260914154128_draft_move.down.sql create mode 100644 backend/migrations/20260914154128_draft_move.up.sql create mode 100644 backend/tests/app_version_order.rs create mode 100644 backend/tests/drafts_move_mirror.rs create mode 100644 backend/tests/drafts_move_taken.rs create mode 100644 backend/tests/drafts_save_follows_move.rs create mode 100644 backend/tests/fixtures/drafts_move_mirror.sql create mode 100644 backend/tests/fixtures/drafts_move_taken.sql create mode 100644 backend/tests/fixtures/drafts_save_follows_move.sql create mode 100644 frontend/src/lib/components/home/homeSelection.dom.test.ts diff --git a/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json new file mode 100644 index 0000000000..be7ac83a7b --- /dev/null +++ b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + false, + false + ] + }, + "hash": "044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55" +} diff --git a/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json b/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json new file mode 100644 index 0000000000..a4d174992e --- /dev/null +++ b/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft AS d\n SET path = $3::text,\n value = (\n SELECT CASE\n WHEN s.clean -> 'path' = to_jsonb($2::text)\n OR s.clean -> 'draft_path' = to_jsonb($2::text)\n THEN to_json(\n s.clean\n || CASE WHEN s.clean -> 'path' = to_jsonb($2::text)\n THEN jsonb_build_object('path', $3::text)\n ELSE '{}'::jsonb END\n || CASE WHEN s.clean -> 'draft_path' = to_jsonb($2::text)\n THEN jsonb_build_object('draft_path', $3::text)\n ELSE '{}'::jsonb END\n )\n ELSE d.value\n END\n FROM (SELECT replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)),\n chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean) s\n )\n WHERE d.workspace_id = $1\n AND d.path = $2::text\n AND d.typ::text = ANY($4::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934" +} diff --git a/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json b/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json new file mode 100644 index 0000000000..23d3b51c76 --- /dev/null +++ b/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value as \"value!: sqlx::types::Json>\",\n created_at, base\n FROM draft\n WHERE workspace_id = $1\n AND (email = $2 OR email IS NULL)\n AND path = $3\n AND typ = $4\n ORDER BY email NULLS LAST\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "base", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c" +} diff --git a/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json b/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json new file mode 100644 index 0000000000..93251596cf --- /dev/null +++ b/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft\n SET path = $3,\n -- Both path keys, not just the typed one: the editors mirror the\n -- typed path into the other while it differs from the row's path,\n -- and the loaders prefer the mirror — left naming the old location\n -- it un-does this move on the next save. `create_missing = false`\n -- on both, so a draft carrying only one keeps only one.\n value = to_json(\n jsonb_set(\n jsonb_set(\n CASE WHEN $7::text IS NULL THEN to_jsonb(value)\n ELSE jsonb_set(to_jsonb(value), ARRAY['summary'], to_jsonb($7::text))\n END,\n ARRAY[$5::text], to_jsonb($3::text), false\n ),\n ARRAY[$8::text], to_jsonb($3::text), false\n )\n )\n WHERE workspace_id = $1\n AND path = $2\n AND typ = $4\n AND email = $6\n -- A pre-sanitizer NUL escape makes `to_jsonb` raise 22P05. Excluded\n -- here so the statement can't 500; reported below instead. Unlike the\n -- passive carry, rewriting the value IS this operation, so skipping it\n -- silently would move the row and leave its typed path stale.\n AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0\n -- Skipped on a summary-only edit, where the \"target\" row is this\n -- row and the guard would refuse the update against itself.\n AND ($2 = $3 OR NOT EXISTS (\n SELECT 1 FROM draft o\n WHERE o.workspace_id = $1 AND o.path = $3 AND o.typ::text = ANY($9::text[])\n -- Of this kind only the caller's own row and the legacy one collide:\n -- teammates' drafts of one item share its path by design, but a deploy\n -- there wipes those two together, so a second would discard edits the\n -- caller never saw. The other app kind is a different item on the same\n -- deployed path, so it collides whoever owns it.\n AND (o.typ <> $4 OR o.email = $6 OR o.email IS NULL)\n ))\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1" +} diff --git a/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json b/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json new file mode 100644 index 0000000000..fdf0049344 --- /dev/null +++ b/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108" +} diff --git a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json b/backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json similarity index 57% rename from backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json rename to backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json index e751b8fc7c..09e5b51ab9 100644 --- a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json +++ b/backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -17,6 +17,11 @@ "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" } ], "parameters": { @@ -28,8 +33,9 @@ "nullable": [ false, true, + false, false ] }, - "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c" + "hash": "148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb" } diff --git a/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json b/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json new file mode 100644 index 0000000000..d26cdf3a7d --- /dev/null +++ b/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a\n JOIN LATERAL (\n SELECT av2.id, COALESCE(v.ord, 0) AS ord\n FROM app_version av2\n LEFT JOIN unnest(a.versions) WITH ORDINALITY AS v(id, ord) ON v.id = av2.id\n WHERE av2.app_id = a.id\n ORDER BY ord DESC, av2.id DESC\n LIMIT $3 OFFSET $4\n ) page ON TRUE\n JOIN app_version av ON av.id = page.id\n LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2\n ORDER BY page.ord DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214" +} diff --git a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json b/backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json similarity index 74% rename from backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json rename to backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json index dcd3573e92..7b78a7d9b4 100644 --- a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json +++ b/backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n SELECT $1, $4, $2, $3, value, now() FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now()\n RETURNING 1 as \"one!\"", + "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value, base\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base)\n SELECT $1, $4, $2, $3, value, now(), base FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now(), base = EXCLUDED.base\n RETURNING 1 as \"one!\"", "describe": { "columns": [ { @@ -55,5 +55,5 @@ null ] }, - "hash": "46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b" + "hash": "1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b" } diff --git a/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json b/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json new file mode 100644 index 0000000000..f32e47b387 --- /dev/null +++ b/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email)\n SELECT m.workspace_id, m.typ, m.old_path, $4, $5::text\n FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ::text = ANY($2::text[])\n AND m.new_path = $3 AND m.email IS NULL\n AND NOT EXISTS (\n SELECT 1 FROM draft_move o\n WHERE o.workspace_id = m.workspace_id AND o.typ = m.typ\n AND o.old_path = m.old_path AND o.email = $5::text\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48" +} diff --git a/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json b/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json new file mode 100644 index 0000000000..c4cc310dbf --- /dev/null +++ b/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email)\n SELECT $1, t::draft_kind, $3, $4, $5 FROM unnest($2::text[]) t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c" +} diff --git a/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json b/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json new file mode 100644 index 0000000000..db2181ae3c --- /dev/null +++ b/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET new_path = $4\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND new_path = $3 AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09" +} diff --git a/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json b/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json new file mode 100644 index 0000000000..b78808a389 --- /dev/null +++ b/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft_move src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.old_path = dest.old_path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f" +} diff --git a/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json b/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json new file mode 100644 index 0000000000..c8b20a20ea --- /dev/null +++ b/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base)\n VALUES ($1, $2, $3::text, $4,\n CASE WHEN $10::bool\n THEN to_json($5::text::jsonb || jsonb_strip_nulls(jsonb_build_object(\n 'path', CASE WHEN $5::text::jsonb -> 'path' = to_jsonb($11::text)\n THEN to_jsonb($3::text) END,\n 'draft_path', CASE WHEN $5::text::jsonb -> 'draft_path' = to_jsonb($11::text)\n THEN to_jsonb($3::text) END)))\n ELSE $5::text::json\n END,\n COALESCE($8::timestamptz, now()), $9)\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = CASE\n WHEN NOT $10::bool\n OR position(chr(92) || 'u0000' in replace(draft.value::text, chr(92) || chr(92), '')) > 0\n THEN EXCLUDED.value\n ELSE to_json((to_jsonb(EXCLUDED.value) - 'path' - 'draft_path')\n || jsonb_strip_nulls(jsonb_build_object(\n 'path', to_jsonb(draft.value) -> 'path',\n 'draft_path', to_jsonb(draft.value) -> 'draft_path')))\n END,\n created_at = EXCLUDED.created_at,\n base = EXCLUDED.base\n WHERE $7::bool = true\n OR $6::timestamptz IS NULL\n OR draft.created_at <= $6::timestamptz\n RETURNING path, created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Timestamptz", + "Bool", + "Timestamptz", + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579" +} diff --git a/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json b/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json new file mode 100644 index 0000000000..ce3d615eb0 --- /dev/null +++ b/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND old_path IN ($3, $4) AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14" +} diff --git a/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json new file mode 100644 index 0000000000..6a004023a0 --- /dev/null +++ b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version\n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2\n ORDER BY flow_version.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35" +} diff --git a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json b/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json deleted file mode 100644 index 8cdb2d5907..0000000000 --- a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email)\n SELECT $2, path, typ,\n CASE WHEN typ IN ('script', 'flow')\n THEN to_json(to_jsonb(value) - 'on_behalf_of')\n ELSE value END,\n created_at, email\n FROM draft\n WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5" -} diff --git a/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json b/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json new file mode 100644 index 0000000000..4704ddd84e --- /dev/null +++ b/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND (old_path = $3 OR (new_path = $3 AND $4::text IS NULL))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5" +} diff --git a/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json b/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json new file mode 100644 index 0000000000..f882ce2986 --- /dev/null +++ b/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND new_path = $4 AND old_path <> $3 AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609" +} diff --git a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json b/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json deleted file mode 100644 index 3ed4f5a316..0000000000 --- a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "created_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "deployment_msg", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a" -} diff --git a/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json b/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json new file mode 100644 index 0000000000..c536425f1d --- /dev/null +++ b/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json @@ -0,0 +1,85 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n -- The guard's own predicate, ordered own row, then legacy, then another\n -- user's other-kind row: each needs a different remedy, and a bare\n -- LIMIT 1 would name an arbitrary one.\n (SELECT typ::text FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[])\n AND (typ <> $2 OR email = $4 OR email IS NULL)\n ORDER BY CASE WHEN email = $4 THEN 0 WHEN email IS NULL THEN 1 ELSE 2 END\n LIMIT 1) as \"at_target\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[]) AND email = $4) as \"at_target_own!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[]) AND email IS NULL) as \"at_target_legacy!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5\n AND typ = $2 AND email = $4\n AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) > 0\n ) as \"poisoned!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5\n AND typ = $2 AND email IS NULL) as \"legacy!\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "at_target", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "at_target_own!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "at_target_legacy!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "poisoned!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "legacy!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907" +} diff --git a/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json b/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json new file mode 100644 index 0000000000..2acc85fcd4 --- /dev/null +++ b/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07" +} diff --git a/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json b/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json new file mode 100644 index 0000000000..8a59831e8a --- /dev/null +++ b/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a JOIN app_version av\n ON av.id = a.versions[array_upper(a.versions, 1)] AND av.app_id = a.id\n LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee" +} diff --git a/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json b/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json new file mode 100644 index 0000000000..57b5e2abd1 --- /dev/null +++ b/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET email = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd" +} diff --git a/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json b/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json new file mode 100644 index 0000000000..f218653d46 --- /dev/null +++ b/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email, base)\n SELECT $2, path, typ,\n to_json(\n CASE WHEN typ IN ('script', 'flow') THEN clean - 'on_behalf_of' ELSE clean END\n - CASE WHEN typ = 'flow' THEN 'version_id'\n WHEN typ IN ('app', 'raw_app') THEN 'parent_version'\n ELSE '' END\n ),\n created_at, email,\n CASE WHEN typ = 'script' THEN base END\n FROM (\n SELECT d.path, d.typ, d.created_at, d.email, d.base,\n replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean\n FROM draft d\n WHERE d.workspace_id = $1 AND (d.email = $3 OR d.email IS NULL)\n ) s", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b" +} diff --git a/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json b/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json new file mode 100644 index 0000000000..926add8159 --- /dev/null +++ b/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT m.new_path FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ = $2 AND m.old_path = $3\n AND (m.email IS NULL OR m.email = $4)\n AND NOT EXISTS (\n SELECT 1 FROM draft d\n WHERE d.workspace_id = $1 AND d.typ = $2 AND d.path = $3\n AND d.email IS NOT DISTINCT FROM $4\n )\n ORDER BY m.email IS NULL\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "new_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2" +} diff --git a/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json b/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json new file mode 100644 index 0000000000..3b89a91024 --- /dev/null +++ b/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780" +} diff --git a/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json b/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json new file mode 100644 index 0000000000..27cb37b876 --- /dev/null +++ b/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT d.email IS NULL as \"legacy!\", COALESCE(u.username, p.username) as username\n FROM draft d\n LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email\n LEFT JOIN password p ON p.email = d.email AND p.super_admin = true\n WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ::text = ANY($3::text[])\n -- A row a move already brought here from the very path being renamed is this\n -- rename's own draft, not an item in its way: the record says which owners\n -- it moved (all of them for an item move, one for a draft-only move). Any\n -- other row is a second item and still collides.\n AND NOT EXISTS (\n SELECT 1 FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ::text = ANY($3::text[])\n AND m.old_path = $4 AND m.new_path = $2\n AND (m.email IS NULL OR m.email IS NOT DISTINCT FROM d.email)\n )\n ORDER BY 2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "legacy!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46" +} diff --git a/backend/migrations/20260914133558_draft_base_version.down.sql b/backend/migrations/20260914133558_draft_base_version.down.sql new file mode 100644 index 0000000000..07fc93ba30 --- /dev/null +++ b/backend/migrations/20260914133558_draft_base_version.down.sql @@ -0,0 +1 @@ +ALTER TABLE draft DROP COLUMN base; diff --git a/backend/migrations/20260914133558_draft_base_version.up.sql b/backend/migrations/20260914133558_draft_base_version.up.sql new file mode 100644 index 0000000000..835816d70c --- /dev/null +++ b/backend/migrations/20260914133558_draft_base_version.up.sql @@ -0,0 +1,14 @@ +-- The version a draft forked from, as one opaque text id whatever the kind: a +-- script hash (hex), a flow_version.id, an app_version.id. NULL for a draft that +-- was never forked from a deploy and for kinds that keep no lineage. +ALTER TABLE draft ADD COLUMN base TEXT; + +-- A U+0000 inside a `json` value makes `->>` raise 22P05; such rows keep NULL and +-- get their base on their next save. +UPDATE draft SET base = CASE typ::text + WHEN 'script' THEN value ->> 'parent_hash' + WHEN 'flow' THEN value ->> 'version_id' + ELSE value ->> 'parent_version' + END +WHERE typ::text IN ('script', 'flow', 'app', 'raw_app') + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0; diff --git a/backend/migrations/20260914154128_draft_move.down.sql b/backend/migrations/20260914154128_draft_move.down.sql new file mode 100644 index 0000000000..6fff84e4bb --- /dev/null +++ b/backend/migrations/20260914154128_draft_move.down.sql @@ -0,0 +1 @@ +DROP TABLE draft_move; diff --git a/backend/migrations/20260914154128_draft_move.up.sql b/backend/migrations/20260914154128_draft_move.up.sql new file mode 100644 index 0000000000..e68e36c5ff --- /dev/null +++ b/backend/migrations/20260914154128_draft_move.up.sql @@ -0,0 +1,17 @@ +-- Where an item's drafts went when it moved, so a draft save still addressed to the +-- old path (an editor left open across the move, a chat, the CLI) lands on the moved +-- draft instead of starting a new item there. `email` NULL records a deployed item's +-- move and applies to every user; set, it records that user's draft-only move. +CREATE TABLE draft_move ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + typ DRAFT_KIND NOT NULL, + old_path VARCHAR(255) NOT NULL, + new_path VARCHAR(255) NOT NULL, + email VARCHAR(255) +); + +CREATE INDEX draft_move_old_path_idx ON draft_move (workspace_id, typ, old_path); + +GRANT ALL ON draft_move TO windmill_user; +GRANT ALL ON draft_move TO windmill_admin; diff --git a/backend/tests/app_version_order.rs b/backend/tests/app_version_order.rs new file mode 100644 index 0000000000..e3557ad55c --- /dev/null +++ b/backend/tests/app_version_order.rs @@ -0,0 +1,193 @@ +//! The deployed order of an app's versions is the order they were appended to +//! `app.versions`, not the order of their `created_at`. +//! +//! `app_version.created_at` defaults to `now()`, which in Postgres is the +//! transaction's start time, while the append happens under the app row's lock. +//! Two deploys that overlap therefore land in one order and carry timestamps in +//! the other. The head the editor guards against, and the sequence the diff +//! picker numbers, both have to follow the array. +//! +//! Users from the `base` fixture: test-user (admin, token SECRET_TOKEN). + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const TOKEN: &str = "SECRET_TOKEN"; + +#[sqlx::test(fixtures("base"))] +async fn test_app_head_follows_the_append_order_not_the_timestamps( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let ws = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let res = client + .post(format!("{ws}/apps/create")) + .header("Authorization", format!("Bearer {TOKEN}")) + .json(&json!({ + "path": "u/test-user/order_app", + "summary": "ordered", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert!(res.status().is_success(), "{}", res.text().await?); + + let first: i64 = sqlx::query_scalar( + "SELECT versions[array_upper(versions, 1)] FROM app + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .fetch_one(&db) + .await?; + + // The overlapping deploy: appended after `first`, so it is the version that + // landed, but stamped before it, so a timestamp sort puts it underneath. + let second: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'racer', created_at - interval '1 hour', raw_app + FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + sqlx::query( + "UPDATE app SET versions = array_append(versions, $1::bigint) + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .bind(second) + .execute(&db) + .await?; + + let head: serde_json::Value = client + .get(format!( + "{ws}/apps/get_latest_version/u/test-user/order_app" + )) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + assert_eq!( + head["version"], second, + "the head is the version appended last, not the newest timestamp: {head}" + ); + + let history: Vec = client + .get(format!("{ws}/apps/history/p/u/test-user/order_app")) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + let listed: Vec = history + .iter() + .map(|v| v["version"].as_i64().unwrap()) + .collect(); + assert_eq!( + listed, + vec![second, first], + "the history lists in deployed order, so it leads with the head" + ); + + // Enough versions that a page cannot hold them, so "asked for nothing" and "asked for + // a page" are visibly different answers. + let mut appended = vec![second, first]; + for _ in 0..24 { + let extra: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'bulk', created_at, raw_app FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + sqlx::query( + "UPDATE app SET versions = array_append(versions, $1::bigint) + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .bind(extra) + .execute(&db) + .await?; + appended.insert(0, extra); + } + + let versions_at = |query: &str| { + let url = format!("{ws}/apps/history/p/u/test-user/order_app{query}"); + let client = client.clone(); + async move { + let rows: Vec = client + .get(url) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + Ok::<_, anyhow::Error>( + rows.iter() + .map(|v| v["version"].as_i64().unwrap()) + .collect::>(), + ) + } + }; + + // The deployment-history panel and the CLI read this endpoint without paging, so + // asking for no page has to keep answering with the whole history. + assert_eq!( + versions_at("").await?, + appended, + "an unpaginated request still answers whole" + ); + assert_eq!( + versions_at("?per_page=10").await?, + appended[..10], + "a page holds what was asked for, newest first" + ); + assert_eq!( + versions_at("?per_page=10&page=2").await?, + appended[10..20], + "the next page carries on where the first left off, skipping nothing" + ); + // A page past the end runs off it rather than overflowing into one. (The clamp on an + // asked-for size is pinned where it lives, in `paginate_optional`'s own test.) + assert!( + versions_at("?per_page=10&page=99999999") + .await? + .is_empty(), + "a page past the end is empty" + ); + // Naming only the page still asks for a page, rather than an offset into a listing + // sized to hold everything — which would answer nothing at all. + assert_eq!( + versions_at("?page=1").await?, + appended, + "a first page with no size named holds the history" + ); + + // A version that never entered the deployed sequence — one a restore or a fork copy + // could leave behind — still has to be reachable, after the ones that did. + let stray: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'stray', created_at, raw_app FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + let mut with_stray = appended.clone(); + with_stray.push(stray); + assert_eq!( + versions_at("").await?, + with_stray, + "a version outside app.versions is listed, and after the deployed sequence" + ); + + Ok(()) +} diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 4520cae848..f4b74525fb 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -521,10 +521,8 @@ def main(): .await .unwrap(); - assert_eq!( - res.text().await.unwrap(), - "app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")" - ); + let deployed: serde_json::Value = res.json().await.unwrap(); + assert_eq!(deployed["path"], "f/rel/root_app_renamed", "{deployed}"); let mut completed = listen_for_completed_jobs(&db).await; in_test_worker(&db, completed.next(), port).await; diff --git a/backend/tests/drafts_move_mirror.rs b/backend/tests/drafts_move_mirror.rs new file mode 100644 index 0000000000..d619381c46 --- /dev/null +++ b/backend/tests/drafts_move_mirror.rs @@ -0,0 +1,74 @@ +//! Moving a draft must carry both of its path keys. +//! +//! A draft value holds a typed path and a mirror the editors keep beside it while +//! it differs from the row's path (`path`/`draft_path`; which is which depends on +//! the kind). The loaders prefer the mirror, so a move that rewrote only the typed +//! key left the mirror naming the old location: reopening the item restored the +//! old path, and the next autosave wrote it back — undoing the move silently. +//! +//! This pins `move_draft`, including that a draft with no mirror never gains one. +//! A deploy's carry follows the keys only where they still name the old path +//! (`drafts_save_follows_move` pins that). + +use serde_json::Value; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn move_to(port: u16, from: &str, to: &str) -> anyhow::Result { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/{from}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&serde_json::json!({ "new_path": to })) + .send() + .await?; + // Asserted here so a rejected move fails as itself, rather than as a JSON + // parse error in the read-back below. + let status = resp.status(); + let body = resp.text().await?; + assert!(status.is_success(), "move {from} -> {to} failed: {body}"); + Ok(body) +} + +/// The stored draft value at `path`, read back through the API so this test needs +/// no `sqlx::query!` (which would want an offline cache entry of its own). +async fn value_at(port: u16, path: &str) -> anyhow::Result { + let body: Value = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/{path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(body + .get("value") + .cloned() + .unwrap_or_else(|| panic!("no draft at {path}: {body}"))) +} + +#[sqlx::test(fixtures("base", "drafts_move_mirror"))] +async fn test_move_carries_both_path_keys(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + move_to(port, "u/test-user/draft_mirror", "u/test-user/renamed").await?; + move_to(port, "u/test-user/draft_plain", "u/test-user/plain2").await?; + + // The mirror follows: left at `u/test-user/friendly` it would win at load and + // walk the item back there. + let moved = value_at(port, "u/test-user/renamed").await?; + assert_eq!(moved["path"], "u/test-user/renamed"); + assert_eq!(moved["draft_path"], "u/test-user/renamed"); + + // A draft that never had a mirror must not be given one. + let plain = value_at(port, "u/test-user/plain2").await?; + assert_eq!(plain["path"], "u/test-user/plain2"); + assert_eq!(plain.get("draft_path"), None, "mirror injected: {plain}"); + + Ok(()) +} diff --git a/backend/tests/drafts_move_taken.rs b/backend/tests/drafts_move_taken.rs new file mode 100644 index 0000000000..40318ca2ef --- /dev/null +++ b/backend/tests/drafts_move_taken.rs @@ -0,0 +1,250 @@ +//! A rename onto a path that already holds a draft is refused. +//! +//! Nothing deployed can sit at a rename's destination (the deploy conflicts on +//! that), but a draft can: a never-deployed item, or a draft left on an archived +//! script. Moving onto it would merge two items or strand a row, so the rename +//! itself fails, in its own transaction, and the source stays deployed. The +//! destination draft here is the deployer's own, which is the same collision. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// Hex form of script hash 7010, the way the API takes a parent hash. +const HEAD_HASH: &str = "0000000000001b62"; + +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_rename_onto_a_draft_is_refused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/mvtaken_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "rename onto a draft was not refused: {body}"); + assert!( + body.contains("already has a draft"), + "unexpected refusal: {body}" + ); + + // The whole deploy rolled back: the source is still the live head, and the + // draft at the destination is untouched. + let head: Value = client + .get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/get/p/u/test-user/mvtaken_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + assert_eq!(head["hash"], HEAD_HASH, "source was replaced: {head}"); + assert_eq!(head["archived"], false, "source was archived: {head}"); + + let draft: Value = client + .get(format!("http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/u/test-user/mvtaken_b")) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + assert_eq!( + draft["value"]["summary"], "B", + "destination draft changed: {draft}" + ); + + Ok(()) +} + +/// A legacy (ownerless) draft occupies its path too: a deploy there deletes it together +/// with the caller's own row, so a move that parks a second draft beside it would discard +/// edits the caller never saw. Only an admin can clear it, so the refusal says so. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_a_legacy_destination(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/mvtaken_b" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/mvtaken_legacy" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "move onto a legacy draft was allowed: {body}"); + assert!( + body.contains("legacy workspace draft") && body.contains("workspace admin"), + "the refusal did not point at the one remedy: {body}" + ); + + // Both rows stayed where they were: the caller's own, and the legacy one the list + // synthesizes under the caller's name. + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + let mut at = list + .iter() + .filter_map(|d| Some((d["kind"].as_str()?, d["path"].as_str()?))) + .filter(|(_, p)| p.starts_with("u/test-user/mvtaken_b") || p.ends_with("mvtaken_legacy")) + .collect::>(); + at.sort(); + assert_eq!( + at, + vec![ + ("script", "u/test-user/mvtaken_b"), + ("script", "u/test-user/mvtaken_legacy") + ], + "{list:?}" + ); + Ok(()) +} + +/// A classic app and a raw app deploy into the same table, so a draft-only move onto +/// the other kind's draft must be refused: deploying either path afterwards deletes +/// the caller's drafts of both kinds, taking the loser's item with it. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_the_other_app_kind(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/raw_app/u/test-user/mvtaken_raw" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/mvtaken_app" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "move onto a classic app draft was allowed: {body}" + ); + assert!( + body.contains("already have a draft at 'u/test-user/mvtaken_app' (app)"), + "the refusal did not name the occupying kind: {body}" + ); + + // Both drafts are untouched. + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + let mut at = list + .iter() + .filter(|d| matches!(d["kind"].as_str(), Some("app") | Some("raw_app"))) + .filter_map(|d| Some((d["kind"].as_str()?, d["path"].as_str()?))) + .collect::>(); + at.sort(); + assert_eq!( + at, + vec![ + ("app", "u/test-user/mvtaken_app"), + ("raw_app", "u/test-user/mvtaken_raw") + ], + "{list:?}" + ); + Ok(()) +} + +/// Teammates' drafts of one item share its path by design, so another user's row is no +/// obstacle — except across the app pair, where the two kinds are different items on one +/// deployed path: deploying either strands the other, and deleting the app takes both. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_another_users_other_app_kind( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let move_to = |kind: &'static str, from: &'static str, to: &'static str| async move { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/{kind}/{from}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": to })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status(), resp.text().await?)) + }; + + let (status, body) = move_to( + "app", + "u/test-user/mvtaken_app", + "u/test-user/mvtaken_theirs", + ) + .await?; + assert_eq!( + status, 400, + "a classic app was moved onto another user's raw app: {body}" + ); + assert!( + body.contains("holds another user's raw app draft"), + "the refusal did not name the occupant: {body}" + ); + + // And the other direction, where the occupant reads as the classic kind. + let (status, body) = move_to( + "raw_app", + "u/test-user/mvtaken_raw", + "u/test-user/mvtaken_app_theirs", + ) + .await?; + assert_eq!( + status, 400, + "a raw app was moved onto another user's classic app: {body}" + ); + assert!( + body.contains("holds another user's app draft"), + "the refusal did not name the occupant: {body}" + ); + + // The same-kind case is the ordinary one: two users' drafts of one raw app. + let (status, body) = move_to( + "raw_app", + "u/test-user/mvtaken_raw", + "u/test-user/mvtaken_theirs", + ) + .await?; + assert!( + status.is_success(), + "a raw app was refused beside another user's raw-app draft: {body}" + ); + Ok(()) +} diff --git a/backend/tests/drafts_nul.rs b/backend/tests/drafts_nul.rs index d7314b8fb6..2a77fe772a 100644 --- a/backend/tests/drafts_nul.rs +++ b/backend/tests/drafts_nul.rs @@ -78,3 +78,46 @@ async fn test_draft_write_strips_nul(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// The lineage the server derives for `draft.base` comes from the same value: read before +/// the sanitizer, a NUL in it costs the draft its base, which then reads as up to date +/// whatever the head is. +#[sqlx::test(fixtures("drafts_nul"))] +async fn test_draft_base_is_derived_from_sanitized_value(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/dnul-ws"); + + let resp = authed(client().post(format!( + "{base}/drafts/update/script/u/dnul-admin/poisoned_base" + ))) + .json(&json!({ + "value": { + "summary": "s", + "path": "u/dnul-admin/poisoned_base", + "content": "x", + "parent_hash": "0000000000001b62\u{0}" + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "save refused: {}", + resp.text().await.unwrap_or_default() + ); + + let stored: Option = sqlx::query_scalar( + "SELECT base FROM draft WHERE workspace_id = 'dnul-ws' AND path = 'u/dnul-admin/poisoned_base'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + stored.as_deref(), + Some("0000000000001b62"), + "the NUL cost the draft its base: {stored:?}" + ); + Ok(()) +} diff --git a/backend/tests/drafts_save_follows_move.rs b/backend/tests/drafts_save_follows_move.rs new file mode 100644 index 0000000000..afbec56c75 --- /dev/null +++ b/backend/tests/drafts_save_follows_move.rs @@ -0,0 +1,587 @@ +//! A draft save addressed to a path its item moved away from lands on the moved draft. +//! +//! A move carries every draft on the item to the new path and records where they +//! went. An editor left open across it still saves to the old path; the server puts +//! the save on the moved draft, keeps the path keys the move gave it, and names the +//! new path so the editor can follow. Without the record the save would plant a +//! phantom draft-only item at the old location. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// Hex form of script hash 7030, the way the API takes a parent hash. +const HEAD_HASH: &str = "0000000000001b76"; + +async fn own_draft_value(port: u16, path: &str) -> anyhow::Result { + let draft: Value = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/{path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(draft["value"].clone()) +} + +async fn own_draft_paths(port: u16) -> anyhow::Result> { + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(list + .iter() + .filter(|d| d["kind"] == "script") + .filter_map(|d| d["path"].as_str().map(String::from)) + .filter(|p| p.starts_with("u/test-user/follow_")) + .collect()) +} + +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_save_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + // Rename from Home: a redeploy of the deployed content at the new path that + // keeps the deployer's own draft, so the draft is carried rather than consumed. + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {}, + "skip_draft_deletion": true + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "rename failed: {}", resp.text().await?); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_b"]); + // A script draft's `path` is where deploying it lands, so it moves with the row. + assert_eq!( + own_draft_value(port, "u/test-user/follow_b").await?["path"], + "u/test-user/follow_b" + ); + + // The editor is still on the old path and writes that path back into the value. + let saved: Value = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { + "path": "u/test-user/follow_a", + "parent_hash": HEAD_HASH, + "summary": "A", + "content": "edited after the move" + } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["status"], "saved", "save refused: {saved}"); + assert_eq!( + saved["path"], "u/test-user/follow_b", + "save did not follow the row: {saved}" + ); + + // The write landed on the carried row; nothing reappeared at the old path. + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_b"]); + let draft = own_draft_value(port, "u/test-user/follow_b").await?; + assert_eq!(draft["content"], "edited after the move", "{draft}"); + assert_eq!(draft["path"], "u/test-user/follow_b", "{draft}"); + + Ok(()) +} + +/// A draft-only move rewrites both path keys. The owner's open editor still carries +/// the typed path it had, which names neither the old nor the new path; the moved +/// draft's own keys have to win, or the save walks the item back. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_save_follows_a_draft_only_move(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/draft_store" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/moved" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + let saved: Value = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/draft_store" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { + "path": "u/test-user/friendly", + "draft_path": "u/test-user/friendly", + "summary": "D", + "content": "edited after the move" + } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["path"], "u/test-user/moved", "{saved}"); + + let draft = own_draft_value(port, "u/test-user/moved").await?; + assert_eq!(draft["content"], "edited after the move", "{draft}"); + assert_eq!(draft["path"], "u/test-user/moved", "{draft}"); + assert_eq!(draft["draft_path"], "u/test-user/moved", "{draft}"); + Ok(()) +} + +/// Rename `from` to `to` the way Home does: redeploy the deployed content at the new +/// path, keeping the deployer's own draft so it is carried rather than consumed. +/// Returns the new head's hash. +async fn rename(port: u16, from_hash: &str, to: &str) -> anyhow::Result { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": to, + "parent_hash": from_hash, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {}, + "skip_draft_deletion": true + })) + .send() + .await?; + let status = resp.status(); + let hash = resp.text().await?; + assert_eq!(status, 201, "rename to {to} failed: {hash}"); + Ok(hash) +} + +/// Save the draft as an editor still bound to `url_path` would. Returns the path the +/// save landed at. +async fn save_at(port: u16, url_path: &str, content: &str) -> anyhow::Result { + let saved: Value = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/{url_path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { "path": url_path, "summary": "A", "content": content, "language": "deno" } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["status"], "saved", "save refused: {saved}"); + Ok(saved["path"].as_str().unwrap_or_default().to_string()) +} + +/// A record is kept to one hop, and a move back to the path it left ends it: both are +/// three statements whose order decides the answer. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_move_records_stay_one_hop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let b = rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + let _c = rename(port, &b, "u/test-user/follow_c").await?; + assert_eq!( + save_at(port, "u/test-user/follow_a", "after two moves").await?, + "u/test-user/follow_c", + "a save at the first path did not reach the last" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_c"]); + Ok(()) +} + +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_move_back_ends_the_record(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let b = rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + let _a = rename(port, &b, "u/test-user/follow_a").await?; + assert_eq!( + save_at(port, "u/test-user/follow_a", "after moving back").await?, + "u/test-user/follow_a", + "a save was routed off the path the item moved back to" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_a"]); + Ok(()) +} + +/// A rename carries a teammate's row too: both its path keys follow, and the version +/// it forked from does not move. A restamp there would clear their out-of-date prompt +/// and let them deploy over the mover's version believing they were current. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_teammates_draft_follows_with_its_base(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + // Read from the pool: the teammate's row is another user's, and this asserts on + // `base`, which no endpoint exposes for someone else's draft. + let row: (String, String, Option) = sqlx::query_as( + "SELECT value::jsonb ->> 'path', value::jsonb ->> 'draft_path', base + FROM draft WHERE workspace_id = 'test-workspace' AND typ = 'script' + AND email = 'test2@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!(row.0, "u/test-user/follow_b", "typed path did not follow"); + assert_eq!(row.1, "u/test-user/follow_b", "mirror did not follow"); + assert_eq!( + row.2.as_deref(), + Some(HEAD_HASH), + "the teammate's base was restamped by someone else's rename" + ); + Ok(()) +} + +/// An item move and then the owner's own move of what is left: the two records have +/// different scopes, so the owner's move has to extend the chain in its own scope or +/// a save addressed to the first path stops at the abandoned middle one. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_an_owner_move_extends_an_item_move(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + // Archiving the script at the new path leaves the carried draft as a draft-only + // item, which its owner can move through `/drafts/move`. + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_b'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_b" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_c" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + assert_eq!( + save_at(port, "u/test-user/follow_a", "after both moves").await?, + "u/test-user/follow_c", + "a save at the first path stopped at the path the owner's move left" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_c"]); + Ok(()) +} + +/// Redeploying at a path an owner's move routed away from ends that route: the live item +/// owns its path again, and its saves must not follow the draft that left. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_redeploy_at_a_routed_path_ends_the_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // `move_draft` ignores archived rows, so an archived script's draft can be moved away. + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + // Unarchiving redeploys at the same path, with the archived version as parent. + rename(port, HEAD_HASH, "u/test-user/follow_a").await?; + + assert_eq!( + save_at(port, "u/test-user/follow_a", "for the live script").await?, + "u/test-user/follow_a", + "a save for the redeployed script followed the draft that moved away" + ); + Ok(()) +} + +/// A draft written before the NUL sanitizer still has to follow a move: its path keys are +/// what a deploy of it would land on, so the carry rewrites them, sanitizing the value it +/// could not otherwise parse. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_poisoned_draft_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The teammate's row, rewritten the way a pre-sanitizer client left one: a real NUL + // escape in the content, both path keys naming the path the item is about to leave. + sqlx::query( + r#"UPDATE draft SET value = '{"path": "u/test-user/follow_a", "draft_path": "u/test-user/follow_a", + "parent_hash": "0000000000001b76", "summary": "A", "content": "a\u0000b"}' + WHERE email = 'test2@windmill.dev'"#, + ) + .execute(&db) + .await?; + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let row: (String, String, String) = sqlx::query_as( + "SELECT value::jsonb ->> 'path', value::jsonb ->> 'draft_path', value::jsonb ->> 'content' + FROM draft WHERE email = 'test2@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!(row.0, "u/test-user/follow_b", "typed path did not follow"); + assert_eq!(row.1, "u/test-user/follow_b", "mirror did not follow"); + assert_eq!(row.2, "ab", "the NUL survived the rewrite"); + Ok(()) +} + +/// A discard whose answer is lost is retried, and the retry finds nothing to delete. It +/// still has to name where the item went: the editor asking is on the path it left, and a +/// reload there lands on nothing. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_retried_discard_still_names_the_destination( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let discard = |token: &'static str| { + let client = client.clone(); + async move { + client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&json!({ "value": null })) + .send() + .await? + .json::() + .await + } + }; + + let first = discard("SECRET_TOKEN").await?; + assert_eq!(first["status"], "saved", "{first}"); + assert_eq!(first["path"], "u/test-user/follow_b", "{first}"); + + let retry = discard("SECRET_TOKEN").await?; + assert_eq!( + retry["status"], "saved", + "the retry was not idempotent: {retry}" + ); + assert_eq!(retry["path"], "u/test-user/follow_b", "{retry}"); + + // A third user has no draft on this item, so their discard deletes nothing and the + // destination is a move record and nothing else: it is answered only to a caller who + // can read that path. + let other = discard("SECRET_TOKEN_3").await?; + assert_eq!(other["status"], "saved", "{other}"); + assert_eq!( + other["path"], + Value::Null, + "a user who cannot read the destination was told where the item went: {other}" + ); + Ok(()) +} + +/// The legacy workspace-level row is carried by a rename like any other draft, and the +/// record that routes saves to it covers every caller — so discarding it from a page that +/// still names the old path has to reach it where it went. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_legacy_discard_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + r#"INSERT INTO draft (workspace_id, path, typ, value, email) + VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "summary": "legacy", "content": "x"}', NULL)"#, + ) + .execute(&db) + .await?; + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "value": null, "legacy": true })) + .send() + .await?; + assert!( + resp.status().is_success(), + "discard failed: {}", + resp.text().await? + ); + + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND email IS NULL", + ) + .fetch_one(&db) + .await?; + assert_eq!( + left, 0, + "the legacy draft survived a discard aimed at its old path" + ); + Ok(()) +} + +/// Deploying a draft that a move carried off an archived script: its parent is still the +/// version at the old path, so the deploy renames from there and carries what is left over +/// — onto the very draft being deployed. That row is this deploy's own, not an item in its +/// way, or the deploy is refused and every retry refuses again. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_moved_draft_deploys_at_its_new_path(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + // Deploy it where it now lives, still parented on the archived version it forked from. + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 2 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 201, "the moved draft could not be deployed: {body}"); + Ok(()) +} + +/// A route is only as good as the item it points at: when an unrelated item claims the +/// destination, a save still addressed to the old path must stay where it is rather than +/// land on that item's draft. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_reused_destination_ends_the_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + // Someone else's item takes the destination, and the moved draft goes with the + // deploy that consumes it. + sqlx::query("DELETE FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/follow_b'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "summary": "unrelated", + "description": "", + "content": "export function main() { return 3 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create failed: {}", resp.text().await?); + + assert_eq!( + save_at(port, "u/test-user/follow_a", "after the destination was reused").await?, + "u/test-user/follow_a", + "a save was routed onto the item that now owns the destination" + ); + Ok(()) +} diff --git a/backend/tests/fixtures/drafts_move_mirror.sql b/backend/tests/fixtures/drafts_move_mirror.sql new file mode 100644 index 0000000000..e96d985eae --- /dev/null +++ b/backend/tests/fixtures/drafts_move_mirror.sql @@ -0,0 +1,16 @@ +-- Fixture for the two-path-key move test. +-- +-- Two draft-only scripts owned by test-user, both parked at a generated storage +-- key the way a session-created draft is. One carries the `draft_path` mirror the +-- session editor writes while the typed path differs from that key; the other +-- carries no mirror at all, which is what a plain draft looks like. + +INSERT INTO draft (workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/test-user/draft_mirror', 'script', + '{"path": "u/test-user/friendly", "draft_path": "u/test-user/friendly", + "content": "x", "language": "bun", "summary": "S"}', + 'test@windmill.dev'), + ('test-workspace', 'u/test-user/draft_plain', 'script', + '{"path": "u/test-user/draft_plain", + "content": "x", "language": "bun", "summary": "S"}', + 'test@windmill.dev'); diff --git a/backend/tests/fixtures/drafts_move_taken.sql b/backend/tests/fixtures/drafts_move_taken.sql new file mode 100644 index 0000000000..5fee11b942 --- /dev/null +++ b/backend/tests/fixtures/drafts_move_taken.sql @@ -0,0 +1,37 @@ +-- Fixture for refusing a rename onto a path a draft already occupies. +-- +-- A deployed script at `u/test-user/mvtaken_a` (hash 7010 = 0x1b62), and a +-- never-deployed draft of test-user's own at `u/test-user/mvtaken_b`, the path +-- the rename will target. Nothing deployed lives at the target, so only the +-- draft can refuse the move. + +INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock, extra_perms) +VALUES ('test-workspace', 7010, 'u/test-user/mvtaken_a', + 'export function main() { return 1 }', + 'deno', 'script', 'test-user', '{}', 'A', '', '', '{}'); + +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/mvtaken_b', 'script', + '{"path": "u/test-user/mvtaken_b", "summary": "B", "content": ""}', + 'test@windmill.dev'); + +-- An ownerless (pre per-user drafts) row at a third path: a deploy there deletes it +-- together with the caller's, so it occupies the path just as an own draft does. +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/mvtaken_legacy', 'script', + '{"path": "u/test-user/mvtaken_legacy", "summary": "legacy", "content": ""}', + NULL); + +-- A draft-only classic app and a draft-only raw app of the same owner. They share +-- the `app` table, so one occupies the other's path. The third is another user's raw +-- app: a different item at its own path, not a second copy of anyone's. +INSERT INTO draft (workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/test-user/mvtaken_app', 'app', + '{"summary": "classic", "value": {}}', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_raw', 'raw_app', + '{"summary": "raw", "files": {}}', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_theirs', 'raw_app', + '{"summary": "theirs", "files": {}}', 'test2@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_app_theirs', 'app', + '{"summary": "theirs", "value": {}}', 'test2@windmill.dev'); diff --git a/backend/tests/fixtures/drafts_save_follows_move.sql b/backend/tests/fixtures/drafts_save_follows_move.sql new file mode 100644 index 0000000000..16a41feb1a --- /dev/null +++ b/backend/tests/fixtures/drafts_save_follows_move.sql @@ -0,0 +1,29 @@ +-- Fixture for saving a draft to the path its item moved away from. +-- +-- A deployed script at `u/test-user/follow_a` (hash 7030 = 0x1b76) with the +-- deployer's own draft on it. The test renames the script and then saves the +-- draft from an editor still bound to the old path. + +INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock, extra_perms) +VALUES ('test-workspace', 7030, 'u/test-user/follow_a', + 'export function main() { return 1 }', + 'deno', 'script', 'test-user', '{}', 'A', '', '', '{}'); + +INSERT INTO draft (workspace_id, path, typ, value, email, base) +VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "parent_hash": "0000000000001b76", "summary": "A", "content": "draft"}', + 'test@windmill.dev', '0000000000001b76'); + +-- A draft-only script parked at a generated storage key, its typed path elsewhere. +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/draft_store', 'script', + '{"path": "u/test-user/friendly", "draft_path": "u/test-user/friendly", "summary": "D", "content": "draft"}', + 'test@windmill.dev'); + +-- A teammate's draft on the same deployed script, forked from the same head. The +-- rename must carry it too, without touching the version it forked from. +INSERT INTO draft (workspace_id, path, typ, value, email, base) +VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "draft_path": "u/test-user/follow_a", "parent_hash": "0000000000001b76", "summary": "A", "content": "teammate draft"}', + 'test2@windmill.dev', '0000000000001b76'); diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 5ac0e0588c..c7df69879e 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -51,7 +51,10 @@ use windmill_common::{ jobs::JobPayload, schedule::Schedule, triggers::MovedNativeTrigger, - utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, + utils::{ + http_get_from_hub, not_found_if_none, paginate, paginate_optional, Pagination, + RunnableKind, StripPath, + }, }; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -747,6 +750,14 @@ async fn create_flow( .execute(&mut *tx) .await?; } + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::Flow], + &nf.path, + None, + ) + .await?; audit_log( &mut *tx, @@ -915,25 +926,36 @@ pub struct FlowVersion { pub created_at: chrono::DateTime, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + /// Who deployed this version — the diff's version picker names them so a reader + /// can tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, } async fn get_flow_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("flows:read:{}", path))?; + // Unasked-for, this listing stays whole: the history panels, the restart picker and + // the CLI all read it without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; let flows = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version - WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 - ORDER BY flow_version.created_at DESC", + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC + LIMIT $3 OFFSET $4", path, - w_id + w_id, + per_page, + offset, ) .fetch_all(&mut *tx) .await?; @@ -953,7 +975,7 @@ async fn get_latest_version( let version = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 ORDER BY flow_version.created_at DESC", @@ -1438,6 +1460,20 @@ async fn update_flow( .await?; } + if is_new_path { + // Everything left at the old path is a draft this deploy didn't consume + // — teammates' rows, and the deployer's own when the caller asked us to + // keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::Flow], + flow_path, + &nf.path, + ) + .await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs index a886d26e17..bc6c10f1ee 100644 --- a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs +++ b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs @@ -407,9 +407,18 @@ async fn test_fork_repoints_unresolvable_identities(db: Pool) -> anyho text("SELECT value->'policy'->>'on_behalf_of' FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/stranger' AND typ = 'raw_app'").await?, "u/test-user" ); + // `clone_drafts` strips a NUL escape as it copies, so the row reaches the fork + // parseable and the repoint below reaches it like any other draft's. The rule this + // guards is that the fork completes and no identity naming nobody survives it; the + // skip only ever existed because `to_jsonb` raises on a value still holding one. assert_eq!( text("SELECT CASE WHEN strpos(value::text, 'u/test-user-2') > 0 THEN 'kept' ELSE 'rewritten' END FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/nul'").await?, - "kept" + "rewritten" + ); + // And it arrives without the poison that made it a special case. + assert_eq!( + text("SELECT CASE WHEN position(chr(92) || 'u0000' in value::text) > 0 THEN 'poisoned' ELSE 'clean' END FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/nul'").await?, + "clean" ); assert_eq!( text("SELECT value->>'permissioned_as' FROM draft WHERE workspace_id = 'wm-fork-repoint' AND typ = 'trigger_websocket'").await?, diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index a4667e4876..51f2619805 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -55,7 +55,7 @@ use windmill_common::{ min_version_supports_runnable_settings_v0, RunnableSettings, RunnableSettingsTrait, }, scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, - utils::{paginate_without_limits, WarnAfterExt}, + utils::{paginate_optional, paginate_without_limits, WarnAfterExt}, worker::CLOUD_HOSTED, }; use windmill_object_store::upload_artifact_to_store; @@ -2312,6 +2312,20 @@ async fn create_script_internal<'c>( .await?; } + if p_path != &ns.path { + // Everything left at the old path is a draft this deploy didn't + // consume — teammates' rows, and the deployer's own when the caller + // asked us to keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::Script], + p_path, + &ns.path, + ) + .await?; + } + sqlx::query!( "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", ns.path, @@ -2389,19 +2403,32 @@ async fn create_script_internal<'c>( tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } - } else if !skip_draft_deletion { - // See the matching branch above — only wipe the deployer's own - // draft (plus the legacy NULL-email row). - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script' \ - AND (email = $3 OR email IS NULL)", - ns.path, - &w_id, - &authed.email, - ) - .execute(&mut *tx) - .await?; + } else { + if !skip_draft_deletion { + // See the matching branch above — only wipe the deployer's own + // draft (plus the legacy NULL-email row). + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script' \ + AND (email = $3 OR email IS NULL)", + ns.path, + &w_id, + &authed.email, + ) + .execute(&mut *tx) + .await?; + } } + // Every deploy, not only a new script: an archived script's draft can be moved away + // (`move_draft` ignores archived rows), and unarchiving redeploys at the same path, + // where a route left behind would send the live script's saves to the moved draft. + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::Script], + &ns.path, + p_path_opt.as_deref(), + ) + .await?; if p_hashes.is_some() && !p_hashes.unwrap().is_empty() { audit_log( &mut *tx, @@ -3061,17 +3088,24 @@ async fn get_script_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + // Unasked-for, this listing stays whole: the deployment-history panels, the restart + // picker and the CLI all read it without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by s.created_at DESC", + ORDER by s.created_at DESC + LIMIT $3 OFFSET $4", w_id, path, + per_page, + offset, ) .fetch_all(&mut *tx) .await?; @@ -3083,6 +3117,7 @@ async fn get_script_history( script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, created_at: Some(row.created_at), + created_by: Some(row.created_by), }) .collect(); return Ok(Json(result)); @@ -3097,7 +3132,7 @@ async fn get_latest_version( check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let row_o = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC LIMIT 1", @@ -3113,6 +3148,7 @@ async fn get_latest_version( script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, created_at: Some(row.created_at), + created_by: Some(row.created_by), }; return Ok(Json(Some(result))); } else { diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 879babff4f..88839633cd 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -7961,14 +7961,32 @@ async fn clone_drafts( // filtered like `clone_scripts`: the address the draft still carries re-derives the // clone's own principal at deploy time, which is the more accurate answer of the two. sqlx::query!( - "INSERT INTO draft (workspace_id, path, typ, value, created_at, email) + // A script hash is content-addressed and copied as-is, so a script draft's base + // still names a version the clone has. `clone_flows` / `clone_apps` mint new ids, + // so those drafts arrive with no base (staleness falls back to the timestamps), + // lineage field included, or the next autosave would re-derive the source id. + // + // `clean` is `strip_json_nul`'s parity rule in SQL, so a pre-sanitizer U+0000 + // escape cannot abort the clone on `to_jsonb` or arrive with its principal + // unstripped: escaped backslashes park on chr(1) (lossless, a `json` value's text + // cannot hold a raw control byte) so only a real NUL is removed, and chr(92) + // spells the backslash so no escape sequence reaches this source file. + r#"INSERT INTO draft (workspace_id, path, typ, value, created_at, email, base) SELECT $2, path, typ, - CASE WHEN typ IN ('script', 'flow') - THEN to_json(to_jsonb(value) - 'on_behalf_of') - ELSE value END, - created_at, email - FROM draft - WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", + to_json( + CASE WHEN typ IN ('script', 'flow') THEN clean - 'on_behalf_of' ELSE clean END + - CASE WHEN typ = 'flow' THEN 'version_id' + WHEN typ IN ('app', 'raw_app') THEN 'parent_version' + ELSE '' END + ), + created_at, email, + CASE WHEN typ = 'script' THEN base END + FROM ( + SELECT d.path, d.typ, d.created_at, d.email, d.base, + replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean + FROM draft d + WHERE d.workspace_id = $1 AND (d.email = $3 OR d.email IS NULL) + ) s"#, source_workspace_id, target_workspace_id, authed_email, diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 45f82227f5..62ab812a8f 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -382,6 +382,13 @@ pub(crate) async fn change_workspace_id( ) .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE draft_move SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; info!("Updating favorite table"); sqlx::query!( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d37c647db2..ee5d082d89 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10392,8 +10392,49 @@ paths: current_timestamp: type: string format: date-time + path: + type: string + description: "`saved` only, upsert or delete: where the write landed. Differs from the URL path when the item had moved away from it; the editor follows it there. Absent when a delete found nothing to remove and the caller cannot read the path it moved to." required: [status, current_timestamp] + /w/{workspace}/drafts/move/{kind}/{path}: + post: + summary: move the current user's draft-only item to another path + description: Relocates the authed user's own draft row, along with both path keys inside its value (the typed path and the mirror the editors keep beside it). Only for draft-only items — a deployed item must be moved through its own deploy endpoint, which carries every draft with it. Restricted to script, flow, app and raw_app; any other kind is rejected with 400, because only these keep their deploy target where this endpoint rewrites it. + operationId: moveDraft + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + description: script, flow, app or raw_app only. + schema: + type: string + enum: [script, flow, app, raw_app] + - $ref: "#/components/parameters/ScriptPath" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + new_path: + type: string + summary: + type: string + description: Also restate the draft's summary. + required: [new_path] + responses: + "200": + description: move result + content: + text/plain: + schema: + type: string + /w/{workspace}/drafts/migrate_legacy/{kind}/{path}: post: summary: resolve a legacy (workspace-level) draft (admin only) @@ -10904,6 +10945,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: script history @@ -12309,6 +12352,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" tags: - flow responses: @@ -14187,6 +14232,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: app history @@ -14454,9 +14501,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/create_raw_source: post: @@ -14596,9 +14643,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/update_raw/{path}: post: @@ -14652,9 +14699,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/custom_path_exists/{custom_path}: get: @@ -27702,6 +27749,13 @@ components: draft_saved_at: type: string format: date-time + draft_base: + type: string + description: | + The deployed version the draft forked from, as text whatever the + kind (script hash, flow version id, app version id). Compare to the + deployed head to tell a draft that is behind. Absent when there is + no draft or it was never forked from a deploy. no_deployed: type: boolean draft: @@ -29597,6 +29651,11 @@ components: type: string deployment_msg: type: string + created_at: + type: string + format: date-time + created_by: + type: string required: - script_hash @@ -35493,6 +35552,20 @@ components: - raw_app + AppDeployed: + type: object + description: "What a deploy of an existing app answers with. `version` is the one this call wrote, which is what an editor pins as the fork base of the draft it starts next: reading the head back afterwards cannot tell it from a deploy that landed beside it. A metadata-only update writes none and reports the head it kept." + properties: + path: + type: string + description: Where the app now lives, which differs from the request path on a rename. + version: + type: integer + format: int64 + required: + - path + - version + AppHistory: type: object properties: @@ -35500,6 +35573,11 @@ components: type: integer deployment_msg: type: string + created_at: + type: string + format: date-time + created_by: + type: string required: - version @@ -35569,6 +35647,8 @@ components: format: date-time deployment_msg: type: string + created_by: + type: string required: - id - created_at diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 918fce53bf..432af1bef0 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -69,8 +69,8 @@ use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - strip_json_nul, Pagination, RunnableKind, StripPath, + http_get_from_hub, not_found_if_none, paginate, paginate_optional, + query_elems_from_hub, require_admin, strip_json_nul, Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -274,6 +274,12 @@ pub struct AppHistory { pub version: i64, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + /// Who deployed this version, and when — the diff's version picker names them so + /// a reader can tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub created_at: Option>, } #[derive(Deserialize)] @@ -567,6 +573,17 @@ pub struct CreateApp { pub skip_draft_deletion: Option, } +/// What a deploy of an existing app answers with. `version` is the one this call wrote, +/// which is what an editor pins as the fork base of the draft it starts next: reading the +/// head back afterwards cannot tell it from a deploy that landed beside it. A +/// metadata-only update writes none and reports the head it kept. +#[derive(Serialize)] +pub struct AppDeployed { + /// Where the app now lives, which differs from the request path on a rename. + pub path: String, + pub version: i64, +} + #[derive(Serialize, Deserialize)] pub struct EditApp { pub path: Option, @@ -1220,18 +1237,42 @@ async fn get_app_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", &path))?; + // Unasked-for, this listing stays whole: the deployment-history panel reads it + // without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; + // Newest first in deployed order, which is a version's position in `app.versions` and + // not its `created_at`: the latter is the deploying transaction's start time, so two + // that overlap can carry it in the opposite order from the one they landed in. A row + // outside the array never sat in that sequence, so it sorts after the ones that did. + // Paging happens before the metadata joins, so a page costs its own rows. let query_result = sqlx::query!( - "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg - FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg, + av.created_by as created_by, av.created_at as created_at + FROM app a + JOIN LATERAL ( + SELECT av2.id, COALESCE(v.ord, 0) AS ord + FROM app_version av2 + LEFT JOIN unnest(a.versions) WITH ORDINALITY AS v(id, ord) ON v.id = av2.id + WHERE av2.app_id = a.id + ORDER BY ord DESC, av2.id DESC + LIMIT $3 OFFSET $4 + ) page ON TRUE + JOIN app_version av ON av.id = page.id + LEFT JOIN deployment_metadata dm ON av.id = dm.app_version WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY created_at DESC", + ORDER BY page.ord DESC", w_id, path, - ).fetch_all(&mut *tx).await?; + per_page, + offset, + ) + .fetch_all(&mut *tx) + .await?; tx.commit().await?; let result: Vec = query_result @@ -1240,6 +1281,8 @@ async fn get_app_history( app_id: row.app_id, version: row.version_id, deployment_msg: row.deployment_msg, + created_by: Some(row.created_by), + created_at: Some(row.created_at), }) .collect(); return Ok(Json(result)); @@ -1253,14 +1296,22 @@ async fn get_latest_version( let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; + // The head is the tail of `app.versions` — the version the runtime serves. Deploys + // append to it under the app row's lock, whereas `app_version.created_at` is the + // deploying transaction's start time, so two that overlap can carry it in either + // order and the newest timestamp is then not the one that landed last. let row = sqlx::query!( - "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg - FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version - WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY created_at DESC", + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg, + av.created_by as created_by, av.created_at as created_at + FROM app a JOIN app_version av + ON av.id = a.versions[array_upper(a.versions, 1)] AND av.app_id = a.id + LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + WHERE a.workspace_id = $1 AND a.path = $2", w_id, path, - ).fetch_optional(&mut *tx).await?; + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; if let Some(row) = row { @@ -1268,6 +1319,8 @@ async fn get_latest_version( app_id: row.app_id, version: row.version_id, deployment_msg: row.deployment_msg, + created_by: Some(row.created_by), + created_at: Some(row.created_at), }; return Ok(Json(Some(result))); @@ -2591,6 +2644,14 @@ async fn create_app_internal<'a>( .execute(&mut *tx) .await?; } + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::App, UserDraftItemKind::RawApp], + &app.path, + None, + ) + .await?; let id = sqlx::query_scalar!( "INSERT INTO app (workspace_id, path, summary, policy, versions, custom_path, labels) @@ -2947,7 +3008,7 @@ async fn update_app( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -2986,7 +3047,7 @@ async fn update_app( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } /// Deploy a raw app from its sources, compiling them on a worker. `update_raw` @@ -3000,7 +3061,7 @@ async fn update_app_raw_source( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3098,7 +3159,7 @@ async fn update_app_raw_source( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } /// Whether the caller may create an app at `path` — asked of the database rather @@ -3324,7 +3385,7 @@ async fn update_app_raw<'a>( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, multipart: Multipart, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3372,7 +3433,7 @@ async fn update_app_raw<'a>( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } // async fn create_app_internal<'a>( // authed: ApiAuthed, @@ -3749,6 +3810,19 @@ async fn update_app_internal<'a>( .execute(&mut *tx) .await?; } + if npath != path { + // Everything left at the old path is a draft this deploy didn't consume + // — teammates' rows, and the deployer's own when the caller asked us to + // keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::App, UserDraftItemKind::RawApp], + path, + &npath, + ) + .await?; + } audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index e5928f761e..8bcbb56384 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -19,7 +19,7 @@ use windmill_common::{ error::{Error, Result}, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, - utils::strip_json_nul, + utils::{check_proper_path, strip_json_nul}, variables::{build_crypt, encrypt}, }; @@ -29,6 +29,7 @@ pub fn workspaced_service() -> Router { .route("/get/{kind}/{*path}", get(get_draft_for_user)) .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route("/move/{kind}/{*path}", post(move_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -320,12 +321,71 @@ pub struct SaveDraftResponse { /// On `saved`: when the change was applied (client remembers it as the /// next `last_sync`). On `conflict`: the existing row's `created_at`. pub current_timestamp: chrono::DateTime, + /// `saved` only: where the write landed. Differs from the URL path when the item + /// had moved away from it; the editor follows it there. Absent when a delete found + /// nothing to remove and the caller cannot read the path it moved to. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, } -/// Apply the current user's draft at (workspace, kind, path): non-null `value` -/// upserts, `null` (or omitted) deletes. Either way, when the existing row is -/// newer than `last_sync` (and `force` is false) the op is skipped and the -/// response is `status = conflict` + the server's current timestamp. +/// The version a draft forked from, as the editors write it into `draft.value`. +/// Each kind names it differently and only one is ever set. +#[derive(Deserialize)] +struct DraftBaseVersion { + /// Scripts: hex-encoded script hash. + #[serde(default)] + parent_hash: Option, + /// Flows: `flow_version.id`. + #[serde(default)] + version_id: Option, + /// Apps / raw apps: `app_version.id`. + #[serde(default)] + parent_version: Option, +} + +impl DraftBaseVersion { + /// The base as the `draft.base` column stores it: one opaque text id whatever + /// the kind, so a reader compares it to the head without knowing the kind's + /// own field name or type. + fn as_text(&self, kind: UserDraftItemKind) -> Option { + use UserDraftItemKind::*; + match kind { + Script => self.parent_hash.clone(), + Flow => self.version_id.map(|v| v.to_string()), + _ => self.parent_version.map(|v| v.to_string()), + } + } +} + +/// The version this draft forked from, or `None` when it has none — a kind +/// that keeps no lineage, a malformed payload, or a draft that was never forked +/// from a deploy. Pure: no queries. +fn draft_lineage(kind: UserDraftItemKind, value: &str) -> Option { + use UserDraftItemKind::*; + if !matches!(kind, Script | Flow | App | RawApp) { + return None; + } + let base = serde_json::from_str::(value).ok()?; + let has_base = match kind { + Script => base + .parent_hash + .as_deref() + .and_then(|h| windmill_common::scripts::to_i64(h).ok()) + .is_some(), + Flow => base.version_id.is_some(), + _ => base.parent_version.is_some(), + }; + has_base.then_some(base) +} + +/// Apply the current user's draft: non-null `value` upserts, `null` (or +/// omitted) deletes. Either way, when the existing row is newer than +/// `last_sync` (and `force` is false) the op is skipped and the response is +/// `status = conflict` + the server's current timestamp. +/// +/// A save addressed to a path its item moved away from lands where the move took +/// the drafts (`draft_move`), unless the caller still has a draft of their own at +/// that path. The response names where it landed. async fn update_draft( authed: ApiAuthed, Extension(db): Extension, @@ -334,7 +394,7 @@ async fn update_draft( Json(req): Json, ) -> Result> { let email = &authed.email; - let path = path.to_path(); + let url_path = path.to_path(); // Saving a draft requires write permission on the underlying path. Deleting // (discarding) one's OWN draft does not: the email-scoped row belongs to the // authed user, so they can always discard it even after losing write access @@ -343,11 +403,68 @@ async fn update_draft( // touch the caller's own row. Legacy (NULL-email) rows aren't owned by anyone // — they keep the write gate. let is_own_discard = req.value.is_none() && !req.legacy; + // `legacy` targets the workspace-level row and is delete-only: an upsert writes the + // caller's own row whatever it says. Every read of that rule goes through this. + let legacy_delete = req.value.is_none() && req.legacy; + + // Whose row this write is for: the caller's, or the workspace-level one on a legacy + // DELETE (`legacy` is delete-only, so an upsert is the caller's own row either way). + // It picks both the record that applies — an item's move (`email IS NULL`) covers the + // legacy row too, since the same rename carried it — and the draft whose presence + // means this path is still the write's own. + let owner: Option<&str> = (!legacy_delete).then_some(email.as_str()); + // The caller's own draft-only move outranks the move of the deployed item. Only the + // kinds whose value carries a deploy target are ever recorded as moved, so for the + // rest this would be a guaranteed-empty query on the autosave hot path. + let moved_to = match kind.typed_path_field() { + None => None, + Some(_) => { + sqlx::query_scalar!( + r#"SELECT m.new_path FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ = $2 AND m.old_path = $3 + AND (m.email IS NULL OR m.email = $4) + AND NOT EXISTS ( + SELECT 1 FROM draft d + WHERE d.workspace_id = $1 AND d.typ = $2 AND d.path = $3 + AND d.email IS NOT DISTINCT FROM $4 + ) + ORDER BY m.email IS NULL + LIMIT 1"#, + &w_id, + kind as UserDraftItemKind, + url_path, + owner, + ) + .fetch_optional(&db) + .await? + } + }; + let path: &str = moved_to.as_deref().unwrap_or(url_path); + + // Everything past here writes, so the gate applies from here on. Answered + // without the path when the item moved: it may have gone somewhere the caller + // cannot see. if !is_own_discard { - require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + match require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await { + // Naming the move is for whoever was editing the item: it tells them why a + // save they were already making stopped landing. Someone who cannot read the + // path they addressed gets the plain denial, or the wording itself would + // answer whether an item was moved away from a path they only guessed at. + Err(Error::NotAuthorized(e)) if moved_to.is_some() => { + return Err(Error::NotAuthorized( + match require_can_read_path(&authed, &user_db, &w_id, kind, url_path).await { + Ok(()) => { + "this draft's item was moved to a path you cannot write".to_string() + } + Err(_) => e, + }, + )); + } + other => other?, + } } - let applied_at = if let Some(value) = &req.value { + let applied = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext // (see `encrypt_secret_variable_value`). let serialized = if kind == UserDraftItemKind::Variable { @@ -359,19 +476,49 @@ async fn update_draft( // escape and later make any `->>`/`to_jsonb` extraction raise `22P05`. // Strip it here so a NUL never reaches the column. let serialized = strip_json_nul(&serialized); + // `base` is derived here from the value's per-kind field rather than sent + // by the client, so every writer (editors, chat, CLI) fills it the same way. + // Read from the sanitized text, which is what the value column gets: a NUL in the + // lineage field otherwise costs the draft its base (the hash no longer parses), + // leaving a draft that reads as up to date whatever the head is. + let base = draft_lineage(kind, serialized.as_ref()).and_then(|l| l.as_text(kind)); // Upsert. The conflict check rides on the DO UPDATE WHERE clause — // when the row is newer than `last_sync`, RETURNING yields nothing. // `created_at` defaults to `now()` but the migration overrides it ($8) // so a migrated draft keeps its original age instead of jumping to top. - sqlx::query_scalar!( - r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - VALUES ($1, $2, $3, $4, $5::text::json, COALESCE($8::timestamptz, now())) + // + // A moved save ($10) carries the path keys its editor had before the move. + // One naming the path it addressed ($11) follows to where it landed; a draft + // already there keeps the keys the move gave it. A pre-sanitizer NUL escape + // in that draft makes `to_jsonb` raise, so it takes the incoming keys. + sqlx::query!( + r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base) + VALUES ($1, $2, $3::text, $4, + CASE WHEN $10::bool + THEN to_json($5::text::jsonb || jsonb_strip_nulls(jsonb_build_object( + 'path', CASE WHEN $5::text::jsonb -> 'path' = to_jsonb($11::text) + THEN to_jsonb($3::text) END, + 'draft_path', CASE WHEN $5::text::jsonb -> 'draft_path' = to_jsonb($11::text) + THEN to_jsonb($3::text) END))) + ELSE $5::text::json + END, + COALESCE($8::timestamptz, now()), $9) ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at + DO UPDATE SET value = CASE + WHEN NOT $10::bool + OR position(chr(92) || 'u0000' in replace(draft.value::text, chr(92) || chr(92), '')) > 0 + THEN EXCLUDED.value + ELSE to_json((to_jsonb(EXCLUDED.value) - 'path' - 'draft_path') + || jsonb_strip_nulls(jsonb_build_object( + 'path', to_jsonb(draft.value) -> 'path', + 'draft_path', to_jsonb(draft.value) -> 'draft_path'))) + END, + created_at = EXCLUDED.created_at, + base = EXCLUDED.base WHERE $7::bool = true OR $6::timestamptz IS NULL OR draft.created_at <= $6::timestamptz - RETURNING created_at"#, + RETURNING path, created_at"#, &w_id, email, path, @@ -380,9 +527,13 @@ async fn update_draft( req.last_sync, req.force, req.created_at, + base.as_deref(), + moved_to.is_some(), + url_path, ) .fetch_optional(&db) .await? + .map(|r| (r.created_at, Some(r.path))) } else { // Delete, same conflict rule in the WHERE clause. Returns NULL when // the row was too new (conflict) OR already absent (idempotent) — @@ -403,16 +554,20 @@ async fn update_draft( kind as UserDraftItemKind, req.last_sync, req.force, - req.legacy, + legacy_delete, ) .fetch_optional(&db) .await? + // Named for the same reason an upsert is: the editor that discarded is still on + // the path the item left, and reloading there would land on nothing. + .map(|ts| (ts, moved_to.clone())) }; - if let Some(ts) = applied_at { + if let Some((ts, path)) = applied { return Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Saved, current_timestamp: ts, + path, })); } @@ -428,7 +583,7 @@ async fn update_draft( email, path, kind as UserDraftItemKind, - req.legacy, + legacy_delete, ) .fetch_optional(&db) .await?; @@ -437,20 +592,307 @@ async fn update_draft( Some(ts) => Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Conflict, current_timestamp: ts, + path: None, })), // Delete + nothing-was-there ⇒ report success with server's NOW(). None => { let now = sqlx::query_scalar!(r#"SELECT now() as "now!""#) .fetch_one(&db) .await?; + // A retry of a routed discard whose answer was lost lands here: the row is + // gone but the editor is still on the path the item left, so it needs the + // destination as much as the first attempt did. Unlike the arm above there is + // no deleted row proving the caller ever held that draft, and an own discard + // is not gated, so this names a path to someone who may have none of it: the + // read gate, which is what keeps a path from being disclosed elsewhere in this + // module. Without it, discarding at a guessed path reads `draft_move`. + let disclosed = match moved_to { + Some(dest) => { + match require_can_read_path(&authed, &user_db, &w_id, kind, &dest).await { + Ok(()) => Some(dest), + Err(Error::NotFound(_)) + | Err(Error::NotAuthorized(_)) + | Err(Error::BadRequest(_)) => None, + Err(e) => return Err(e), + } + } + None => None, + }; Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Saved, current_timestamp: now, + path: disclosed, })) } } } +#[derive(Deserialize)] +pub struct MoveDraftRequest { + pub new_path: String, + /// Also restate the draft's summary, so the same drawer that renames a + /// deployed item can retitle a draft-only one. + #[serde(default)] + pub summary: Option, +} + +/// Relocate the authed user's own DRAFT-ONLY item. Such an item is nothing but +/// its draft row, so moving it is a rewrite of that row's path plus both path +/// keys inside its value — there is no deployed row, schedule or trigger to +/// cascade to. +/// +/// The owner's own open editor follows: its next save, still addressed to the old +/// path, lands at the new one through the move record, and it is told where. +/// +/// Scoped to the caller's own row on purpose: two users can each have a draft +/// at the same never-deployed path, and those are two separate items. +/// +/// A DEPLOYED item must move through its own deploy endpoint instead, which +/// cascades everything that references the path and carries every draft along. +async fn move_draft( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Json(req): Json, +) -> Result { + let path = path.to_path(); + let new_path = req.new_path.as_str(); + // Only the full-page editor kinds, which is exactly the set that has a typed + // path to rewrite. Reading the movable set off the same mapping the rewrite + // uses keeps them from drifting apart: a resource, a variable or a trigger + // keeps its deploy target in `value.path` with no editor to stage a rename, + // so moving one would leave the real target naming the old location and the + // next deploy would recreate it where it came from. + let (Some(typed_field), Some(mirror_field)) = + (kind.typed_path_field(), kind.mirror_path_field()) + else { + return Err(Error::BadRequest(format!( + "moving a {kind:?} draft is not supported — only scripts, flows and apps" + ))); + }; + // Validate before authorizing: `require_can_write_path` is not a format check + // (an admin returns immediately, and a user returns early inside their own + // namespace), so without this a malformed path is stored as-is, and an over-long + // or NUL-bearing one reaches Postgres as a raw server error. The source is a URL + // segment and arrives decoded, so it needs the same check as the destination. + check_proper_path(path)?; + check_proper_path(new_path)?; + // A summary-only edit is a legitimate use of this endpoint: the drawer edits + // both fields, and for a draft-only script the path it posts back is the row + // path unchanged (`list_scripts` only reports `draft_path` when it differs). + // Returning early on the path alone would drop the new summary silently. + if new_path == path && req.summary.is_none() { + return Ok("unchanged".to_string()); + } + require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + if new_path != path { + require_can_write_path(&authed, &db, &user_db, &w_id, kind, new_path).await?; + } + + if let Some(table) = kind.deployed_table() { + // `table` is from the closed `deployed_table()` enum, never user input. + // Archived and soft-deleted rows keep sitting at their path — a script + // move archives its parent in place — so an existence check that counted + // them would refuse a move away from, or into, a path nothing occupies. + // `create_script_internal` resolves its own path clashes the same way. + let archived_filter = if table == "script" { + " AND NOT archived AND NOT deleted" + } else { + "" + }; + let query = format!( + "SELECT 1 FROM {table} WHERE path = $1 AND workspace_id = $2{archived_filter} LIMIT 1" + ); + let mut tx = user_db.clone().begin(&authed).await?; + let deployed_at_old = sqlx::query_scalar::<_, i32>(&query) + .bind(path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + let deployed_at_new = sqlx::query_scalar::<_, i32>(&query) + .bind(new_path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if deployed_at_old.is_some() { + return Err(Error::BadRequest(format!( + "'{path}' is deployed — move it from its editor so schedules and triggers follow" + ))); + } + if deployed_at_new.is_some() { + return Err(Error::BadRequest(format!( + "'{new_path}' already has a deployed item — moving there would turn this into a draft on top of it" + ))); + } + } + + // A NUL in the summary reaches Postgres as a raw byte and fails the statement with + // an encoding error, so it is dropped here the way `strip_json_nul` drops one from a + // draft value. `json` cannot store it either. + let summary = req.summary.as_ref().map(|s| s.replace('\0', "")); + + // A classic app and a raw app share the `app` table, so a draft of either kind + // occupies the destination for both: deploying there deletes the caller's drafts + // of both kinds, taking the item that lost the collision with it. + let collision_typs: Vec<&str> = match kind { + UserDraftItemKind::App | UserDraftItemKind::RawApp => vec![ + UserDraftItemKind::App.as_str(), + UserDraftItemKind::RawApp.as_str(), + ], + _ => vec![kind.as_str()], + }; + + // One transaction with the move record, so a save addressed to the old path + // never sees the row gone without knowing where it went. + let mut tx = db.begin().await?; + let moved = sqlx::query_scalar!( + r#"UPDATE draft + SET path = $3, + -- Both path keys, not just the typed one: the editors mirror the + -- typed path into the other while it differs from the row's path, + -- and the loaders prefer the mirror — left naming the old location + -- it un-does this move on the next save. `create_missing = false` + -- on both, so a draft carrying only one keeps only one. + value = to_json( + jsonb_set( + jsonb_set( + CASE WHEN $7::text IS NULL THEN to_jsonb(value) + ELSE jsonb_set(to_jsonb(value), ARRAY['summary'], to_jsonb($7::text)) + END, + ARRAY[$5::text], to_jsonb($3::text), false + ), + ARRAY[$8::text], to_jsonb($3::text), false + ) + ) + WHERE workspace_id = $1 + AND path = $2 + AND typ = $4 + AND email = $6 + -- A pre-sanitizer NUL escape makes `to_jsonb` raise 22P05. Excluded + -- here so the statement can't 500; reported below instead. Unlike the + -- passive carry, rewriting the value IS this operation, so skipping it + -- silently would move the row and leave its typed path stale. + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0 + -- Skipped on a summary-only edit, where the "target" row is this + -- row and the guard would refuse the update against itself. + AND ($2 = $3 OR NOT EXISTS ( + SELECT 1 FROM draft o + WHERE o.workspace_id = $1 AND o.path = $3 AND o.typ::text = ANY($9::text[]) + -- Of this kind only the caller's own row and the legacy one collide: + -- teammates' drafts of one item share its path by design, but a deploy + -- there wipes those two together, so a second would discard edits the + -- caller never saw. The other app kind is a different item on the same + -- deployed path, so it collides whoever owns it. + AND (o.typ <> $4 OR o.email = $6 OR o.email IS NULL) + )) + RETURNING id"#, + &w_id, + path, + new_path, + kind as UserDraftItemKind, + typed_field, + &authed.email, + summary, + mirror_field, + &collision_typs as &[&str], + ) + .fetch_optional(&mut *tx) + .await?; + if moved.is_some() && new_path != path { + windmill_common::user_drafts::record_draft_move( + &mut tx, + &w_id, + &[kind], + path, + new_path, + Some(&authed.email), + ) + .await?; + } + tx.commit().await?; + + if moved.is_none() { + let row = sqlx::query!( + r#"SELECT + -- The guard's own predicate, ordered own row, then legacy, then another + -- user's other-kind row: each needs a different remedy, and a bare + -- LIMIT 1 would name an arbitrary one. + (SELECT typ::text FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) + AND (typ <> $2 OR email = $4 OR email IS NULL) + ORDER BY CASE WHEN email = $4 THEN 0 WHEN email IS NULL THEN 1 ELSE 2 END + LIMIT 1) as "at_target", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) AND email = $4) as "at_target_own!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) AND email IS NULL) as "at_target_legacy!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5 + AND typ = $2 AND email = $4 + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) > 0 + ) as "poisoned!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5 + AND typ = $2 AND email IS NULL) as "legacy!" "#, + &w_id, + kind as UserDraftItemKind, + new_path, + &authed.email, + path, + &collision_typs as &[&str], + ) + .fetch_one(&db) + .await?; + return Err(Error::BadRequest(if row.poisoned { + // This endpoint also serves a summary-only edit, so name the operation + // the caller actually asked for rather than always saying "moved". + let attempted = if new_path == path { "updated" } else { "moved" }; + format!( + "'{path}' contains a NUL character and predates the sanitizer, so it cannot be \ + {attempted}. Reopen it, re-save to rewrite it cleanly, then retry." + ) + } else if row.legacy { + // The home list synthesizes a draft-only row for the legacy draft with the + // caller's own name on it, so this is reachable from the row menu. Only an + // admin can claim or discard that row, and only from the drafts page. + format!( + "'{path}' is a legacy workspace draft with no owner, so it cannot be moved. \ + A workspace admin can claim or discard it on the Review & deploy drafts page." + ) + } else if let Some(occupant) = row.at_target.filter(|_| new_path != path) { + // Naming the kind matters for the app pair: a classic-app draft refusing a + // raw-app move is invisible in the raw-app list the caller is looking at. + let occupant = occupant.replace('_', " "); + if row.at_target_own { + format!("You already have a draft at '{new_path}' ({occupant})") + } else if row.at_target_legacy { + // An ownerless row the caller cannot clear themselves, so send them to + // the one place it can be resolved rather than to "discard your draft". + format!( + "A legacy workspace draft with no owner is already at '{new_path}' \ + ({occupant}). A workspace admin can claim or discard it on the Review & \ + deploy drafts page." + ) + } else { + // The other app kind, owned by someone else: one deployed path cannot hold + // both, so this is the other item's path, not a teammate's copy of this one. + format!( + "'{new_path}' holds another user's {occupant} draft, and an app and a raw \ + app cannot share a path. Pick another path, or ask them to move or \ + discard theirs." + ) + } + } else { + format!("You have no draft at '{path}'") + })); + } + + if new_path == path { + return Ok(format!("updated draft {path}")); + } + Ok(format!("moved draft {path} to {new_path}")) +} + #[derive(Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub enum MigrateLegacyDraftAction { @@ -504,12 +946,12 @@ async fn migrate_legacy_draft( r#"WITH legacy AS ( DELETE FROM draft WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL - RETURNING value + RETURNING value, base ) - INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - SELECT $1, $4, $2, $3, value, now() FROM legacy + INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base) + SELECT $1, $4, $2, $3, value, now(), base FROM legacy ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = now() + DO UPDATE SET value = EXCLUDED.value, created_at = now(), base = EXCLUDED.base RETURNING 1 as "one!""#, &w_id, path, @@ -767,8 +1209,8 @@ async fn require_can_write_path( /// Resolves to `Ok(())` if `authed` can read at `path`. Three layers: /// 1. admin → always. -/// 2. Path-prefix match against own `u/{username}` or any folder in -/// `authed.folders` (the precomputed read set, with groups + direct +/// 2. Path-prefix match against own `u/{username}`, a group in `authed.groups`, or any +/// folder in `authed.folders` (the precomputed read set, with groups + direct /// grants already factored in). /// 3. RLS-aware `SELECT 1` against the backing table — covers item-level /// extra_perms grants that bypass folder/owner checks. @@ -793,6 +1235,10 @@ async fn require_can_read_path( if parts.len() >= 2 { match parts[0] { "u" if parts[1] == authed.username => return Ok(()), + // As `require_can_write_path` and the `see_member` RLS policy read it: a + // draft-only `g/` path has no row for the probe below to fall back on, so + // without this a member cannot see a draft their group owns. + "g" if authed.groups.iter().any(|g| g == parts[1]) => return Ok(()), "f" => { let folder = parts[1]; if authed.folders.iter().any(|(name, _, _)| name == folder) { diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index aed9b5c5de..ebd0ce363f 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -160,6 +160,34 @@ impl UserDraftItemKind { } } + /// The `draft.value` key holding the user-typed target path — where a + /// deploy of this draft would land when the user staged a rename. A script + /// draft round-trips its own `path`; flows and apps write a separate + /// `draft_path`, and only when it differs from the row's path. + /// + /// `None` for every kind with no editor to stage a rename in — a resource, + /// a variable, a trigger. Their drafts carry no such key, so `Some` is also + /// the test for whether a kind can be moved at all. + pub fn typed_path_field(&self) -> Option<&'static str> { + use UserDraftItemKind::*; + match self { + Script => Some("path"), + Flow | App | RawApp => Some("draft_path"), + _ => None, + } + } + + /// The other of the two path keys a draft value can carry. The editors mirror + /// the typed path into it while it differs from the row's path, so a move that + /// rewrote only `typed_path_field` would leave the mirror naming the old + /// location — and the loaders prefer the mirror, which un-does the move. + pub fn mirror_path_field(&self) -> Option<&'static str> { + match self.typed_path_field()? { + "path" => Some("draft_path"), + _ => Some("path"), + } + } + /// Whether OTHER users' drafts at a path are visible to a viewer (the /// "others are editing" list, owner circles, and the `get_draft_for_user` /// View JSON / Fork endpoint). Enabled only for the full-page editor items @@ -212,6 +240,12 @@ pub struct WithDraftOverlay { pub is_draft: bool, #[serde(skip_serializing_if = "Option::is_none")] pub draft_saved_at: Option>, + /// The deployed version the draft forked from (`draft.base`), as text + /// whatever the kind. The editor compares it to the head it loaded to tell + /// a draft that is behind. Absent when there is no draft or it was never + /// forked from a deploy. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_base: Option, /// True when no deployed row exists at this path: `inner` is only a /// best-effort stand-in synthesized from the draft and only `draft` is /// canonical. Frontend uses this to disable "diff/reset vs deployed" and @@ -297,6 +331,7 @@ where inner: Box::new(deployed), is_draft: false, draft_saved_at: None, + draft_base: None, no_deployed: false, draft: None, other_drafts_users: Vec::new(), @@ -317,7 +352,7 @@ where // row when an owned one exists. let row = sqlx::query!( r#"SELECT value as "value!: sqlx::types::Json>", - created_at + created_at, base FROM draft WHERE workspace_id = $1 AND (email = $2 OR email IS NULL) @@ -338,6 +373,7 @@ where inner: Box::new(deployed), is_draft: false, draft_saved_at: None, + draft_base: None, no_deployed: false, draft: None, other_drafts_users, @@ -350,6 +386,7 @@ where inner: Box::new(deployed), is_draft: true, draft_saved_at: Some(row.created_at), + draft_base: row.base, no_deployed: false, draft: Some(draft_json), other_drafts_users, @@ -468,7 +505,8 @@ pub async fn overlay_or_draft_only( } } -/// Delete the drafts an address owns, across every workspace. +/// Delete the drafts an address owns, and the move records routing saves to them, +/// across every workspace. /// /// `draft.email` carries no foreign key to `password`: a draft's owner is any principal the /// instance authenticates, and an external JWT's subject never has a `password` row. Deleting an @@ -479,12 +517,14 @@ pub async fn overlay_or_draft_only( /// No authorization of its own: it acts instance-wide on whatever address it is handed, so the /// caller must already have authorized removing that account (superadmin, the account's own /// holder, or SCIM). -pub async fn delete_drafts_of_email<'c>( - executor: impl sqlx::PgExecutor<'c>, - email: &str, -) -> Result<()> { +pub async fn delete_drafts_of_email(conn: &mut sqlx::PgConnection, email: &str) -> Result<()> { sqlx::query!("DELETE FROM draft WHERE email = $1", email) - .execute(executor) + .execute(&mut *conn) + .await?; + // The records routing saves to those drafts go with them: one left behind would send + // a later holder of the address to a draft that no longer exists. + sqlx::query!("DELETE FROM draft_move WHERE email = $1", email) + .execute(&mut *conn) .await?; Ok(()) } @@ -526,6 +566,29 @@ pub async fn rename_drafts_of_email( ) .execute(&mut *conn) .await?; + // The records that route saves to those drafts follow the same address, or a save + // still addressed to a path the account moved away from would start a second draft + // there. The moving account's record for a path wins, as its draft just did. + sqlx::query!( + "DELETE FROM draft_move dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft_move src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.old_path = dest.old_path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft_move SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; Ok(()) } @@ -582,6 +645,247 @@ pub async fn delete_own_draft_for_path( Ok(()) } +/// Carry every draft at `old_path` over to `new_path` when an item MOVES +/// (rename or relocation). A draft is bound to its item by nothing but the path +/// string, so without this a move detaches every draft on the item. No owner +/// filter: teammates' rows and the legacy NULL-email row follow too. +/// +/// **The caller must have authorized the underlying item move first.** This rewrites +/// rows the caller has no permission on and enforces nothing itself, taking both paths +/// on trust: it is safe only inside a deploy that has already cleared them, and is a +/// cross-user write with no gate anywhere else. +/// +/// The value keeps its base version, so every carried draft reads as behind the head the +/// move minted. Of its two path keys, one still naming `old_path` follows the row (the +/// editors write the item's own path there on every save, so it is the item's path and +/// not a staged rename); any other value is a rename the user staged, and is kept. +/// +/// A draft already at `new_path` occupies it the way a deployed item does, so the move is +/// refused with `BadRequest` inside the deploy's transaction, refusing the rename itself: +/// moving onto it would merge two items or strand the row that lost. A row a `draft_move` +/// record already brought there from `old_path` is this rename's own draft and does not +/// count: deploying it would otherwise be refused against itself, with no way out. +pub async fn move_drafts_for_path( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + old_path: &str, + new_path: &str, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + // Named by workspace username, as the editors name other users' drafts: the + // caller is often not the owner, and cannot clear a draft they cannot find. + let owners = sqlx::query!( + r#"SELECT d.email IS NULL as "legacy!", COALESCE(u.username, p.username) as username + FROM draft d + LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true + WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ::text = ANY($3::text[]) + -- A row a move already brought here from the very path being renamed is this + -- rename's own draft, not an item in its way: the record says which owners + -- it moved (all of them for an item move, one for a draft-only move). Any + -- other row is a second item and still collides. + AND NOT EXISTS ( + SELECT 1 FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ::text = ANY($3::text[]) + AND m.old_path = $4 AND m.new_path = $2 + AND (m.email IS NULL OR m.email IS NOT DISTINCT FROM d.email) + ) + ORDER BY 2"#, + w_id, + new_path, + &typs as &[&str], + old_path, + ) + .fetch_all(&mut **tx) + .await?; + if !owners.is_empty() { + let names = owners + .into_iter() + .map(|o| match (o.username, o.legacy) { + (Some(name), _) => name, + (None, true) => "a legacy workspace draft".to_string(), + (None, false) => "another user".to_string(), + }) + .collect::>() + .join(", "); + return Err(crate::error::Error::BadRequest(format!( + "'{new_path}' already has a draft on it ({names}) — it must be moved or discarded first" + ))); + } + // `draft.value` is `json`, so a NUL escape left by a pre-sanitizer write makes + // `to_jsonb` raise 22P05. `clean` is `strip_json_nul`'s parity rule in SQL (escaped + // backslashes parked on chr(1), which a `json` value's text cannot hold, so nothing + // collides with it; chr(92) spells the backslash so no escape sequence reaches this + // file). A row whose keys need re-pointing is therefore rewritten clean rather than + // left naming the old path, and one that needs nothing keeps its value byte for byte. + sqlx::query!( + r#"UPDATE draft AS d + SET path = $3::text, + value = ( + SELECT CASE + WHEN s.clean -> 'path' = to_jsonb($2::text) + OR s.clean -> 'draft_path' = to_jsonb($2::text) + THEN to_json( + s.clean + || CASE WHEN s.clean -> 'path' = to_jsonb($2::text) + THEN jsonb_build_object('path', $3::text) + ELSE '{}'::jsonb END + || CASE WHEN s.clean -> 'draft_path' = to_jsonb($2::text) + THEN jsonb_build_object('draft_path', $3::text) + ELSE '{}'::jsonb END + ) + ELSE d.value + END + FROM (SELECT replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), + chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean) s + ) + WHERE d.workspace_id = $1 + AND d.path = $2::text + AND d.typ::text = ANY($4::text[])"#, + w_id, + old_path, + new_path, + &typs as &[&str], + ) + .execute(&mut **tx) + .await?; + record_draft_move(tx, w_id, kinds, old_path, new_path, None).await +} + +/// Record that the drafts at `old_path` now live at `new_path`, so a draft save still +/// addressed to `old_path` lands on them (see `update_draft`). `email` scopes the +/// record to one user's draft-only move; `None` is a deployed item's move, for everyone. +/// +/// Kept to one hop: records pointing at `old_path` are re-pointed (an owner's move +/// leaves the item's own record alone and copies it into its own scope instead), and +/// records leaving either path are replaced, since `new_path` now holds the item. +/// +/// **The caller must have authorized the move first.** A record routes every later +/// draft write at `old_path` (any owner's, for an item move), and enforces nothing +/// itself. +/// +/// A record outlives the editors that need it: it ends when a later move touches either +/// path or an item is deployed at `old_path`, so a save meaning to start a NEW draft at a +/// vacated path would be routed instead. Nothing does that today: every surface parks a +/// new item at a minted `u//draft_` key (`mintDraftPath.ts`). +pub async fn record_draft_move( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + old_path: &str, + new_path: &str, + email: Option<&str>, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + sqlx::query!( + "DELETE FROM draft_move + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND old_path IN ($3, $4) AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + // Routes that ended at the destination before this move describe drafts that were + // carried there for an item this one is replacing: left alive, a save addressed to + // the start of that chain would land on this move's draft instead. + sqlx::query!( + "DELETE FROM draft_move + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND new_path = $4 AND old_path <> $3 AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE draft_move SET new_path = $4 + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND new_path = $3 AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + // An owner's move must not re-point what everyone else follows, so the item's own + // records ending at `old_path` are copied into this scope: a save addressed to the + // start of that chain still reaches this destination in one hop. Only those: another + // user's owner-scoped record routes their own draft-only item, which this move has + // nothing to do with. + if email.is_some() { + sqlx::query!( + "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email) + SELECT m.workspace_id, m.typ, m.old_path, $4, $5::text + FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ::text = ANY($2::text[]) + AND m.new_path = $3 AND m.email IS NULL + AND NOT EXISTS ( + SELECT 1 FROM draft_move o + WHERE o.workspace_id = m.workspace_id AND o.typ = m.typ + AND o.old_path = m.old_path AND o.email = $5::text + )", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + } + sqlx::query!( + "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email) + SELECT $1, t::draft_kind, $3, $4, $5 FROM unnest($2::text[]) t", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Drop the move records at `path`: an item was just created there, so saves addressed to +/// it are its own (records leaving `path`). A deploy that is not a rename also drops the +/// records arriving, which point at an item that no longer owns the path; `keep_from`, +/// the path a rename came from, suppresses that, since the chain ending here is the one +/// this very deploy just wrote. +/// +/// **The caller must have authorized the deploy that created the item first.** +/// Dropping a record sends later draft writes at `path` back to `path`. +pub async fn clear_draft_moves_from( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + path: &str, + keep_from: Option<&str>, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + sqlx::query!( + "DELETE FROM draft_move WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND (old_path = $3 OR (new_path = $3 AND $4::text IS NULL))", + w_id, + &typs as &[&str], + path, + keep_from, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + /// Fetch the authed user's draft as a standalone payload, for "get by path" /// routes when no deployed row exists but a draft might. Returns it as a /// `WithDraftOverlay` with `inner` and `draft` both set to the same JSON and @@ -600,7 +904,7 @@ pub async fn fetch_draft_only( // Own draft first, legacy NULL-email row as fallback (see `maybe_overlay_draft`). let row = sqlx::query!( r#"SELECT value as "value!: sqlx::types::Json>", - created_at + created_at, base FROM draft WHERE workspace_id = $1 AND (email = $2 OR email IS NULL) @@ -631,6 +935,7 @@ pub async fn fetch_draft_only( inner: Box::new(draft_json.clone()), is_draft: true, draft_saved_at: Some(row.created_at), + draft_base: row.base, no_deployed: true, draft: Some(draft_json), other_drafts_users, diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index fd0bd634ea..29fc9e99f3 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -479,6 +479,28 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) { (per_page, offset) } +/// [`paginate`] for a listing that answers whole unless a size is asked for: the deploy +/// histories, which the history panels and the CLI read unpaged while the diff picker takes +/// a page at a time. An asked-for size is still clamped, and the offset saturates rather +/// than wrapping, so no caller can turn this into an unbounded scan or a negative bind. +pub fn paginate_optional(pagination: Pagination) -> (i64, i64) { + // Naming neither parameter asks for the whole listing, the contract these endpoints + // have always answered on. Naming either makes it a page like any other listing's. + if pagination.page.is_none() && pagination.per_page.is_none() { + return (i64::MAX, 0); + } + let per_page = pagination + .per_page + .unwrap_or(DEFAULT_PER_PAGE) + .clamp(1, MAX_PER_PAGE) as i64; + // Bound before Postgres sees it: an unchecked cast of a caller-controlled page becomes + // a negative OFFSET, which is an error rather than an empty page. + let offset = i64::try_from(pagination.page.unwrap_or(1).max(1) - 1) + .unwrap_or(i64::MAX) + .saturating_mul(per_page); + (per_page, offset) +} + pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) { let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE); let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; @@ -1672,6 +1694,31 @@ pub fn truncate_with_ellipsis(s: &str, max_chars: usize) -> String { mod tests { use super::*; + #[test] + fn test_paginate_optional_answers_whole_but_bounds_what_is_asked_for() { + // Nothing asked for: every row there can be, which is what the history panels and + // the CLI read. + assert_eq!( + paginate_optional(Pagination { page: None, per_page: None }), + (i64::MAX, 0) + ); + assert_eq!( + paginate_optional(Pagination { page: Some(3), per_page: Some(20) }), + (20, 40) + ); + // An asked-for size is still capped, so no caller turns this into an unbounded scan. + assert_eq!( + paginate_optional(Pagination { page: None, per_page: Some(usize::MAX) }), + (MAX_PER_PAGE as i64, 0) + ); + // A page nobody could mean lands past the end rather than going negative, which + // Postgres would reject outright. + let (per_page, offset) = + paginate_optional(Pagination { page: Some(usize::MAX), per_page: Some(20) }); + assert_eq!(per_page, 20); + assert_eq!(offset, i64::MAX); + } + /// A 5-field crontab line is the most common way to get a schedule rejected, and both /// parsers report it in terms a crontab user cannot act on, so the seconds field and the /// equivalent expression must reach the caller for v1 and v2 alike. diff --git a/backend/windmill-runtime-nativets/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js index 7c7d649042..c2dbd1fef1 100644 --- a/backend/windmill-runtime-nativets/src/windmill-client.js +++ b/backend/windmill-runtime-nativets/src/windmill-client.js @@ -6457,7 +6457,7 @@ var AppService = class { * @param data.workspace * @param data.path * @param data.requestBody update app - * @returns string app updated + * @returns AppDeployed the path the app now lives at and the version this deploy wrote * @throws ApiError */ static updateApp(data) { diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 9a059da6e8..9e29731505 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -507,6 +507,10 @@ pub struct ScriptHistory { pub deployment_msg: Option, #[serde(skip_serializing_if = "Option::is_none")] pub created_at: Option>, + /// Who deployed this version — the version picker names it, so a reader can + /// tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, } #[derive(Deserialize)] diff --git a/frontend/src/lib/components/DiffDrawer.svelte b/frontend/src/lib/components/DiffDrawer.svelte index d70046d62a..2e5810e036 100644 --- a/frontend/src/lib/components/DiffDrawer.svelte +++ b/frontend/src/lib/components/DiffDrawer.svelte @@ -1,6 +1,6 @@ {#if !onLatest} - - By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff. + + {headUnknown + ? "Deploying may overwrite changes made by other users. Press 'Deploy' to see diff." + : "By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff."}
              {/if} diff --git a/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts b/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts index 122c42bcf9..f799cb864a 100644 --- a/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts +++ b/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts @@ -1,7 +1,7 @@ -import { base } from "$lib/base" -import { workspaceStore } from "$lib/stores" -import { get } from "svelte/store" +import { base } from '$lib/base' +import { workspaceStore } from '$lib/stores' +import { get } from 'svelte/store' export function computeSecretUrl(secretUrl: string) { - return `${window.location.origin}${base}/public/${get(workspaceStore)}/${secretUrl}` + return `${window.location.origin}${base}/public/${get(workspaceStore)}/${secretUrl}` } diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 960e273ac3..75de2d7faf 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -183,6 +183,11 @@ export interface AppEditorProps { // AppEditorHeader as a callback prop rather than `on:restore` forwarding, // which does not propagate through these runes-mode components. onRestore?: (restoredApp: any) => void + // Fired after a successful deploy, which keeps this editor open: `version` is what + // the deploy wrote, for the next draft's fork base, and `head` what is deployed now, + // with its author and time. `version` and `head` differ when another deploy landed + // beside this one, which is when the out-of-date prompt has something to say. + onDeploy?: (e: { version?: number; head?: number; headBy?: string; headAt?: string }) => void } export type App = { diff --git a/frontend/src/lib/components/common/alert/Alert.svelte b/frontend/src/lib/components/common/alert/Alert.svelte index c3d86693be..e4cdd7bd7d 100644 --- a/frontend/src/lib/components/common/alert/Alert.svelte +++ b/frontend/src/lib/components/common/alert/Alert.svelte @@ -71,7 +71,9 @@ style={bgStyle} >
              -
              + +
              {#if showIcon} +
              {/if} -
              + +

              {title}

              diff --git a/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte index 8d41fff902..bd830de1b7 100644 --- a/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte @@ -11,6 +11,15 @@ confirmCallback: () => void deployedBy?: string | undefined open?: boolean + /** Takes a drawer opening on the host editor's behalf, so the diff below is one the + * editor can take back down. Override anyway deploys that editor's content: without + * this the drawer would outlive it (a relocation remounts the editor) and deploy it + * at a path it has left. Omit where the host cannot be remounted under the drawer. */ + claimOpening?: () => number | undefined + /** The host could not read the deployed head, so this confirmation is caution rather + * than an observed newer version: nobody may have deployed over the user, and + * `deployedBy` is then whoever wrote the head, possibly themselves. */ + headUnknown?: boolean } let { @@ -19,13 +28,15 @@ diffDrawer = undefined, confirmCallback, deployedBy = undefined, - open = $bindable(false) + open = $bindable(false), + claimOpening = undefined, + headUnknown = false }: Props = $props() { open = false @@ -33,7 +44,11 @@ on:confirmed={() => confirmCallback()} >
              - A new version was deployed while you were editing this one. + + {headUnknown + ? 'This editor could not check whether a newer version is deployed, so it cannot tell whether this overwrites newer work.' + : 'A new version was deployed while you were editing this one.'} + {#if diffDrawer} diff --git a/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte b/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte index 26366325b5..20c29286b5 100644 --- a/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte +++ b/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte @@ -13,6 +13,10 @@ import type { UserDraftItemKind } from '$lib/gen' import DraftSyncConflictModal from './DraftSyncConflictModal.svelte' import OtherUsersDraftsModal, { type OtherDraftUser } from './OtherUsersDraftsModal.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { sendUserToast } from '$lib/toast' + import { base } from '$app/paths' + import { goto } from '$app/navigation' import StaleDraftModal from './StaleDraftModal.svelte' import ConfirmationModal from './ConfirmationModal.svelte' import { OtherUserDraftLoad } from '$lib/components/otherUserDraftLoad.svelte' @@ -38,15 +42,27 @@ draftSavedAt?: string | undefined /** ISO timestamp of the latest deploy at this path. */ deployedAt?: string | undefined - /** Precise staleness inputs (flows/apps): the deployed version the draft was - * forked from, and the current deployed head. When both are set they drive - * `isStale` and the dedup key instead of the timestamps — exact, and stable - * across autosaves (the timestamp drifts past `deployedAt` as you keep - * editing). Absent (pre-feature drafts, scripts) ⇒ timestamp fallback. */ - draftBaseVersion?: number | undefined - deployedHeadVersion?: number | undefined + /** Precise staleness inputs: the deployed version the draft forked from + * (`draft_base` on the get-by-path response) and the current deployed head, + * both as text whatever the kind. When both are set they drive `isStale` and + * the dedup key instead of the timestamps, which drift past `deployedAt` as + * you keep editing. Absent (a draft never forked from a deploy) ⇒ timestamp + * fallback. */ + draftBaseVersion?: string | undefined + deployedHeadVersion?: string | undefined + /** Who deployed the head, named in the stale prompt. */ + deployedBy?: string | undefined /** Discard the draft and reload deployed (same as "Reset to deployed"). */ onLoadLatestDeploy?: () => void | Promise + /** Opens the editor's Deployed↔Current diff from the stale prompt, so the + * choice between keeping and discarding is informed. Omit where the editor + * has no diff drawer; the action is then not rendered. */ + onViewDiff?: () => void | Promise + /** Runs before this editor follows its draft to the item's new path: the + * editor's own draft save, which materializes text the code editor still + * holds. Without it, keystrokes typed since the relocating save are lost + * to the navigation. */ + onBeforeRelocate?: () => void | Promise /** Defaults to true; set to false to suppress all modals. */ enabled?: boolean } @@ -65,7 +81,10 @@ deployedAt = undefined, draftBaseVersion = undefined, deployedHeadVersion = undefined, + deployedBy = undefined, onLoadLatestDeploy, + onViewDiff, + onBeforeRelocate, enabled = true }: Props = $props() @@ -74,9 +93,9 @@ let staleAlertKey = $state(undefined) let staleModalOpen = $state(false) - // Prefer the exact version comparison (flows/apps) over the timestamp: the - // draft's pinned fork base never drifts, whereas `draftSavedAt` advances past - // `deployedAt` once you keep editing a stale draft, hiding the staleness. + // Prefer the version comparison over the timestamp for every kind that supplies + // one: `draftSavedAt` advances past `deployedAt` as you keep editing, hiding the + // staleness outright. const useVersion = $derived(draftBaseVersion != null && deployedHeadVersion != null) const isStale = $derived( !!onLoadLatestDeploy && @@ -105,6 +124,48 @@ } }) }) + + const EDITOR_SEGMENT: Partial> = { + script: 'scripts/edit', + flow: 'flows/edit', + app: 'apps/edit', + raw_app: 'apps_raw/edit' + } + + // The item was moved while this editor was open: the draft row followed it + // and the save just landed there. Follow it too — the route reloads the item + // at its new path, and the stale prompt above then says what changed. Edits + // typed since that save are flushed first, so leaving this path drops none. + $effect(() => { + if (!enabled || !workspace || !path) return + const seg = EDITOR_SEGMENT[itemKind] + if (!seg) return + const query = { workspace, itemKind, path } + // The flush below saves again and can land here a second time, and a second move + // can land while it runs: the last destination reported is the one to follow. + let relocating = false + let destination: string | undefined = undefined + return UserDraftDbSyncer.onRelocated(query, async (newPath) => { + destination = newPath + if (relocating) return + relocating = true + await onBeforeRelocate?.() + await UserDraftDbSyncer.flush(query) + // `flush` resolves on a failed or rejected save as well, and leaving the + // route drops what it was carrying: stay, so the editor keeps the edits + // and its own failure indicator. + if ( + UserDraftDbSyncer.getState(query).failureMessage || + UserDraftDbSyncer.getConflict(query).conflict + ) { + relocating = false + return + } + const target = destination ?? newPath + sendUserToast(`This item was moved to ${target}. You are now editing it there.`) + await goto(`${base}/${seg}/${target}`) + }) + }) {#if enabled && workspace && path} @@ -130,9 +191,14 @@ {#if onLoadLatestDeploy} {/if} /** - * Modal opened on editor mount when the authed user's per-user draft - * is older than the latest deployed version at the same path — i.e. - * someone else deployed a new version while the draft was sitting - * around. The user is asked to either pick up the latest deploy - * (discards the stale draft) or keep editing what they had. - * - * The parent threads `draftSavedAt` and `deployedAt` raw and the - * modal computes staleness internally; this keeps each route from - * re-implementing the comparison and the threshold (we treat a - * draft as stale only when it's strictly older — a deploy at the - * exact same instant is treated as not stale). + * The prompt for a draft that is behind: someone deployed a newer version + * of the item after the draft forked from it. Opened on every load while + * that holds (the parent computes it; see DraftEditorModals), it names the + * two versions and offers two ways out: look at the diff, or drop the draft + * for the latest deploy. Closing it is "keep editing". Taking the latest as + * the new base while keeping the edits lives in the diff drawer, where the + * user can see what they are taking. * * Open-state is bindable so the parent can dismiss programmatically * (e.g. after the load-latest-deploy callback completes). */ import Modal2 from '$lib/components/common/modal/Modal2.svelte' import Button from '$lib/components/common/button/Button.svelte' - import { AlertTriangle } from 'lucide-svelte' + import { AlertTriangle, GitCompare } from 'lucide-svelte' import { sendUserToast } from '$lib/toast' + import type { UserDraftItemKind } from '$lib/gen' type Props = { isOpen: boolean - /** ISO timestamp the authed user's draft was saved. */ + itemKind: UserDraftItemKind + /** ISO timestamp the authed user's draft was saved. Shown only when the + * versions below are unknown (a draft that predates the base). */ draftSavedAt: string | undefined /** ISO timestamp the latest deploy at this path landed. */ deployedAt: string | undefined + /** The version the draft forked from and the deployed head, as text. */ + draftBaseVersion?: string | undefined + deployedHeadVersion?: string | undefined + /** Who deployed the head. */ + deployedBy?: string | undefined /** Discards the draft and reloads the deployed payload — the route * already has this callback for the AutosaveIndicator's "Reset to * deployed" button; pass the same function in. */ onLoadLatestDeploy: () => void | Promise + /** Opens the editor's own Deployed↔Current diff. Without it the user is + * asked to choose between keeping and discarding their draft with no way + * to see what actually differs — and after a rename the difference is + * often only the path. Omitted where the editor has no diff drawer. */ + onViewDiff?: () => void | Promise } - let { isOpen = $bindable(), draftSavedAt, deployedAt, onLoadLatestDeploy }: Props = $props() + let { + isOpen = $bindable(), + itemKind, + draftSavedAt, + deployedAt, + draftBaseVersion = undefined, + deployedHeadVersion = undefined, + deployedBy = undefined, + onLoadLatestDeploy, + onViewDiff + }: Props = $props() let loading = $state(false) + // Scripts are versioned by hash, the other kinds by a numeric version id; + // the diff picker renders them the same way. + function formatVersion(v: string): string { + return itemKind === 'script' ? v.slice(0, 8) : v + } + async function loadLatestDeploy() { if (loading) return loading = true @@ -49,6 +74,13 @@ } } + // Dismisses on the way out: the diff drawer opens behind this modal, so + // leaving it up would cover the thing the user asked to see. + async function viewDiff() { + isOpen = false + await onViewDiff?.() + } + function formatTs(ts: string | undefined): string { if (!ts) return '' try { @@ -64,22 +96,38 @@
              -

              - A newer version was deployed after you started editing. Your draft is based on the older - deploy. -

              -

              - Draft saved {formatTs(draftSavedAt)} · Deployed {formatTs(deployedAt)} -

              +

              A newer version was deployed after you started editing.

              + {#if draftBaseVersion && deployedHeadVersion} +

              + Your draft is based on {formatVersion(draftBaseVersion)} + · latest is {formatVersion(deployedHeadVersion)} + {#if deployedBy}by {deployedBy}{/if}{#if deployedAt}, {formatTs(deployedAt)}{/if} +

              + {:else} +

              + Draft saved {formatTs(draftSavedAt)} · Deployed {formatTs(deployedAt)} +

              + {/if}
              -
              - - + {:else} +
              + {/if} + +
              diff --git a/frontend/src/lib/components/common/table/AppRow.svelte b/frontend/src/lib/components/common/table/AppRow.svelte index 1059a986ce..bd881a31b7 100644 --- a/frontend/src/lib/components/common/table/AppRow.svelte +++ b/frontend/src/lib/components/common/table/AppRow.svelte @@ -11,7 +11,7 @@ import { createEventDispatcher } from 'svelte' import Button from '../button/Button.svelte' import Row from './Row.svelte' - import type { RowSelection } from './rowSelection' + import { selectMenuItems, type RowSelection } from './rowSelection' import InheritedLabels from '$lib/components/InheritedLabels.svelte' import Badge from '../badge/Badge.svelte' import { @@ -188,6 +188,21 @@ const canEdit = canWrite && showEditButton if (draft_only) { return [ + ...selectMenuItems(rowSelection), + { + displayName: 'Move/Rename', + icon: FolderOpen, + action: () => { + // Addressed by the generated path its draft row sits at, but + // named by the path typed in the editor. + moveDrawer.openDrawer((app as any).draft_path ?? path, summary, 'app', { + storagePath: path, + rawApp: !!app.raw_app + }) + }, + disabled: !showEditButton, + hide: $userStore?.operator + }, { displayName: 'Delete', icon: Trash, @@ -221,6 +236,7 @@ ] } return [ + ...selectMenuItems(rowSelection), { displayName: 'Duplicate/Fork', icon: GitFork, diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index 1eb0d35878..f9af803694 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -14,7 +14,7 @@ import Badge from '../badge/Badge.svelte' import Button from '../button/Button.svelte' import Row from './Row.svelte' - import type { RowSelection } from './rowSelection' + import { selectMenuItems, type RowSelection } from './rowSelection' import { sendUserToast } from '$lib/toast' import { copyToClipboard, isOwner } from '$lib/utils' import { isDeployable } from '$lib/utils_deployable' @@ -219,6 +219,20 @@ const canEdit = flow.canWrite && showEditButton if (draft_only) { return [ + ...selectMenuItems(rowSelection), + { + displayName: 'Move/Rename', + icon: FolderOpen, + action: () => { + // Addressed by the generated path its draft row sits at, but + // named by the path typed in the editor. + moveDrawer.openDrawer((flow as any).draft_path ?? path, flow.summary, 'flow', { + storagePath: path + }) + }, + disabled: !showEditButton, + hide: $userStore?.operator + }, { displayName: 'Delete', icon: Trash, @@ -242,6 +256,7 @@ ] } return [ + ...selectMenuItems(rowSelection), { displayName: 'View runs', icon: List, diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index ab5a460a5e..938086a0e8 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -23,7 +23,7 @@ import Badge from '../badge/Badge.svelte' import Button from '../button/Button.svelte' import Row from './Row.svelte' - import type { RowSelection } from './rowSelection' + import { selectMenuItems, type RowSelection } from './rowSelection' import { sendUserToast } from '$lib/toast' import { capitalize, copyToClipboard, isOwner } from '$lib/utils' import { isDeployable } from '$lib/utils_deployable' @@ -281,6 +281,7 @@ const canEdit = script.canWrite && showEditButton if (script.draft_only) { return [ + ...selectMenuItems(rowSelection), { displayName: 'View code', icon: Code, @@ -288,6 +289,19 @@ showCode(script.path, script.summary) } }, + { + displayName: 'Move/Rename', + icon: FolderOpen, + action: () => { + // Addressed by the generated path its draft row sits at, but + // named by the path typed in the editor. + moveDrawer.openDrawer(script.draft_path ?? script.path, script.summary, 'script', { + storagePath: script.path + }) + }, + disabled: !showEditButton, + hide: $userStore?.operator + }, { displayName: 'Delete', icon: Trash, @@ -311,6 +325,7 @@ ] } return [ + ...selectMenuItems(rowSelection), { displayName: 'View code', icon: Code, diff --git a/frontend/src/lib/components/common/table/rowSelection.ts b/frontend/src/lib/components/common/table/rowSelection.ts index 081f29addc..a78fe0aa1d 100644 --- a/frontend/src/lib/components/common/table/rowSelection.ts +++ b/frontend/src/lib/components/common/table/rowSelection.ts @@ -1,3 +1,6 @@ +import { SquareCheckBig } from 'lucide-svelte' +import type { Item } from '$lib/utils' + /** * Wiring for a row whose kind icon doubles as a selection control: the icon * swaps to a checkbox on hover, and stays one while a selection is active. @@ -14,3 +17,19 @@ export type RowSelection = { active: boolean onToggle: (e: MouseEvent | KeyboardEvent) => void } + +/** + * The row menu's way into a selection, for the rows that offer one. The icon + * checkbox is the fast path but only appears on hover; this is the one a user + * can find by looking. + */ +export function selectMenuItems(rowSelection: RowSelection | undefined): Item[] { + if (!rowSelection) return [] + return [ + { + displayName: rowSelection.selected ? 'Deselect' : 'Select', + icon: SquareCheckBig, + action: (e) => rowSelection.onToggle(e) + } + ] +} diff --git a/frontend/src/lib/components/diff_drawer.ts b/frontend/src/lib/components/diff_drawer.ts index c6e9b5cdf8..c1e7e8980a 100644 --- a/frontend/src/lib/components/diff_drawer.ts +++ b/frontend/src/lib/components/diff_drawer.ts @@ -1,9 +1,53 @@ import type { Value } from '$lib/utils' +export type DiffVersionOption = { + /** Opaque to the drawer — a script hash, a flow version id, an app version id. */ + id: string + /** Identifies the version: its number and hash. Carries the weight in the list. */ + label: string + /** Who deployed it, and when — rendered under the label in secondary text, so the + * version reads first and the attribution second. */ + subtitle?: string + /** The version currently deployed. */ + isHead?: boolean +} + +/** How many deployed versions the diff picker asks for at a time. The history endpoints + * still answer whole when nobody asks — the panels and the CLI read them that way — so + * this is the picker's own appetite, not their default. */ +export const VERSION_PAGE_SIZE = 20 + export type DiffDrawerDiff = | { mode: 'normal' deployed: Value + /** Names the deployed side, e.g. `Deployed d2154d55 by bob`. The version + * identity is stripped from the displayed metadata, so without this the + * reader cannot tell what their draft is being compared against. */ + deployedLabel?: string + /** Deployed versions the reader can compare against, newest first. The + * drawer only renders the picker; each editor supplies its own list and + * fetcher because scripts, flows and apps identify versions differently. */ + versions?: DiffVersionOption[] + /** Loads one version's payload. Returning `undefined` leaves the current + * comparison in place rather than blanking the diff. */ + loadVersion?: (id: string) => Promise + /** Fetches the next, older page of `versions`. `versions` holds one page so the + * drawer opens without waiting on a path a pipeline has deployed thousands of + * times; the reader asks for the rest. Returning nothing ends the list. */ + loadMoreVersions?: () => Promise + /** Moves the draft's base to the head and keeps its content, rendered as a + * header action so the user takes the latest with the diff in front of them. + * Called with the version the drawer is showing as head (from `versions`), so + * the base adopted is the one the reader just looked at. */ + onTakeLatest?: (head?: string) => void | Promise + /** The version the draft forked from. The drawer offers `onTakeLatest` only + * while it differs from the head, and passes that head to it. */ + draftBase?: string + /** The version the deployed payload passed here came from, used as the head when + * no `versions` list loaded. Without either, the drawer cannot tell whether the + * draft is behind and offers nothing. */ + deployedHead?: string draft?: Value | undefined current: Value defaultDiffType?: 'deployed' | 'draft' @@ -18,7 +62,19 @@ export type DiffDrawerDiff = } export interface DiffDrawerI { - openDrawer: () => void + /** Pass the token from `beginOpening` to continue that opening; called without one, + * the drawer claims a fresh opening, so any reuse invalidates a fetch still in + * flight rather than being overwritten by it. */ + openDrawer: (token?: number) => void closeDrawer: () => void - setDiff: (diff: DiffDrawerDiff) => void + /** Takes the opening's token on the same terms as `openDrawer`. */ + setDiff: (diff: DiffDrawerDiff, token?: number) => void + /** Claim the drawer for one opening. Filling it takes awaited fetches, and a path + * change remounts the editor while this drawer stays, so the token lives here: an + * editor checks `ownsOpening` before every write and drops the opening it started + * when it goes away. */ + beginOpening: () => number + ownsOpening: (token: number) => boolean + /** Drop that opening and, when it still owns the drawer, what it put on screen. */ + abandonOpening: (token: number) => void } diff --git a/frontend/src/lib/components/flow_builder.ts b/frontend/src/lib/components/flow_builder.ts index ce848c6f83..f763e92faa 100644 --- a/frontend/src/lib/components/flow_builder.ts +++ b/frontend/src/lib/components/flow_builder.ts @@ -10,6 +10,14 @@ import type { WorkspaceItem } from './workspacePicker' export type FlowBuilderProps = { initialPath?: string + /** The draft row's own path (the URL path). `initialPath` is re-seeded from the + * draft's `draft_path` so the topbar shows the pending name, so it can't be used + * to resolve what is actually deployed. */ + userDraftPath?: string + /** Moves the draft's base to the head and keeps its content; offered in the diff + * drawer while the draft is behind, and called with the version the drawer shows + * as head. */ + onTakeLatest?: (head?: string) => void | Promise pathStoreInit?: string | undefined newFlow: boolean selectedId: string | undefined @@ -24,9 +32,10 @@ export type FlowBuilderProps = { disabledFlowInputs?: boolean savedPrimarySchedule?: ScheduleTrigger | undefined // used to set the primary schedule in the legacy primaryScheduleStore version?: number | undefined - /** flow_version the draft was forked from; when set, the deploy-time staleness - * check compares it (not the load-time head `version`) against the latest. */ - draftBaseVersion?: number | undefined + /** flow_version the draft was forked from (as text, like `draft_base`); when + * set, the deploy-time staleness check compares it (not the load-time head + * `version`) against the latest. */ + draftBaseVersion?: string | undefined draftTriggersFromUrl?: Trigger[] | undefined selectedTriggerIndexFromUrl?: number | undefined children?: import('svelte').Snippet diff --git a/frontend/src/lib/components/home/BulkActionsBar.svelte b/frontend/src/lib/components/home/BulkActionsBar.svelte index f36ed4848a..fffd18a3c0 100644 --- a/frontend/src/lib/components/home/BulkActionsBar.svelte +++ b/frontend/src/lib/components/home/BulkActionsBar.svelte @@ -10,13 +10,16 @@ import Select from '$lib/components/select/Select.svelte' import Label from '$lib/components/Label.svelte' import { sendUserToast } from '$lib/toast' + import { useReducedMotion } from '$lib/svelte5Utils.svelte' import { invalidateWorkspaceDrafts } from '$lib/workspaceDrafts.svelte' import { Archive, ArchiveRestore, FolderInput, MoreHorizontal, Trash, X } from 'lucide-svelte' + import { fly } from 'svelte/transition' import { blockedReason, eligible, movedPath, runBulk, + sourcePath, type BulkAction, type BulkContext, type BulkOutcome @@ -36,7 +39,17 @@ let { selection, workspace, isAdmin, moveTargets, onDone }: Props = $props() let ctx: BulkContext = $derived({ workspace, isAdmin }) - let items = $derived(selection.items) + // The bar is still on screen while it flies out, and unticking the last row is + // what starts that — by then the live selection is empty. Rendering the last + // non-empty one keeps the count and the action states from blinking on the way + // out. `keepOnly` only ever removes keys, so acting on the held copy is inert. + let lastNonEmpty = $state([]) + $effect(() => { + if (selection.items.length > 0) lastNonEmpty = selection.items + }) + let items = $derived(selection.items.length > 0 ? selection.items : lastNonEmpty) + + let reducedMotion = useReducedMotion() const plural = (n: number) => (n === 1 ? '' : 's') const ACTION_LABEL: Record = { @@ -71,9 +84,6 @@ function actionTitle(action: BulkAction): string { const n = targets(action).length - // Selection mode is entered from the toolbar with nothing picked yet, so this - // is the state the primary entry point lands on — it has no blocked reason. - if (items.length === 0) return `Select items to ${ACTION_LABEL[action].toLowerCase()}` if (n === 0) return `Cannot ${ACTION_LABEL[action].toLowerCase()}: ${blockedSummary(action)}` if (n < items.length) return `${ACTION_LABEL[action]} ${n} of the ${items.length} selected` return `${ACTION_LABEL[action]} ${n} item${plural(n)}` @@ -98,6 +108,11 @@ let discardReverts = $derived(pendingItems.filter((i) => !i.draftOnly)) function open(action: BulkAction) { + // Gate on the LIVE selection, not `items`. That holds the last non-empty + // copy so the bar doesn't blink to zero while it flies out, and the bar stays + // clickable for the whole 200ms outro — so without this a click landing in + // that window would open a dialog acting on rows the user just deselected. + if (selection.items.length === 0) return if (targets(action).length === 0) return pending = action outcomes = undefined @@ -156,92 +171,99 @@ } -
              - - {selection.size} selected - -
              - - - {#if targets('archive').length > 0 || targets('unarchive').length === 0} + + {items.length} selected + +
              - {/if} - {#if targets('unarchive').length > 0} - - {/if} - open('discard'), - disabled: targets('discard').length === 0, - // Only while blocked: a disabled entry can't open its modal, so the reason - // has to live here — and an enabled one would render a pointless ⓘ. - tooltip: targets('discard').length === 0 ? actionTitle('discard') : undefined - }, - { - displayName: `Delete${countSuffix('delete')}`, - icon: Trash, - type: 'delete' as const, - action: () => open('delete'), - disabled: targets('delete').length === 0, - tooltip: targets('delete').length === 0 ? actionTitle('delete') : undefined - } - ]} - > - {#snippet buttonReplacement()} + + {#if targets('archive').length > 0 || targets('unarchive').length === 0} - {/snippet} - -
              - + {/if} + open('discard'), + disabled: targets('discard').length === 0, + // Only while blocked: a disabled entry can't open its modal, so the reason + // has to live here — and an enabled one would render a pointless ⓘ. + tooltip: targets('discard').length === 0 ? actionTitle('discard') : undefined + }, + { + displayName: `Delete${countSuffix('delete')}`, + icon: Trash, + type: 'delete' as const, + action: () => open('delete'), + disabled: targets('delete').length === 0, + tooltip: targets('delete').length === 0 ? actionTitle('delete') : undefined + } + ]} + > + {#snippet buttonReplacement()} + + {/snippet} + +
              +
              `${i.path} → ${movedPath(i, target)}`) + pendingItems.map((i) => `${sourcePath(i)} → ${movedPath(i, target)}`) )} {/if} {:else if pending === 'discard'} diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 7038a316fa..f86ec9d5bf 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -20,7 +20,6 @@ import type uFuzzy from '@leeoniya/ufuzzy' import { ArrowDownUp, - CheckSquare, ChevronsDownUp, ChevronsUpDown, Code2, @@ -1805,20 +1804,10 @@ {/if} {#if !loading && !contentActive && !workspaceEmpty} - +
              - {#if homeSelection.available && !homeSelection.active} -
              {/if}
              - {#if isCloudHosted() && !menuHidden} - + {#if $enterpriseLicense && !menuHidden} + {/if} {#if $userStore?.is_service_account} From 9994e03bc75404503001a5db37e175b66e7371f2 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Sat, 19 Sep 2026 23:23:50 +0200 Subject: [PATCH 06/53] feat: add an ACL editor for data table roles (#11063) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * chore: update ee-repo-ref to 7e338e4dabf91689bfd7fb0333c6534040b17b59 This commit updates the EE repository reference after PR #787 was merged in windmill-ee-private. Previous ee-repo-ref: 0edd40979cf36bfba59323f3f6a0811ae1369cf5 New ee-repo-ref: 7e338e4dabf91689bfd7fb0333c6534040b17b59 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- .../tests/datatable_acl.rs | 108 + .../src/datatable_acl.rs | 1988 +++++++++++++++++ .../src/datatable_acl_oss.rs | 44 + .../src/datatable_permissions.rs | 12 + .../src/datatable_permissions_oss.rs | 18 +- backend/windmill-api-workspaces/src/lib.rs | 5 + .../windmill-api-workspaces/src/workspaces.rs | 1 + backend/windmill-api/openapi.yaml | 340 +++ backend/windmill-common/src/lib.rs | 14 + .../lib/components/InstanceSettings.svelte | 12 +- .../datatableAcl/AclTargetPicker.svelte | 50 + .../datatableAcl/PgAclEditor.svelte | 289 +++ .../datatableAcl/PgGrantBuilder.svelte | 95 + .../components/datatableAcl/aclScopes.test.ts | 178 ++ .../lib/components/datatableAcl/aclScopes.ts | 211 ++ .../DataTablePermissionsButton.svelte | 75 +- .../DataTableRolesSection.svelte | 11 - .../DataTableSettings.svelte | 41 +- .../InstanceRolesButton.svelte | 29 + 19 files changed, 3469 insertions(+), 52 deletions(-) create mode 100644 backend/windmill-api-integration-tests/tests/datatable_acl.rs create mode 100644 backend/windmill-api-workspaces/src/datatable_acl.rs create mode 100644 backend/windmill-api-workspaces/src/datatable_acl_oss.rs create mode 100644 frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte create mode 100644 frontend/src/lib/components/datatableAcl/PgAclEditor.svelte create mode 100644 frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte create mode 100644 frontend/src/lib/components/datatableAcl/aclScopes.test.ts create mode 100644 frontend/src/lib/components/datatableAcl/aclScopes.ts create mode 100644 frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte diff --git a/backend/windmill-api-integration-tests/tests/datatable_acl.rs b/backend/windmill-api-integration-tests/tests/datatable_acl.rs new file mode 100644 index 0000000000..4310830ae2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/datatable_acl.rs @@ -0,0 +1,108 @@ +//! Who may read and change a data table's grants and owners. On the Enterprise Edition: its +//! administrators, from the workspace that governs it. Without it: nobody. Each refusal is decided +//! before anything connects to the data table, so the fixture's database never has to exist. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn grant_select_on_public() -> Value { + json!({ + "target": {"kind": "schema", "schema": "public"}, + "change": {"type": "grant", "role": "analytics", "privileges": ["SELECT"], + "scope": "all_tables"}, + "statements": [r#"GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO "analytics""#] + }) +} + +async fn post_acl( + port: u16, + w_id: &str, + action: &str, + token: &str, +) -> anyhow::Result { + Ok(reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/datatable_acl/main/{action}" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&grant_select_on_public()) + .send() + .await?) +} + +/// A fork reaches the data table through a pointer: it may use it, never change what each role may +/// touch on it — not even as an admin of the fork. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_cannot_change_access_on_the_data_table_it_points_at( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "wm-fork-dt", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_member_who_is_not_an_admin_cannot_change_access( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "test-workspace", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +/// Not even reading, and not even on a data table that is not under roles — which any member +/// reaches, so only the edition stands between them and the instance's credentials. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn only_the_enterprise_edition_has_the_access_editor( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings + SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let read = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/datatable_acl/main?kind=database" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await?; + let mut responses = vec![("read", read)]; + for action in ["plan", "apply"] { + responses.push(( + action, + post_acl(port, "test-workspace", action, "SECRET_TOKEN").await?, + )); + } + for (action, resp) in responses { + assert_eq!(resp.status(), 400, "{action}"); + let body = resp.text().await?; + assert!( + body.contains("Data table roles are a Windmill Enterprise Edition feature"), + "{action}: {body}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs new file mode 100644 index 0000000000..6bd14316fc --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -0,0 +1,1988 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Ownership and grants on the objects of an instance data table. +//! +//! [`datatable_permissions`](crate::datatable_permissions) decides who may connect as which role; +//! this decides what each role may then touch. Every change is a real `GRANT`, `REVOKE`, +//! `ALTER ... OWNER TO` or `ALTER DEFAULT PRIVILEGES`, so Postgres is what enforces it. +//! +//! Reading is open to anyone who reaches the data table. Planning and applying are for those who +//! administer it — admins of the workspace that governs it, and superadmins. All of it is +//! Enterprise Edition ([`crate::datatable_acl_oss`]). + +use std::collections::BTreeMap; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{get, post}, + Json, Router, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use tokio_postgres::error::{DbError, SqlState}; +use tokio_postgres::AsyncMessage; + +use windmill_api_auth::ApiAuthed; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::datatable_roles::{ + lock_role_catalog, quote_ident, read_role_catalog, read_role_catalog_tx, DatatableRoleCatalog, + ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, +}; +use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; +use windmill_common::workspaces::{resolve_governing_datatable, DataTable, GoverningDatatable}; +use windmill_common::{PgDatabase, DB}; + +use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_governing_datatable}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/datatable_acl/{datatable_name}", get(get_datatable_acl)) + .route( + "/datatable_acl/{datatable_name}/plan", + post(plan_datatable_acl), + ) + .route( + "/datatable_acl/{datatable_name}/apply", + post(apply_datatable_acl), + ) +} + +/// What a read or a change is about. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AclTarget { + /// The data table's own database — where the privilege to create schemas lives. + Database, + Schema { + schema: String, + }, + Table { + schema: String, + table: String, + }, +} + +impl AclTarget { + /// The schema the target is in, absent for the database itself. + pub(crate) fn schema(&self) -> Option<&str> { + match self { + AclTarget::Database => None, + AclTarget::Schema { schema } => Some(schema), + AclTarget::Table { schema, .. } => Some(schema), + } + } + + /// What it is called in a message. + pub(crate) fn label(&self, dbname: &str) -> String { + match self { + AclTarget::Database => dbname.to_string(), + AclTarget::Schema { schema } => schema.clone(), + AclTarget::Table { schema, table } => format!("{schema}.{table}"), + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclTargetQuery { + kind: String, + schema: Option, + table: Option, +} + +impl TryFrom for AclTarget { + type Error = Error; + fn try_from(q: AclTargetQuery) -> Result { + match (q.kind.as_str(), q.schema, q.table) { + ("database", _, _) => Ok(AclTarget::Database), + ("schema", Some(schema), _) => Ok(AclTarget::Schema { schema }), + ("table", Some(schema), Some(table)) => Ok(AclTarget::Table { schema, table }), + ("schema" | "table", None, _) => { + Err(Error::BadRequest("This target needs a schema".to_string())) + } + ("table", _, None) => Err(Error::BadRequest( + "A table target needs a table".to_string(), + )), + (kind, _, _) => Err(Error::BadRequest(format!("Unknown ACL target '{kind}'"))), + } + } +} + +/// Where a set of privileges applies, relative to the target. +/// +/// `Future*` covers what does not exist yet: those become `ALTER DEFAULT PRIVILEGES`, which only +/// binds objects created by the roles it names. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum GrantScope { + /// The target itself — the database, the schema, or the table. + Target, + AllTables, + AllSequences, + AllFunctions, + FutureTables, + FutureSequences, + FutureFunctions, +} + +impl GrantScope { + pub(crate) fn is_future(&self) -> bool { + matches!( + self, + GrantScope::FutureTables | GrantScope::FutureSequences | GrantScope::FutureFunctions + ) + } +} + +/// A change to plan. One at a time: each is confirmed against its own SQL. +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AclChange { + /// Hand the target — and, for a schema, everything already in it but an extension's members, + /// which stay with the extension — to another role. + SetOwner { + role: String, + }, + Grant { + role: String, + privileges: Vec, + scope: GrantScope, + }, + Revoke { + role: String, + privileges: Vec, + scope: GrantScope, + /// Objects inside the target, empty for the target itself. `ON ALL TABLES` grants read + /// back per object, so they are revoked per object — and the same privileges on several + /// of them are revoked together. + #[serde(default)] + objects: Vec, + }, +} + +impl AclChange { + /// The role the change is about, as the editor names it. + fn role(&self) -> &str { + match self { + AclChange::SetOwner { role } + | AclChange::Grant { role, .. } + | AclChange::Revoke { role, .. } => role, + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclChangeRequest { + pub target: AclTarget, + pub change: AclChange, + /// The statements the plan showed. An apply runs only those: it plans again and refuses if the + /// result differs. + #[serde(default)] + pub statements: Option>, +} + +/// An object inside a schema, named the way `REVOKE ... ON ` needs it. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct AclObject { + pub name: String, + /// `TABLE`, `SEQUENCE`, `FUNCTION`, `PROCEDURE` or `TYPE`: what the object is. + /// [`object_keyword`] turns it into the keyword a revoke takes; a type has none. + pub kind: String, + /// A routine is identified by its argument types, not by its name: two `f` in one schema are + /// two objects. Absent for everything else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub args: Option, +} + +/// A grant as the database has it, under the role names the editor uses. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclGrant { + /// A data table role's name, `admin` for `custom_instance_user`, else the raw Postgres role + /// (`PUBLIC` included). + pub grantee: String, + pub privileges: Vec, + /// `None` for the target itself, else the object inside it. + #[serde(skip_serializing_if = "Option::is_none")] + pub object: Option, + /// `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES` (or, on the database, `SCHEMAS`) when this + /// is a default privilege, which applies to objects that do not exist yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub future: Option, + /// Where the grant comes from, each role once: who granted it, or for a default privilege the + /// role whose future objects it covers. A revoke of some of the privileges takes them back from + /// every source that gave them. + pub sources: Vec, +} + +/// One role a grant comes from. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclSource { + /// Under the same names as [`AclGrant::grantee`]. + pub role: String, + /// What `role` gave of the grant's privileges. A revoke is held back only by a source out of + /// reach that gave some of what it takes back. + pub privileges: Vec, + /// Whether this data table's connection can take back what `role` gave: on an object only the + /// owner's grants when it acts for the owner, or else its own (`grant_source!`); for a default + /// privilege, a creating role it acts for. What a source out of reach gave is not revocable + /// from here; privileges only other sources gave still are. + pub reachable: bool, +} + +#[derive(Serialize, Debug)] +pub struct DatatableAclInfo { + /// Under the same names as [`AclGrant::grantee`]. + pub owner: String, + /// The roles a change may name: `admin`, then every role of the instance catalog. Only for a + /// caller who may change anything, as the catalog is in the permissions drawer. + pub roles: Vec, + /// Whether this caller may plan and apply changes: they administer the data table, on an + /// edition that has the planner. + pub editable: bool, + /// Whether the server is Postgres 17 or later, which added the `MAINTAIN` table privilege. + pub supports_maintain: bool, + /// The database the target lives in, which no target carries itself. + pub dbname: String, + pub grants: Vec, + /// What the target holds that is a target of its own: a database's schemas, a schema's tables. + pub children: Vec, +} + +#[derive(Serialize, Debug)] +pub struct AclPlan { + pub statements: Vec, + pub warnings: Vec, +} + +/// The Postgres role a role name stands for. A data table role is a login named exactly like the +/// role, so this is the identity — except `admin`, which is `custom_instance_user`. +/// +/// Anything else is refused, never resolved to some default: every statement a plan writes names +/// the role it is about. +pub(crate) fn pg_role_of(name: &str, catalog: &DatatableRoleCatalog) -> Result { + if name == ADMIN_DATATABLE_ROLE { + return Ok(CUSTOM_INSTANCE_USER.to_string()); + } + if catalog.values().any(|r| r.name == name) { + return Ok(name.to_string()); + } + Err(Error::BadRequest(format!( + "'{name}' is not a data table role of this instance" + ))) +} + +/// The reverse of [`pg_role_of`], for display. A role that is not a data table role reads back as +/// itself. +pub(crate) fn role_name_of(pg_role: &str) -> String { + if pg_role == CUSTOM_INSTANCE_USER { + ADMIN_DATATABLE_ROLE.to_string() + } else { + pg_role.to_string() + } +} + +/// Every role a change may name: `admin` first, then the catalog. +fn role_names(catalog: &DatatableRoleCatalog) -> Vec { + let mut names: Vec = catalog.values().map(|r| r.name.clone()).collect(); + names.sort(); + names.insert(0, ADMIN_DATATABLE_ROLE.to_string()); + names +} + +fn ensure_instance(governing: &GoverningDatatable) -> Result<()> { + if governing.is_instance() { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Data table '{}' is backed by a Postgres resource, so its access is managed on that \ + server directly. Only a data table on the Windmill instance's own database has data \ + table roles to grant to.", + governing.name + ))) +} + +/// The data table's `admin` connection, and the notices Postgres sends on it. +/// +/// Authorization: connects as `custom_instance_user` with the instance's own credentials and checks +/// nothing. Callers MUST have authorized the request first — a request about to be refused must +/// not get as far as this connection. +async fn connect_as_admin_unchecked( + db: &DB, + governing: &GoverningDatatable, +) -> Result<( + tokio_postgres::Client, + mpsc::UnboundedReceiver, + String, +)> { + ensure_instance(governing)?; + // Built from the authorized entry, never by resolving the settings again: a save in between + // could point the entry at a resource on another server and back, and this connection would + // then alter a database the later checks of the entry never see. + let mut pg = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + pg.dbname = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database") + .resource_path + .clone(); + pg.user = Some(CUSTOM_INSTANCE_USER.to_string()); + pg.password = Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); + let dbname = pg.dbname.clone(); + let (client, mut connection) = pg.connect(Some(db)).await?; + // Unbounded: the driver must never wait on the receiver, which only drains once the statement + // the driver is carrying has completed. + let (notices_tx, notices) = mpsc::unbounded_channel(); + tokio::spawn(async move { + loop { + match std::future::poll_fn(|cx| connection.poll_message(cx)).await { + Some(Ok(AsyncMessage::Notice(notice))) => { + let _ = notices_tx.send(notice); + } + Some(Ok(_)) => {} + Some(Err(e)) => { + tracing::error!("Datatable ACL connection error: {e}"); + break; + } + None => break, + } + } + }); + Ok((client, notices, dbname)) +} + +/// An object whose ownership follows the schema's. +#[derive(Debug, PartialEq)] +pub(crate) struct OwnedObject { + /// The keyword `ALTER ... OWNER TO` takes for this kind of object. + pub(crate) keyword: &'static str, + /// How Postgres names the object (`pg_identify_object`): schema-qualified, quoted where + /// needed, with a routine's arguments or an operator class's access method. It goes into the + /// statement as it is. + pub(crate) identity: String, +} + +/// Everything a schema's change of owner takes along, in schema `$1`, as (kind, identity, owner +/// oid), the first two as `pg_identify_object` gives them. The plan, the check of what this +/// connection may move and the check after the move all read this one list, so what is moved and +/// what is checked cannot differ. +/// +/// Read from what depends on the schema rather than catalog by catalog, so no kind of object is +/// left out by omission; array types, row types and indexes depend on another object instead. +/// Left out: an object's internal parts (a range type's constructors and multirange, an identity +/// column's sequence), a sequence tied to a column (it follows its table, and refuses an owner of +/// its own), an extension's members, and what has no `ALTER ... OWNER` at all (an extension, a text +/// search parser or template). Postgres records no owner for the bootstrap superuser, oid 10. +macro_rules! schema_owned_objects { + () => { + "SELECT o.type AS kind, o.identity, COALESCE(s.refobjid, 10::oid) AS owner + FROM pg_depend d + CROSS JOIN LATERAL pg_identify_object(d.classid, d.objid, 0) o + LEFT JOIN pg_shdepend s + ON s.dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND s.classid = d.classid AND s.objid = d.objid AND s.deptype = 'o' + WHERE d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n' + AND d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND d.classid <> ALL(ARRAY['pg_extension'::regclass, 'pg_ts_parser'::regclass, + 'pg_ts_template'::regclass]::oid[]) + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = d.classid AND x.objid = d.objid AND x.objsubid = 0 + AND (x.deptype IN ('i', 'e') + OR (x.deptype = 'a' AND d.classid = 'pg_class'::regclass + AND x.refobjsubid <> 0)))" + }; +} + +/// The keyword `ALTER ... OWNER TO` takes for a kind of object, as `pg_identify_object` names the +/// kind. A kind missing here is refused rather than skipped, which would leave it behind. +fn owned_keyword(kind: &str) -> Option<&'static str> { + Some(match kind { + "table" => "TABLE", + "view" => "VIEW", + "materialized view" => "MATERIALIZED VIEW", + "sequence" => "SEQUENCE", + "foreign table" => "FOREIGN TABLE", + "type" => "TYPE", + "function" | "procedure" | "aggregate" => "ROUTINE", + "collation" => "COLLATION", + "conversion" => "CONVERSION", + "operator" => "OPERATOR", + "operator class" => "OPERATOR CLASS", + "operator family" => "OPERATOR FAMILY", + "statistics object" => "STATISTICS", + "text search dictionary" => "TEXT SEARCH DICTIONARY", + "text search configuration" => "TEXT SEARCH CONFIGURATION", + _ => return None, + }) +} + +async fn read_owned_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + concat!( + "SELECT kind, identity FROM (", + schema_owned_objects!(), + ") o ORDER BY kind, identity" + ), + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + rows.into_iter() + .map(|row| { + let kind: &str = row.get(0); + let identity: String = row.get(1); + match owned_keyword(kind) { + Some(keyword) => Ok(OwnedObject { keyword, identity }), + None => Err(Error::BadRequest(format!( + "{identity} is a {kind}, whose owner cannot be changed from here, and schema \ + {schema} would change hands without it. Move it to another schema first." + ))), + } + }) + .collect() +} + +/// Default privileges set database-wide (no `IN SCHEMA`), as `(defaclrole, defaclobjtype, grantee, +/// privilege_type)`. Postgres stores such an entry as a whole acl, the creator's own privileges +/// included, so only what goes beyond the built-in default is a grant. It applies in every schema +/// on top of the schema's own defaults, which cannot take it back. +macro_rules! database_wide_defaults { + () => { + "SELECT d.defaclrole, d.defaclobjtype, a.grantee, a.privilege_type + FROM pg_default_acl d, aclexplode(d.defaclacl) a + WHERE d.defaclnamespace = 0 + AND NOT EXISTS ( + SELECT 1 FROM aclexplode(acldefault( + CASE d.defaclobjtype WHEN 'S' THEN 's' ELSE d.defaclobjtype END, d.defaclrole)) x + WHERE x.grantee = a.grantee AND x.privilege_type = a.privilege_type)" + }; +} + +/// `TABLES`, `SEQUENCES`, `FUNCTIONS` or `TYPES`, as a `defaclobjtype` names them. +fn default_objects_keyword(objtype: &str) -> &'static str { + match objtype { + "r" => "TABLES", + "S" => "SEQUENCES", + "T" => "TYPES", + _ => "FUNCTIONS", + } +} + +/// What a schema's owner holds on what gets created there later. A change of owner hands the new +/// owner the same and takes these back: otherwise every former owner keeps reaching whatever the +/// other roles create there. +#[derive(Debug, PartialEq)] +pub(crate) struct FormerOwnerDefaults { + pub(crate) pg_role: String, + /// (creating role, `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES`), one per default privilege + /// it holds. + pub(crate) defaults: Vec<(String, &'static str)>, +} + +/// The default privileges schema `schema`'s owner holds there — `None` when it holds none, or is +/// `new_owner` already. Refused when one was set by a role this connection cannot act for: only +/// a member of the creating role may change its defaults, so the revoke would fail at apply. Also +/// refused when the owner holds defaults set database-wide, which no change to this schema takes +/// back. +async fn read_former_owner_defaults( + client: &tokio_postgres::Client, + schema: &str, + new_owner: &str, +) -> Result> { + let database_wide = client + .query_opt( + concat!( + "SELECT pg_get_userbyid(n.nspowner), pg_get_userbyid(g.defaclrole), + g.defaclobjtype::text + FROM pg_namespace n, (", + database_wide_defaults!(), + ") g + WHERE n.nspname = $1 AND g.grantee = n.nspowner + AND g.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3 + LIMIT 1" + ), + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the database-wide default privileges: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = database_wide { + let pg_role: String = row.get(0); + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges set database-wide by {}, which no default of schema \ + {schema} takes back, so it would keep reaching what is created here after the change \ + of owner. Revoke them database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + role_name_of(&creator), + quote_ident(&creator), + default_objects_keyword(row.get(2)), + quote_ident(&pg_role) + ))); + } + + let rows = client + .query( + "SELECT DISTINCT pg_get_userbyid(n.nspowner), pg_get_userbyid(d.defaclrole), + d.defaclobjtype::text, pg_has_role(d.defaclrole, 'USAGE') + FROM pg_namespace n + JOIN pg_default_acl d ON d.defaclnamespace = n.oid + CROSS JOIN LATERAL aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND a.grantee = n.nspowner + -- What the owner gives itself is about the objects it creates, not about owning + -- the schema, so a move leaves it alone. + AND a.grantee <> d.defaclrole + AND d.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3", + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the default privileges of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + let Some(first) = rows.first() else { + return Ok(None); + }; + let pg_role: String = first.get(0); + let plural = |row: &tokio_postgres::Row| default_objects_keyword(row.get(2)); + if let Some(row) = rows.iter().find(|row| !row.get::<_, bool>(3)) { + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges in schema {schema} from {creator}, which this data \ + table's connection cannot act for, so they would outlive the change of owner. \ + Revoke them as {creator} first: ALTER DEFAULT PRIVILEGES FOR ROLE {} IN SCHEMA {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + quote_ident(&creator), + quote_ident(schema), + plural(row), + quote_ident(&pg_role) + ))); + } + Ok(Some(FormerOwnerDefaults { + defaults: rows + .iter() + .map(|row| (row.get::<_, String>(1), plural(row))) + .collect(), + pg_role, + })) +} + +/// What the catalog holds that a plan depends on, read before planning so the planner stays pure. +#[derive(Debug, Default, PartialEq)] +pub(crate) struct CatalogFacts { + /// Every role that may create objects here, but the one the change is about. + pub(crate) other_pg_roles: Vec, + /// For a schema's change of owner: what moves along with it. + pub(crate) existing_objects: Vec, + /// For a schema's change of owner: the defaults it takes back from the owner it replaces. + pub(crate) former_owner: Option, + /// For a revoke: each grant it takes back. + pub(crate) revoked_grants: Vec, +} + +/// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the +/// target itself when `None`), or for a default privilege on what `source` creates later. +#[derive(Debug, PartialEq)] +pub(crate) struct RevokedGrant { + pub(crate) object: Option, + /// The role that made the grant: its grantor, or the creating role of a default privilege. + pub(crate) source: String, + pub(crate) privileges: Vec, +} + +/// An `aclexplode` row's source and whether this connection can take back what it gave, on an +/// object owned by `$owner`. A REVOKE speaks for the grantor Postgres picks itself — the owner, +/// when the connection acts for the owner, and otherwise the connection — and `GRANTED BY` names +/// nobody else, so a grant any other role made stays whatever the connection runs. +macro_rules! grant_source { + ($owner:literal) => { + concat!( + "pg_get_userbyid(a.grantor), a.grantor = CASE WHEN pg_has_role(", + $owner, + ", 'USAGE') THEN ", + $owner, + " ELSE (SELECT oid FROM pg_roles WHERE rolname = current_user) END" + ) + }; +} + +/// What relation `$2` of schema `$1` grants role `$3`, as (source, whether this connection can +/// take it back, privilege). +const RELATION_GRANTS: &str = concat!( + "SELECT ", + grant_source!("c.relowner"), + ", a.privilege_type + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)" +); + +/// The grants a revoke takes back, read from the catalog rather than from the request: one per +/// object and source that gave `pg_role` any of `privileges`. Refused when a source's grant is out +/// of this connection's reach (`grant_source!`, or for a default privilege a creating role it does +/// not act for): the revoke would leave that grant in place. +async fn read_revoked_grants( + client: &tokio_postgres::Client, + dbname: &str, + target: &AclTarget, + scope: GrantScope, + objects: &[AclObject], + privileges: &[String], + pg_role: &str, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read what the revoke takes back: {}", + pg_error_message(&e) + )) + }; + let wanted: Vec = privileges.iter().map(|p| p.to_uppercase()).collect(); + // (object, how it reads in a refusal, one row per source and privilege) + let mut read = Vec::new(); + match (scope, target) { + (scope, AclTarget::Schema { schema }) if scope.is_future() => { + let (objtype, plural) = match scope { + GrantScope::FutureTables => ("r", "tables"), + GrantScope::FutureSequences => ("S", "sequences"), + _ => ("f", "functions"), + }; + let database_wide = client + .query( + concat!( + "SELECT pg_get_userbyid(g.defaclrole), g.privilege_type FROM (", + database_wide_defaults!(), + ") g + WHERE g.defaclobjtype::text = $1 + AND g.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1, 2" + ), + &[&objtype, &pg_role], + ) + .await + .map_err(read_error)?; + let mut still_granted: BTreeMap> = BTreeMap::new(); + for row in database_wide { + let privilege: String = row.get(1); + if wanted.contains(&privilege) { + still_granted.entry(row.get(0)).or_default().push(privilege); + } + } + if let Some((creator, taken)) = still_granted.into_iter().next() { + return Err(Error::BadRequest(format!( + "{} also receives {} on {plural} {} creates through a default privilege set \ + database-wide, which no default of schema {schema} takes back. Revoke it \ + database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} REVOKE {} ON {} \ + FROM {}", + role_name_of(pg_role), + taken.join(", "), + role_name_of(&creator), + quote_ident(&creator), + taken.join(", "), + default_objects_keyword(objtype), + quote_ident(pg_role) + ))); + } + let rows = client + .query( + "SELECT pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE'), + a.privilege_type + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND d.defaclobjtype::text = $2 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)", + &[schema, &objtype, &pg_role], + ) + .await + .map_err(read_error)?; + read.push(( + None, + format!("{plural} created later in schema {schema}"), + rows, + )); + } + (GrantScope::Target, _) if objects.is_empty() => { + let rows = match target { + AclTarget::Database => { + client + .query( + concat!( + "SELECT ", + grant_source!("d.datdba"), + ", a.privilege_type + FROM pg_database d, + aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $1)" + ), + &[&pg_role], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + concat!( + "SELECT ", + grant_source!("n.nspowner"), + ", a.privilege_type + FROM pg_namespace n, + aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2)" + ), + &[schema, &pg_role], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query(RELATION_GRANTS, &[schema, table, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push((None, target.label(dbname), rows)); + } + (GrantScope::Target, AclTarget::Schema { schema }) => { + for object in objects { + let rows = match object_keyword(&object.kind)? { + "ROUTINE" => { + client + .query( + concat!( + "SELECT ", + grant_source!("p.proowner"), + ", a.privilege_type + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND p.proname = $2 + AND pg_get_function_identity_arguments(p.oid) = $3 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $4)" + ), + &[ + schema, + &object.name, + &object.args.as_deref().unwrap_or(""), + &pg_role, + ], + ) + .await + } + _ => { + client + .query(RELATION_GRANTS, &[schema, &object.name, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push(( + Some(object.clone()), + format!("{} {schema}.{}", object.kind.to_lowercase(), object.name), + rows, + )); + } + } + // Every other scope and target is the planner's to refuse. + _ => {} + } + + let mut revoked = Vec::new(); + for (object, label, rows) in read { + let mut by_source: BTreeMap)> = BTreeMap::new(); + for row in rows { + let privilege: String = row.get(2); + if wanted.contains(&privilege) { + by_source + .entry(row.get(0)) + .or_insert_with(|| (row.get(1), vec![])) + .1 + .push(privilege); + } + } + for (source, (reachable, mut privileges)) in by_source { + if !reachable { + return Err(Error::BadRequest(format!( + "{} on {label} was granted to {} by {source}, and Postgres takes a grant \ + back only through the role that made it, which this data table's \ + connection cannot speak for here. Revoke it as {source}.", + privileges.join(", "), + role_name_of(pg_role), + ))); + } + privileges.sort(); + privileges.dedup(); + revoked.push(RevokedGrant { object: object.clone(), source, privileges }); + } + } + Ok(revoked) +} + +/// The keyword a `REVOKE ... ON` takes for one object, checked rather than interpolated: it lands +/// in SQL unquoted. +pub(crate) fn object_keyword(kind: &str) -> Result<&'static str> { + match kind.to_uppercase().as_str() { + "TABLE" | "VIEW" | "MATERIALIZED VIEW" | "FOREIGN TABLE" => Ok("TABLE"), + "SEQUENCE" => Ok("SEQUENCE"), + // `FUNCTION` names no procedure; `ROUTINE` names either. + "FUNCTION" | "PROCEDURE" | "ROUTINE" => Ok("ROUTINE"), + other => Err(Error::BadRequest(format!("Unknown object kind '{other}'"))), + } +} + +/// Every object of a schema, named the way the catalog names it. +async fn read_schema_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + "SELECT CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, c.relname, NULL::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + UNION ALL + SELECT CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, p.proname, + pg_get_function_identity_arguments(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = $1", + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + Ok(rows + .into_iter() + .map(|row| AclObject { kind: row.get(0), name: row.get(1), args: row.get(2) }) + .collect()) +} + +/// Replace the objects a revoke names with the catalog's own entry for each. +/// +/// A routine is identified by its argument types, and those go into the statement as written — +/// there is no quoting for them — so the request may name an object but never spell one: what +/// reaches the SQL is read back from Postgres. An object that resolves to nothing is refused rather +/// than dropped, since a revoke that silently covers less than it says is worse than an error. +async fn resolve_acl_objects( + client: &tokio_postgres::Client, + target: &AclTarget, + objects: &[AclObject], +) -> Result> { + if objects.is_empty() { + return Ok(vec![]); + } + let Some(schema) = target.schema() else { + return Err(Error::BadRequest( + "A database has no objects of its own to revoke on".to_string(), + )); + }; + let known = read_schema_objects(client, schema).await?; + objects + .iter() + .map(|requested| { + let keyword = object_keyword(&requested.kind)?; + known + .iter() + .find(|k| { + k.name == requested.name + && k.args == requested.args + && object_keyword(&k.kind).is_ok_and(|k| k == keyword) + }) + .cloned() + .ok_or_else(|| { + Error::NotFound(format!( + "'{}' is not an object of schema '{schema}'", + requested.name + )) + }) + }) + .collect() +} + +async fn read_owner(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let row = match target { + AclTarget::Database => { + client + .query_opt( + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query_opt( + // `public` is owned by `pg_database_owner`, a placeholder role whose membership + // is whoever owns the database — naming it back would say nothing, so resolve + // it to that owner. + "SELECT pg_get_userbyid(owner) FROM ( + SELECT CASE WHEN n.nspowner = (SELECT oid FROM pg_roles WHERE rolname = 'pg_database_owner') + THEN (SELECT d.datdba FROM pg_database d WHERE d.datname = current_database()) + ELSE n.nspowner END AS owner + FROM pg_namespace n WHERE n.nspname = $1 + ) o", + &[schema], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2", + &[schema, table], + ) + .await + } + } + .map_err(|e| Error::internal_err(format!("Failed to read the owner: {}", pg_error_message(&e))))?; + Ok(row.map(|row| row.get(0))) +} + +async fn read_children(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let rows = match target { + AclTarget::Database => { + client + .query( + "SELECT nspname::text FROM pg_namespace + WHERE nspname <> 'information_schema' AND nspname NOT LIKE 'pg\\_%' + ORDER BY nspname", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + "SELECT c.relname::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p']::\"char\"[]) + ORDER BY c.relname", + &[schema], + ) + .await + } + AclTarget::Table { .. } => return Ok(vec![]), + } + .map_err(|e| { + Error::internal_err(format!( + "Failed to list what the target holds: {}", + pg_error_message(&e) + )) + })?; + Ok(rows.into_iter().map(|row| row.get(0)).collect()) +} + +async fn get_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let target: AclTarget = query.try_into()?; + let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; + ensure_reaches_governing_datatable(&db, &w_id, &datatable_name, &governing, &authed).await?; + ensure_instance(&governing)?; + let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) + .await + .is_ok(); + let roles = if editable { + role_names(&read_role_catalog(&db).await?) + } else { + vec![] + }; + + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + let owner = read_owner(&client, &target) + .await? + .ok_or_else(|| Error::NotFound(format!("{} not found", target.label(&dbname))))?; + let grants = read_grants(&client, &target).await?; + let supports_maintain: bool = client + .query_one( + "SELECT current_setting('server_version_num')::int >= 170000", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the server version: {}", + pg_error_message(&e) + )) + })? + .get(0); + let children = read_children(&client, &target).await?; + + Ok(Json(DatatableAclInfo { + owner: role_name_of(&owner), + roles, + editable, + supports_maintain, + dbname, + grants, + children, + })) +} + +async fn read_grants(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + // `aclexplode` turns an acl array into one row per (grantee, privilege); grantee 0 is PUBLIC, + // which has no name to resolve. A NULL acl is not "no access" but Postgres's built-in default — + // the owner holds everything and, on a routine, PUBLIC may EXECUTE — hence `acldefault`. The + // owner's own entries are left out: what it holds comes with ownership, which the owner shows, + // not with a grant a revoke here could take back. Each row ends with its source — the grantor, + // or a default privilege's creating role — and whether this connection can take back what it + // gave. + // Column-level grants (`pg_attribute.attacl`) are not supported yet: they are neither read here + // nor revocable from the editor. + let mut rows = match target { + AclTarget::Database => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("d.datdba"), + " FROM pg_database d, aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() AND a.grantee <> d.datdba" + ), + &[], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + // Default privileges set database-wide reach what is created in every + // schema, so they are the database's to show rather than any schema's. + concat!( + "SELECT CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END, + g.privilege_type, NULL::text, + CASE g.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' WHEN 'n' THEN 'SCHEMAS' + ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(g.defaclrole), pg_has_role(g.defaclrole, 'USAGE') + FROM (", + database_wide_defaults!(), + ") g" + ), + &[], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Schema { schema } => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("n.nspowner"), + " FROM pg_namespace n, aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 AND a.grantee <> n.nspowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, c.relname, NULL::text, + CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, + NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee <> c.relowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // Routines carry their own acl in `pg_proc`; without this a grant made here + // would vanish on the next read and could never be revoked back. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, p.proname, NULL::text, + CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, + pg_get_function_identity_arguments(p.oid), ", + grant_source!("p.proowner"), + " FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND a.grantee <> p.proowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // `USAGE` on a type is what lets a role use it in a column. Only a type the + // schema holds in its own right has an acl: an array, a row type and a + // multirange answer to their element, table or range. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, t.typname, NULL::text, 'TYPE', NULL::text, ", + grant_source!("t.typowner"), + " FROM pg_type t + JOIN pg_depend d ON d.classid = 'pg_type'::regclass AND d.objid = t.oid + AND d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n', + aclexplode(COALESCE(t.typacl, acldefault('T', t.typowner))) a + WHERE d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND a.grantee <> t.typowner + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = 'pg_type'::regclass AND x.objid = t.oid + AND x.objsubid = 0 AND x.deptype = 'i')" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // What a creating role set is taken back `FOR ROLE` that role, which only + // a role acting for it may do. + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, + CASE d.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE') + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1", + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Table { schema, table } => client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 AND a.grantee <> c.relowner" + ), + &[schema, table], + ) + .await + .map_err(grant_read_error)?, + }; + + // One row per privilege and source — and, for default privileges, per creating role. Fold them + // back into one entry per grantee and object that keeps every source and what each gave: a + // revoke takes a privilege back from every source that gave it. + let mut folded: BTreeMap< + ( + String, + Option<(String, String, Option)>, + Option, + ), + (Vec, BTreeMap)>), + > = BTreeMap::new(); + for row in rows.drain(..) { + let grantee: String = row.get(0); + let privilege: String = row.get(1); + let object: Option = row.get(2); + let future: Option = row.get(3); + let object_kind: Option = row.get(4); + let object_args: Option = row.get(5); + let source: String = row.get(6); + let reachable: bool = row.get(7); + let (privileges, sources) = folded + .entry(( + role_name_of(&grantee), + object.map(|name| { + ( + name, + object_kind.unwrap_or_else(|| "TABLE".to_string()), + object_args, + ) + }), + future, + )) + .or_default(); + sources + .entry(role_name_of(&source)) + .or_insert_with(|| (reachable, vec![])) + .1 + .push(privilege.clone()); + privileges.push(privilege); + } + Ok(folded + .into_iter() + .map(|((grantee, object, future), (mut privileges, sources))| { + privileges.sort(); + privileges.dedup(); + AclGrant { + grantee, + privileges, + object: object.map(|(name, kind, args)| AclObject { name, kind, args }), + future, + sources: sources + .into_iter() + .map(|(role, (reachable, mut privileges))| { + privileges.sort(); + privileges.dedup(); + AclSource { role, privileges, reachable } + }) + .collect(), + } + }) + .collect()) +} + +fn grant_read_error(e: tokio_postgres::Error) -> Error { + Error::internal_err(format!("Failed to read grants: {}", pg_error_message(&e))) +} + +/// Changing a data table's access is administering it. Checked in full before anything connects +/// with the instance's credentials. +async fn authorize_acl_change( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + ensure_governs_datatable(db, authed, w_id, &governing).await?; + ensure_instance(&governing)?; + Ok(governing) +} + +static APPLY_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +/// A role passes on only privileges it holds with grant option, and an instance database +/// provisioned before data table roles gave `custom_instance_user` none. Adds that option to its +/// database and `public` privileges, and nothing else: default privileges are left alone, since a +/// schema's change of owner is planned against them. Best-effort, as a grant it fails to enable is +/// refused when it runs. +async fn ensure_grant_options(client: &tokio_postgres::Client, db: &DB, dbname: &str) { + let held = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CONNECT WITH GRANT OPTION') + AND has_database_privilege(current_database(), 'CREATE WITH GRANT OPTION') + AND (to_regnamespace('public') IS NULL + OR (has_schema_privilege('public', 'USAGE WITH GRANT OPTION') + AND has_schema_privilege('public', 'CREATE WITH GRANT OPTION')))", + &[], + ) + .await + .is_ok_and(|row| row.get::<_, bool>(0)); + if held { + return; + } + if let Err(e) = grant_options_as_server(db, dbname).await { + tracing::warn!("Could not enable grant options on '{dbname}': {e}"); + } +} + +/// Only the database's owner, the server's own Postgres user, can hand out an option it holds. +async fn grant_options_as_server(db: &DB, dbname: &str) -> Result<()> { + let server = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + let creds = PgDatabase { dbname: dbname.to_string(), ..server }; + let (client, connection) = creds.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let role = quote_ident(CUSTOM_INSTANCE_USER); + let result = client + .batch_execute(&format!( + "GRANT CONNECT, CREATE ON DATABASE {} TO {role} WITH GRANT OPTION; + DO $$ BEGIN + IF to_regnamespace('public') IS NOT NULL THEN + GRANT USAGE, CREATE ON SCHEMA public TO {role} WITH GRANT OPTION; + END IF; + END $$;", + quote_ident(dbname) + )) + .await; + drop(client); + windmill_common::shutdown_pg_connection(join_handle).await?; + result.map_err(|e| { + Error::internal_err(format!( + "Failed to grant options on '{dbname}': {}", + pg_error_message(&e) + )) + }) +} + +/// Whether the governing entry an apply was authorized on is still the one in the settings, read +/// under the lock: a save in between could have pointed it at another database or changed its roles. +fn entry_unchanged(governing: &GoverningDatatable, entry_now: Option) -> bool { + let Some(Ok(now)) = entry_now.map(serde_json::from_value::) else { + return false; + }; + match ( + serde_json::to_value(&now), + serde_json::to_value(&governing.datatable), + ) { + (Ok(now), Ok(authorized)) => now == authorized, + _ => false, + } +} + +/// Plan one change against the catalog and the database as they are now. +async fn build_plan( + client: &tokio_postgres::Client, + dbname: &str, + catalog: &DatatableRoleCatalog, + target: &AclTarget, + change: &AclChange, +) -> Result { + let role = change.role(); + // `admin` is the login the data table itself reaches Postgres through, and the one every + // change here runs as: a revoke that lands leaves nothing able to grant it back. + if matches!(change, AclChange::Revoke { .. }) && role == ADMIN_DATATABLE_ROLE { + return Err(Error::BadRequest(format!( + "'{ADMIN_DATATABLE_ROLE}' is how this data table reaches its database; \ + its own access is not revocable from here" + ))); + } + let pg_role = pg_role_of(role, catalog)?; + let change = match change { + AclChange::Revoke { role, privileges, scope, objects } => AclChange::Revoke { + role: role.clone(), + privileges: privileges.clone(), + scope: *scope, + objects: resolve_acl_objects(client, target, objects).await?, + }, + change => change.clone(), + }; + // Default privileges are recorded per creating role, and a schema's new owner is kept in reach + // of what the others create there, so both are written for every role there is. + let other_pg_roles = role_names(catalog) + .iter() + .map(|name| pg_role_of(name, catalog)) + .filter(|r| r.as_ref().map_or(true, |r| *r != pg_role)) + .collect::>>()?; + let (existing_objects, former_owner) = match (&change, target) { + (AclChange::SetOwner { .. }, AclTarget::Schema { schema }) => ( + read_owned_objects(client, schema).await?, + read_former_owner_defaults(client, schema, &pg_role).await?, + ), + _ => (vec![], None), + }; + let revoked_grants = match &change { + AclChange::Revoke { privileges, scope, objects, .. } => { + read_revoked_grants( + client, dbname, target, *scope, objects, privileges, &pg_role, + ) + .await? + } + _ => vec![], + }; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some((object, owner)) = unmanaged_owner(client, target).await? { + return Err(Error::BadRequest(format!( + "{object} is owned by {owner}, which this data table's connection cannot act \ + for, so its owner cannot be changed from here" + ))); + } + } + let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let mut plan = + crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some(missing) = missing_owner_privilege(client, target, &pg_role).await? { + plan.warnings.push(format!( + "{role} does not have {missing}, which Postgres requires of a new owner, so this \ + will be refused. Grant it first." + )); + } + } + Ok(plan) +} + +/// Postgres only hands an object to a role that could have created it: a table to one with +/// `CREATE` on its schema, a schema to one with `CREATE` on the database. +async fn missing_owner_privilege( + client: &tokio_postgres::Client, + target: &AclTarget, + pg_role: &str, +) -> Result> { + let (row, missing) = match target { + AclTarget::Table { schema, .. } => ( + client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'CREATE')", + &[&pg_role, schema], + ) + .await, + format!("CREATE on schema {schema}"), + ), + AclTarget::Schema { .. } => ( + client + .query_one( + "SELECT has_database_privilege($1::name, current_database(), 'CREATE')", + &[&pg_role], + ) + .await, + "CREATE on the database".to_string(), + ), + AclTarget::Database => return Ok(None), + }; + let has: bool = row + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the new owner's privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + Ok((!has).then_some(missing)) +} + +/// The first thing a change of owner would move that this connection cannot act for, with its +/// owner. Postgres lets only a member of the current owner move an object, and every change runs as +/// `custom_instance_user`, so an object it does not hold the owner of — `public`, owned by the +/// database's owner, above all — is refused here rather than at apply. Running as the instance's +/// own user instead would reach objects Windmill never created. +async fn unmanaged_owner( + client: &tokio_postgres::Client, + target: &AclTarget, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read who owns what the change moves: {}", + pg_error_message(&e) + )) + }; + match target { + AclTarget::Database => Ok(None), + AclTarget::Table { schema, table } => { + let row = client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 AND NOT pg_has_role(c.relowner, 'USAGE')", + &[schema, table], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| (format!("{schema}.{table}"), row.get(0)))) + } + AclTarget::Schema { schema } => { + let row = client + .query_opt( + concat!( + "SELECT ord, identity, pg_get_userbyid(owner) FROM ( + SELECT 0 AS ord, NULL::text AS identity, n.nspowner AS owner + FROM pg_namespace n WHERE n.nspname = $1 + UNION ALL + SELECT 1, identity, owner FROM (", + schema_owned_objects!(), + ") m + ) o WHERE NOT pg_has_role(owner, 'USAGE') ORDER BY ord, identity LIMIT 1" + ), + &[schema], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| { + let label = row + .get::<_, Option>(1) + .unwrap_or_else(|| format!("schema {schema}")); + (label, row.get(2)) + })) + } + } +} + +async fn plan_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + let catalog = read_role_catalog(&db).await?; + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + Ok(Json( + build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?, + )) +} + +async fn apply_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> Result { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let confirmed = req.statements.as_ref().ok_or_else(|| { + Error::BadRequest( + "An apply runs exactly the statements its plan showed; plan the change first" + .to_string(), + ) + })?; + // Everything that needs the pool happens before the locks: once `tx` holds them, a second pool + // connection could wait forever on a pool that concurrent applies, queued on the same locks, + // have exhausted. + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + // Applies queue on an instance-wide lock while each holds a direct connection to the instance's + // Postgres; unbounded, the queue alone could exhaust its connection limit. One at a time per + // server, and the ones waiting hold no connection at all. + let _slot = APPLY_SLOT + .acquire() + .await + .map_err(|e| Error::internal_err(format!("ACL apply slot closed: {e}")))?; + let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + ensure_grant_options(&client, &db, &dbname).await; + + // Held until the change is committed: a role renamed or dropped meanwhile would change what + // the plan names, and a settings save could move the entry onto another database. Taken in the + // same order as the permissions save, so the two cannot deadlock. + let mut tx = db.begin().await?; + lock_role_catalog(&mut tx).await?; + let entry_now = sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&governing.workspace_id) + .bind(&governing.name) + .fetch_optional(&mut *tx) + .await? + .flatten(); + let catalog = read_role_catalog_tx(&mut tx).await?; + + let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; + if !entry_unchanged(&governing, entry_now) || &plan.statements != confirmed { + return Err(Error::BadRequest( + "The data table or its roles changed since this was planned, so it would no longer \ + run what was confirmed. Plan it again." + .to_string(), + )); + } + + // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two + // different roles. + let pg_tx = client.transaction().await.map_err(|e| { + Error::internal_err(format!( + "Failed to open a transaction on the data table: {}", + pg_error_message(&e) + )) + })?; + for statement in &plan.statements { + pg_tx.batch_execute(statement).await.map_err(|e| { + Error::ExecutionErr(format!( + "Failed to run `{statement}`: {}", + pg_error_message(&e) + )) + })?; + // A privilege the connection cannot pass on is only a warning to Postgres, which then + // carries on having changed nothing. Returning drops the transaction, rolling back + // everything before it. + while let Ok(notice) = notices.try_recv() { + if *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_GRANTED + || *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_REVOKED + { + return Err(Error::ExecutionErr(format!( + "`{statement}` did not take effect ({}), so nothing was applied", + notice.message() + ))); + } + } + } + + // A schema's objects were listed before the transaction opened; one committed since would stay + // with its old owner. One created while this transaction is still open can still slip past, as + // Postgres has no lock that holds creation in a schema back. That is benign: it stays with its + // creator, like anything created there later, and moving that table fixes it. + if let (AclChange::SetOwner { role }, AclTarget::Schema { schema }) = (&req.change, &req.target) + { + let new_owner = pg_role_of(role, &catalog)?; + let straggler = pg_tx + .query_opt( + concat!( + "SELECT identity FROM (", + schema_owned_objects!(), + ") m WHERE owner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1 LIMIT 1" + ), + &[schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to check what the schema holds: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = straggler { + return Err(Error::BadRequest(format!( + "{} appeared while this ran and would keep its old owner, so nothing was \ + applied. Plan it again.", + row.get::<_, String>(0) + ))); + } + } + + let target_label = req.target.label(&dbname); + audit_log( + &mut *tx, + &authed, + "workspaces.datatable_acl", + ActionKind::Update, + &governing.workspace_id, + Some(&governing.name), + Some( + [ + ("target", target_label.as_str()), + ("change", change_kind(&req.change)), + ("role", req.change.role()), + ] + .into(), + ), + ) + .await?; + pg_tx.commit().await.map_err(|e| { + Error::internal_err(format!( + "Failed to commit the changes: {}", + pg_error_message(&e) + )) + })?; + tx.commit().await?; + + windmill_common::feature_usage::log_feature_usage( + "datatable", + "acl_applied", + change_kind(&req.change), + ); + + Ok(format!("Updated access on {target_label}")) +} + +/// What kind of change, never what it named: the telemetry key and the audit's summary. +fn change_kind(change: &AclChange) -> &'static str { + match change { + AclChange::SetOwner { .. } => "owner", + AclChange::Grant { scope, .. } | AclChange::Revoke { scope, .. } if scope.is_future() => { + "default_privileges" + } + AclChange::Grant { .. } => "grant", + AclChange::Revoke { .. } => "revoke", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::datatable_roles::InstanceDatatableRole; + + #[test] + fn a_role_is_its_own_postgres_role_except_admin() { + let catalog: DatatableRoleCatalog = BTreeMap::from([( + "role1".to_string(), + InstanceDatatableRole { name: "analytics".to_string(), enabled: true, pwd: None }, + )]); + assert_eq!( + pg_role_of("admin", &catalog).unwrap(), + "custom_instance_user" + ); + assert_eq!(pg_role_of("analytics", &catalog).unwrap(), "analytics"); + assert_eq!(role_name_of("custom_instance_user"), "admin"); + assert_eq!(role_name_of("analytics"), "analytics"); + // A catalog id, a name the catalog lacks, or the admin login spelled out never stands for + // some other role. + for unknown in ["role1", "operator", "custom_instance_user", "PUBLIC", ""] { + assert!( + matches!(pg_role_of(unknown, &catalog), Err(Error::BadRequest(_))), + "{unknown}" + ); + } + } + + /// A connection to the test's own database, the way the handlers reach a data table's. + async fn catalog_client(pool: &sqlx::PgPool) -> tokio_postgres::Client { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + config.dbname(pool.connect_options().get_database().unwrap()); + let (client, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + client + } + + /// What a revoke takes back is read from the catalog, per object and source, and only the + /// privileges it asks for: the planner renders exactly this, and refuses an empty read. + #[sqlx::test(migrations = false)] + async fn a_revoke_reads_back_only_what_it_asks_for(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + // A predefined role, so that nothing is granted outside the test's own database. + client + .batch_execute( + "CREATE SCHEMA granted; + CREATE TABLE granted.g (id int); + GRANT SELECT, INSERT ON granted.g TO pg_read_all_data;", + ) + .await + .unwrap(); + let target = AclTarget::Table { schema: "granted".to_string(), table: "g".to_string() }; + let revoked = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(revoked.len(), 1, "{revoked:?}"); + assert_eq!(revoked[0].object, None); + assert_eq!(revoked[0].privileges, ["SELECT"]); + let held_none = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["update".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert!(held_none.is_empty(), "{held_none:?}"); + // Each source says what it gave: that, and not the whole row, is what a revoke of some of + // its privileges is held back by. + let grants = read_grants( + &client, + &AclTarget::Schema { schema: "granted".to_string() }, + ) + .await + .unwrap(); + let on_g = grants + .iter() + .find(|g| { + g.grantee == "pg_read_all_data" && g.object.as_ref().is_some_and(|o| o.name == "g") + }) + .unwrap(); + assert_eq!(on_g.sources.len(), 1, "{:?}", on_g.sources); + assert_eq!(on_g.sources[0].privileges, ["INSERT", "SELECT"]); + assert!(on_g.sources[0].reachable); + } + + /// A default privilege set database-wide applies in every schema on top of the schema's own, + /// which cannot take it back: the database shows it, and neither a schema's revoke of it nor a + /// change of owner away from its grantee may go ahead as if it were gone. + #[sqlx::test(migrations = false)] + async fn a_schema_cannot_take_back_a_database_wide_default(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA owned AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA owned + GRANT SELECT, INSERT ON TABLES TO pg_read_all_data; + CREATE SCHEMA creators AUTHORIZATION pg_monitor; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_monitor + GRANT SELECT ON TABLES TO pg_read_all_stats;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + + let grants = read_grants(&client, &AclTarget::Database).await.unwrap(); + let database_wide = grants + .iter() + .find(|g| g.grantee == "pg_read_all_data" && g.future.as_deref() == Some("TABLES")) + .unwrap_or_else(|| panic!("{grants:?}")); + assert_eq!(database_wide.privileges, ["SELECT"]); + // A database-wide entry also holds its creator's own privileges, which come with creating + // and are no grant: neither a row nor a refusal may stem from them. + assert!( + !grants + .iter() + .any(|g| g.future.is_some() && (g.grantee == creator || g.grantee == "pg_monitor")), + "{grants:?}" + ); + assert_eq!( + database_wide.sources.len(), + 1, + "{:?}", + database_wide.sources + ); + assert_eq!(database_wide.sources[0].role, creator); + + let schema = AclTarget::Schema { schema: "owned".to_string() }; + let still_granted = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await; + assert!( + matches!(&still_granted, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{still_granted:?}" + ); + let schema_only = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["insert".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(schema_only.len(), 1, "{schema_only:?}"); + assert_eq!(schema_only[0].privileges, ["INSERT"]); + + let moved = read_former_owner_defaults(&client, "owned", "pg_write_all_data").await; + assert!( + matches!(&moved, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{moved:?}" + ); + let moved_from_creator = + read_former_owner_defaults(&client, "creators", "pg_write_all_data").await; + assert!( + matches!(moved_from_creator, Ok(None)), + "{moved_from_creator:?}" + ); + } + + /// Defaults on types go with a schema's owner like the other kinds: once a database-wide + /// default takes PUBLIC's USAGE on types away, they are all that reaches a new type. + #[sqlx::test(migrations = false)] + async fn a_schemas_former_owner_defaults_include_types(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA typed AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA typed GRANT USAGE ON TYPES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_read_all_data IN SCHEMA typed + GRANT SELECT ON TABLES TO pg_read_all_data;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + let former = read_former_owner_defaults(&client, "typed", "pg_write_all_data") + .await + .unwrap(); + assert_eq!( + former, + Some(FormerOwnerDefaults { + pg_role: "pg_read_all_data".to_string(), + defaults: vec![(creator, "TYPES")], + }) + ); + } + + /// A kind of object the list misses stays with its old owner while the schema changes hands, + /// which only a real catalog shows. + #[sqlx::test(migrations = false)] + async fn a_schemas_owner_change_takes_every_object_in_it(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA moved; + CREATE TABLE moved.t (id serial PRIMARY KEY, a int, b int); + CREATE STATISTICS moved.st ON a, b FROM moved.t; + CREATE TABLE moved.pt (id int) PARTITION BY RANGE (id); + CREATE TABLE moved.pt1 PARTITION OF moved.pt FOR VALUES FROM (0) TO (10); + CREATE TYPE moved.r AS RANGE (subtype = float8); + CREATE TYPE moved.pair AS (a int, b int); + CREATE TYPE moved.mood AS ENUM ('ok'); + CREATE DOMAIN moved.tags AS text[]; + CREATE COLLATION moved.coll (provider = libc, locale = 'C');", + ) + .await + .unwrap(); + let owned = read_owned_objects(&client, "moved").await.unwrap(); + // Not the serial's sequence, the range's constructors and multirange, or any array or row + // type: each follows the object it belongs to. + assert_eq!( + owned + .iter() + .map(|o| (o.keyword, o.identity.as_str())) + .collect::>(), + [ + ("COLLATION", "moved.coll"), + ("STATISTICS", "moved.st"), + ("TABLE", "moved.pt"), + ("TABLE", "moved.pt1"), + ("TABLE", "moved.t"), + ("TYPE", "moved.mood"), + ("TYPE", "moved.pair"), + ("TYPE", "moved.r"), + ("TYPE", "moved.tags"), + ] + ); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl_oss.rs b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs new file mode 100644 index 0000000000..59eb8cdbe0 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the ACL planner comes from: the enterprise one, or a refusal. +//! +//! Data table roles are an Enterprise Edition feature, and so is everything here — reading who +//! owns what included. `private` alone is not that edition — community builds carry it — so the +//! planner is behind `enterprise` as well. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_acl_ee::plan_statements; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_datatable_acl_available() -> windmill_common::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use { + crate::datatable_acl::{AclChange, AclPlan, AclTarget, CatalogFacts}, + windmill_common::{datatable_roles_oss::datatable_roles_unavailable, error::Result}, +}; + +/// Checked first by every ACL route, before anything is read or connected to. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn ensure_datatable_acl_available() -> Result<()> { + Err(datatable_roles_unavailable()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn plan_statements( + _target: &AclTarget, + _change: &AclChange, + _dbname: &str, + _pg_role: &str, + _facts: &CatalogFacts, +) -> Result { + Err(datatable_roles_unavailable()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index 5cb1f3c1c3..69e04e328d 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -65,3 +65,15 @@ pub(crate) async fn ensure_reaches_datatable( ) -> Result<()> { roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await } + +/// [`ensure_reaches_datatable`] against an entry already resolved, for a caller that goes on to +/// connect from that same entry. +pub(crate) async fn ensure_reaches_governing_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + governing: &GoverningDatatable, + authed: &ApiAuthed, +) -> Result<()> { + roles::ensure_reaches_governing_datatable(db, w_id, datatable_name, governing, authed).await +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs index f4d8c6a7ad..c7c7314885 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -12,8 +12,8 @@ #[cfg(all(feature = "private", feature = "enterprise"))] pub(crate) use crate::datatable_permissions_ee::{ - ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions, - list_usable_datatable_roles, set_datatable_permissions, + ensure_governs_datatable, ensure_reaches_datatable, ensure_reaches_governing_datatable, + get_datatable_permissions, list_usable_datatable_roles, set_datatable_permissions, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -56,6 +56,20 @@ mod ce { } } + pub(crate) async fn ensure_reaches_governing_datatable( + _db: &DB, + _w_id: &str, + _datatable_name: &str, + governing: &GoverningDatatable, + _authed: &ApiAuthed, + ) -> Result<()> { + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + // The routes stay registered so the API has one shape; each answers after authentication, // before anything is read. diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index c1eb6c1ffc..5dfdece644 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,6 +1,8 @@ #[cfg(feature = "parquet")] pub mod ai_session_backups; pub mod data_metrics; +pub mod datatable_acl; +pub mod datatable_acl_oss; pub mod datatable_migrations; pub mod datatable_permissions; pub mod datatable_permissions_oss; @@ -12,5 +14,8 @@ pub mod workspaces_oss; #[cfg(feature = "private")] pub mod workspaces_ee; +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_acl_ee; + #[cfg(all(feature = "private", feature = "enterprise"))] pub mod datatable_permissions_ee; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 88839633cd..70e023b537 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -144,6 +144,7 @@ pub fn workspaced_service() -> Router { ) .merge(crate::datatable_migrations::routes()) .merge(crate::datatable_permissions::routes()) + .merge(crate::datatable_acl::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) .route("/edit_git_sync_config", post(edit_git_sync_config)) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ee5d082d89..95736e63be 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5325,6 +5325,97 @@ paths: schema: type: string + /w/{workspace}/workspaces/datatable_acl/{datatable_name}: + get: + summary: read the owner and grants of an instance data table's database, schema or table + operationId: getDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: kind + in: query + required: true + schema: + type: string + enum: [database, schema, table] + - name: schema + in: query + required: false + schema: + type: string + - name: table + in: query + required: false + schema: + type: string + responses: + "200": + description: owner and grants + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableAclInfo" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/plan: + post: + summary: preview the SQL an ownership or grant change would run (data table administrators only) + operationId: planDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: statements that would run, in a single transaction + content: + application/json: + schema: + $ref: "#/components/schemas/AclPlan" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/apply: + post: + summary: run an ownership or grant change exactly as planned (data table administrators only) + operationId: applyDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: change applied + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}: get: summary: list the data table roles the caller may connect as @@ -33778,6 +33869,255 @@ components: datatable: type: string + AclTarget: + description: what access is read or changed on + oneOf: + - $ref: "#/components/schemas/AclTargetDatabase" + - $ref: "#/components/schemas/AclTargetSchema" + - $ref: "#/components/schemas/AclTargetTable" + discriminator: + propertyName: kind + mapping: + database: "#/components/schemas/AclTargetDatabase" + schema: "#/components/schemas/AclTargetSchema" + table: "#/components/schemas/AclTargetTable" + + AclTargetDatabase: + type: object + required: [kind] + properties: + kind: + type: string + enum: [database] + + AclTargetSchema: + type: object + required: [kind, schema] + properties: + kind: + type: string + enum: [schema] + schema: + type: string + + AclTargetTable: + type: object + required: [kind, schema, table] + properties: + kind: + type: string + enum: [table] + schema: + type: string + table: + type: string + + AclChange: + description: one change to plan or apply + oneOf: + - $ref: "#/components/schemas/AclChangeSetOwner" + - $ref: "#/components/schemas/AclChangeGrant" + - $ref: "#/components/schemas/AclChangeRevoke" + discriminator: + propertyName: type + mapping: + set_owner: "#/components/schemas/AclChangeSetOwner" + grant: "#/components/schemas/AclChangeGrant" + revoke: "#/components/schemas/AclChangeRevoke" + + AclChangeSetOwner: + type: object + description: >- + hands the target to role — for a schema, with everything already in it but an extension's + members, which stay with the extension + required: [type, role] + properties: + type: + type: string + enum: [set_owner] + role: + type: string + description: a data table role of the instance, or admin + + AclChangeGrant: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [grant] + role: + type: string + description: a data table role of the instance, or admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + + AclChangeRevoke: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [revoke] + role: + type: string + description: a data table role of the instance, other than admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + objects: + type: array + description: >- + objects inside the target the revoke covers, empty for the target itself. Only with the + target scope; a revoke on all objects of a kind is refused, since it cannot say which + grants it takes back. + items: + $ref: "#/components/schemas/AclObject" + + AclGrantScope: + type: string + enum: + [ + target, + all_tables, + all_sequences, + all_functions, + future_tables, + future_sequences, + future_functions, + ] + + AclChangeRequest: + type: object + required: [target, change] + properties: + target: + $ref: "#/components/schemas/AclTarget" + change: + $ref: "#/components/schemas/AclChange" + statements: + type: array + description: >- + The statements the plan showed. Required to apply, which plans again and refuses if + the result differs. + items: + type: string + + AclPlan: + type: object + required: [statements, warnings] + properties: + statements: + type: array + items: + type: string + warnings: + type: array + items: + type: string + + AclObject: + type: object + required: [name, kind] + properties: + name: + type: string + kind: + type: string + description: >- + TABLE, SEQUENCE, FUNCTION, PROCEDURE or TYPE — what the object is. A revoke turns it + into the keyword it takes, ROUTINE for both routine kinds; a type's grants are read + only. + args: + type: string + description: identity arguments of a routine, which is what tells two of the same name apart + + AclGrant: + type: object + required: [grantee, privileges, sources] + properties: + grantee: + type: string + privileges: + type: array + items: + type: string + object: + $ref: "#/components/schemas/AclObject" + future: + type: string + description: >- + set for a default privilege, naming the kind of object it covers (TABLES, SEQUENCES, + FUNCTIONS, TYPES, or SCHEMAS). On a schema, the defaults set in that schema; on the + database, the ones set database-wide, which apply in every schema and which no schema's + own defaults take back. + sources: + type: array + description: >- + the roles the grant comes from, each once — who granted it, or for a default privilege + the role whose future objects it covers. A revoke of some of the grant's privileges + takes them back from every source that gave them. + items: + $ref: "#/components/schemas/AclSource" + + AclSource: + type: object + required: [role, privileges, reachable] + properties: + role: + type: string + privileges: + type: array + description: >- + what role gave of the grant's privileges. A revoke is held back only by a source out of + reach that gave some of what it takes back. + items: + type: string + reachable: + type: boolean + description: >- + whether the data table's connection can take back what role gave. On an object that is + the owner, when the connection acts for the owner, and otherwise the connection itself; + for a default privilege, a creating role the connection acts for. What a source out of + reach gave is not revocable from here; privileges only other sources gave still are. + + DatatableAclInfo: + type: object + required: [owner, roles, editable, supports_maintain, dbname, grants, children] + properties: + owner: + type: string + roles: + type: array + description: the roles a change may name; empty unless the caller may change anything + items: + type: string + editable: + type: boolean + description: whether the caller may plan and apply changes + supports_maintain: + type: boolean + description: whether the server is Postgres 17+, which added the MAINTAIN table privilege + dbname: + type: string + description: the database the target lives in + grants: + type: array + items: + $ref: "#/components/schemas/AclGrant" + children: + type: array + description: a database's schemas, or a schema's tables + items: + type: string + CustomInstanceDb: type: object required: diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 52d1a85aec..a67bb6bfa1 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -1024,6 +1024,20 @@ impl Future for TokioPgConnection { } } +impl TokioPgConnection { + /// Drive the connection and hand back what the server sends outside of a query's response — + /// notices above all, which driving it as a future silently discards. + pub fn poll_message( + &mut self, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll>> { + match self { + TokioPgConnection::Tls(conn) => conn.poll_message(cx), + TokioPgConnection::NoTls(conn) => conn.poll_message(cx), + } + } +} + impl PgDatabase { /// The role the connection logs in as, whichever way it authenticates. pub fn login_name(&self) -> &str { diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 6ceab37872..abe37c3537 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1093,8 +1093,10 @@ migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that - import got, whether a pre-approved trial offer was opened, and whether data tables are - put under roles and whether callers name a role or take the default, last 30 days)
            • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1160,8 +1162,10 @@ migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that - import got, whether a pre-approved trial offer was opened, and whether data tables are - put under roles and whether callers name a role or take the default, last 30 days)
            • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte b/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte new file mode 100644 index 0000000000..cbfb43ad39 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte @@ -0,0 +1,50 @@ + + +
              + ({ value: t, label: t }))} + bind:value={table} + placeholder="The whole schema" + clearable + size="sm" + class="w-56" + /> + {/if} +
              diff --git a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte new file mode 100644 index 0000000000..e0ae596a3d --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte @@ -0,0 +1,289 @@ + + +{#if acl.error} + {errorText(acl.error)} +{:else if !info} + Loading… +{:else} +
              + {#if !info.editable} + + Read only: access is changed by the admins of the workspace that governs this data table, on + Windmill Enterprise Edition. + + {/if} + + {#if target.kind !== 'database'} +
              +
              + Owner + + {target.kind === 'schema' + ? 'The role that owns the schema and everything already in it, except what belongs to an extension, which stays with the extension. Changing it also keeps the new owner in reach of what the current roles create here from then on; a role added afterwards is not covered.' + : 'The role that owns the table. Its owner may always read and write it, and is who ALTER and DROP answer to.'} + +
              + {#if info.editable} + scope, + (s) => { + if (!s) return + scope = s + // A privilege only exists for some objects — SELECT means nothing on a function — + // so drop what the new scope cannot carry rather than send it. + const allowed = privilegesOf(s, target.kind, supportsMaintain) + privileges = privileges.filter((p) => allowed.includes(p)) + } + } + items={scopesOf(target.kind)} + {disabled} + size="sm" + class="w-52" + /> + TO + `Schema: ${s}`} - RightIcon={ChevronDownIcon} - placeholder="Search or create schema..." - showPlaceholderOnOpen - onCreateItem={(schema) => { - schema = schema.trim().replace(/[^a-zA-Z0-9_]/g, '') - if (dbType === 'snowflake') schema = schema.toUpperCase() - askingForConfirmation = { - confirmationText: `Create ${schema}`, - type: 'reload', - title: `This will run 'CREATE SCHEMA ${schema}' on your database. Are you sure ?`, - open: true, - id: 'db-create-schema-confirmation-modal', - onConfirm: async () => { - askingForConfirmation && (askingForConfirmation.loading = true) - try { - await dbSchemaOps.onCreateSchema({ schema }) - refresh?.() - selected.schemaKey = schema - } finally { - askingForConfirmation = undefined - } - } - } - }} - /> - {/if} - +
              -
              - {#if multiSelectMode} - - {#if dbSupportsSchemas} - +
              + + {#if datatableTreeLoading && (datatableTree?.length ?? 0) === 0} +
              + + Loading... +
              + {/if} + {#each treeRoots as root (root.datatable ?? '')} + {@const dtOpen = isExpanded(root.datatable)} + {#if root.datatable !== undefined} + {@const hasMenu = !multiSelectMode && onDatatableAction !== undefined} + {@const roleInfo = roleOf(root.datatable)} {/if} - {#each schemaKeys as schemaKey} - {@const schemaTables = getTablesForSchema(schemaKey)} - {@const isFullySelected = isSchemaFullySelected(schemaKey)} - {@const isPartiallySelected = isSchemaPartiallySelected(schemaKey)} - {@const hasNoTables = schemaTables.length === 0} - -
              { - if (!hasNoTables) { - toggleSchemaSelection(schemaKey) - } - }} - onkeydown={(e) => { - if (e.key === 'Enter' || e.key === ' ') { - if (!hasNoTables) { - toggleSchemaSelection(schemaKey) - } - } - }} - > - {#if hasNoTables} - - - {:else} - - e.stopPropagation()} - onchange={() => toggleSchemaSelection(schemaKey)} - /> - - {/if} - {schemaKey} - - {schemaTables.length} - - - -
              - - {#each schemaTables as tableKey} - {@const isDisabled = isTableDisabled(schemaKey, tableKey)} - {@const isChecked = isTableSelected(schemaKey, tableKey) || isDisabled} - {@const isCurrentPreview = - selected.schemaKey === schemaKey && selected.tableKey === tableKey} -
              { - selectTable(schemaKey, tableKey) - toggleTableSelection(schemaKey, tableKey) - }} - onkeydown={(e) => { - if (e.key === 'Enter' || e.key === ' ') { - selectTable(schemaKey, tableKey) - toggleTableSelection(schemaKey, tableKey) - } - }} - > - - e.stopPropagation()} - onchange={() => toggleTableSelection(schemaKey, tableKey)} - /> - - -

              {tableKey}

              - + {#if dtOpen} + {#if root.error} +

              {root.error}

              + {/if} + {#each root.schemas as sc (sc.schemaKey)} + {@const schemaOpen = isExpanded(root.datatable, sc.schemaKey)} + {@const indent = root.datatable !== undefined ? 'pl-7' : 'pl-3'} + {#if dbSupportsSchemas} -
              - {/each} - - - {/each} - {:else} - - {#each filteredTableKeys as tableKey} - - - {/each} - {/if} + ]} + btnId={'db-manager-schema-actions-' + onlyAlphaNumAndUnderscore(sc.schemaKey)} + /> + {/if} +
              + + {/if} + + + {#if schemaOpen || !dbSupportsSchemas} + {@const tableIndent = dbSupportsSchemas + ? root.datatable !== undefined + ? 'pl-11' + : 'pl-7' + : root.datatable !== undefined + ? 'pl-7' + : 'pl-3'} + {#each sc.tables as tableKey (tableKey)} + {@const entry = { + datatable: root.datatable, + schema: sc.schemaKey, + table: tableKey + }} + {@const hasMenu = !multiSelectMode} + {@const isSelected = + root.datatable === currentDatatable && + selected.schemaKey === sc.schemaKey && + selected.tableKey === tableKey} + + {/each} + {#if canCreateTableIn(root.datatable, sc.schemaKey)} + + {/if} + {/if} + + {/each} + {#if dbSupportsSchemas && search.trim() === '' && canCreateSchemaIn(root.datatable)} + + {/if} + {/if} + {/each}
              - {#if !multiSelectMode} - - {/if} - {#if tableKey && colDefs?.[tableKey]?.length} + {#if mainPane} + {@render mainPane()} + {:else if tableKey && colDefs?.[tableKey]?.length} {@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey, whereClause })} + (aclDrawer = undefined)}> + (aclDrawer = undefined)} + tooltip="Who owns this, and what each role may do with it." + > + {#if aclDrawer && workspace} + {@const dt = aclDrawer.datatable ?? currentDatatable} + {#if dt} + {#key `${dt}~${JSON.stringify(aclDrawer.target)}`} + + {/key} + {/if} + {/if} + + + (askingForConfirmation = undefined)} @@ -754,20 +1164,12 @@ - { - newSchemaDialogOpen = false - newSchemaName = '' - }} -> + { - newSchemaDialogOpen = false - newSchemaName = '' - }} - title="Create a new schema" + on:close={closeSchemaDialog} + title={schemaDialog?.mode === 'rename' + ? `Rename ${schemaDialog.schema}` + : 'Create a new schema'} >
              @@ -779,27 +1181,7 @@ placeholder="Enter schema name..." autofocus on:keydown={(e) => { - if (e.key === 'Enter' && sanitizedNewSchemaName && !schemaAlreadyExists) { - askingForConfirmation = { - confirmationText: `Create ${sanitizedNewSchemaName}`, - type: 'reload', - title: `This will run 'CREATE SCHEMA ${sanitizedNewSchemaName}' on your database. Are you sure?`, - open: true, - id: 'db-create-schema-confirmation-modal', - onConfirm: async () => { - askingForConfirmation && (askingForConfirmation.loading = true) - try { - await dbSchemaOps.onCreateSchema({ schema: sanitizedNewSchemaName }) - refresh?.() - selected.schemaKey = sanitizedNewSchemaName - newSchemaDialogOpen = false - newSchemaName = '' - } finally { - askingForConfirmation = undefined - } - } - } - } + if (e.key === 'Enter') submitSchemaName() }} /> {#if schemaAlreadyExists} @@ -814,32 +1196,8 @@
              {#snippet actions()} - {/snippet}               diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index 3ee98538ba..d4e00ce30b 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -1,16 +1,21 @@ + + (open = false) }} +> + { + // The row underneath folds on click, and picking a role is not that. + e.stopPropagation() + open = !open + }} + > + + {role} + + + anchorEl!.getBoundingClientRect())} + onSelectValue={(item) => { + open = false + if (item.value !== role) onSelect(item.value) + }} + /> + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index 3bda6aa216..0dab262fd0 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -6,8 +6,18 @@ import { joinSqlStatements, splitSqlRuns } from './sqlDdl' import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' + import { withMigrationRole } from './datatableMigrationRole' - let { workspace, datatable }: { workspace: string; datatable: string } = $props() + let { + workspace, + datatable, + role + }: { + workspace: string + datatable: string + /** The role the editor runs as. The migration declares it, or it would run as admin. */ + role?: string + } = $props() type Choice = 'run' | 'migrate' | 'cancel' @@ -73,7 +83,7 @@ function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: sql }) + newMigrationModal?.open({ codeUp: withMigrationRole(sql, role) }) }) } @@ -145,6 +155,11 @@ migrations rather than run ad-hoc. Create a migration for it instead? {/if}

              + {#if role} +

              + It will run as role {role}. +

              + {/if} 
              {promptSql}
            • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1163,9 +1164,10 @@ the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that import got, whether a pre-approved trial offer was opened, whether data tables are put - under roles and whether callers name a role or take the default, and which kinds of - access change (grant, revoke, ownership, default privileges) are applied to data - tables, last 30 days)
            • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index d6c2d288e6..d58ac65898 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -225,5 +225,10 @@ {#if datatableName && ws} - + {/if} diff --git a/frontend/src/lib/components/Star.svelte b/frontend/src/lib/components/Star.svelte index dd91f671c8..4f0f169109 100644 --- a/frontend/src/lib/components/Star.svelte +++ b/frontend/src/lib/components/Star.svelte @@ -9,9 +9,10 @@ kind: FavoriteKind summary?: string workspaceId?: string + size?: number } - let { path, kind, workspaceId, summary }: Props = $props() + let { path, kind, workspaceId, summary, size = 16 }: Props = $props() let buttonHover = $state(false) let starred = $derived(favoriteManager.isStarred(path, kind)) @@ -31,14 +32,14 @@ > {#if starred} {#if buttonHover} - + {:else} - + {/if} {:else} {/if} diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index c0477e8408..f2337d699b 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -282,7 +282,8 @@ const scriptsV2: typeof legacyScripts = { ...legacyScripts.postgresql, code: ` SELECT table_name, column_name, udt_name, column_default, is_nullable, nsp.nspname AS table_schema FROM information_schema.columns -RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')` +RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') +AND NOT starts_with(nsp.nspname, 'pg_') AND has_schema_privilege(nsp.oid, 'USAGE')` } } diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index bfba6a6a82..524817ba34 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -197,6 +197,7 @@ one long unbreakable string (a path list, a URL) sizes this column by that string and pushes it out of the panel — and any `truncate` inside never engages. --> +

              {title} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 71c06d6194..21c3d24770 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -732,12 +732,14 @@ export class AIChatManager implements ChatViewHost { /** Every mounted flow editor. */ #flowEditors = new Set() appAiChatHelpers = $state(undefined) - /** Datatable creation policy: enabled flag, datatable name, and optional schema */ + /** Datatable creation policy: enabled flag, datatable name, optional schema, and the role the + * app uses each data table through */ datatableCreationPolicy = $state<{ enabled: boolean datatable: string | undefined schema: string | undefined - }>({ enabled: false, datatable: undefined, schema: undefined }) + roles?: Record + }>({ enabled: false, datatable: undefined, schema: undefined, roles: undefined }) pendingNewCode = $state(undefined) apiTools = $state[]>([]) aiChatInput = $state(null) diff --git a/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte b/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte index 3aec0fa2b6..6b4cbbd710 100644 --- a/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte +++ b/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte @@ -69,6 +69,7 @@ diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 5ab271cc6f..634341b966 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -18,6 +18,7 @@ import { type AppCodeSelectionElement, type AppDatatableElement } from '../context' +import { appDatatableRole, sdkDatatableCall } from '$lib/components/raw_apps/dataTableRefUtils' // Backend runnable types export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline' @@ -921,9 +922,20 @@ export function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSy const policy = aiChatManager.datatableCreationPolicy const datatableName = policy.datatable ?? 'main' const schemaPrefix = policy.schema ? `${policy.schema}.` : '' - // Use wmill.datatable() for 'main' (default), otherwise wmill.datatable('name') - const datatableCall = - datatableName === 'main' ? 'wmill.datatable()' : `wmill.datatable('${datatableName}')` + // A role names the privileges the app's queries run with, so it has to be in the code the + // model writes. + const datatableRole = appDatatableRole(policy.roles, datatableName) + const tsDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'typescript') + const pyDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'python') + const roleEntries = Object.entries(policy.roles ?? {}) + const rolesNote = + roleEntries.length > 0 + ? `\n\nThis app uses these data tables through a role: ${roleEntries + .map(([dt, role]) => `\`${dt}\` as \`${role}\``) + .join( + ', ' + )}. Always pass that role when calling \`wmill.datatable\` on them, as in the examples. The role only reaches what it was granted, so a query on a table it lacks privileges on fails with \`permission denied\`.` + : '' let content = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend. @@ -1024,7 +1036,7 @@ Backend runnables should only perform **data operations** (SELECT, INSERT, UPDAT import * as wmill from 'windmill-client'; export async function main(user_id: string) { - const sql = ${datatableCall}; + const sql = ${tsDatatableCall}; const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne(); return user; } @@ -1035,12 +1047,12 @@ export async function main(user_id: string) { import wmill def main(user_id: str): - db = ${datatableCall} + db = ${pyDatatableCall} user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one() return user \`\`\` -Use these examples for normal datatable access. +Use these examples for normal datatable access.${rolesNote} ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts index 7232500898..bb7976d396 100644 --- a/frontend/src/lib/components/copilot/chat/datatableTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { WorkspaceService, type CompletedJob } from '$lib/gen' import type { DataTableTables } from '$lib/gen/types.gen' import { runScript } from '$lib/components/jobs/utils' +import { datatableReference } from '$lib/components/dbTypes' import { createToolDef, executeTestRun, @@ -15,9 +16,9 @@ import { * * Datatables are workspace-level managed PostgreSQL databases. The backend * endpoints used here (`list_datatable_tables`, `get_datatable_table_schema`) - * and SQL execution (`datatable://`) are gated only by workspace - * membership, so these tools need no app context and operate directly on the - * workspace. This is the unrestricted counterpart to the app-mode datatable + * and SQL execution (`datatable://`) need no app context: the server + * decides what the caller reaches, as the datatable role they name or its + * default. This is the unrestricted counterpart to the app-mode datatable * tools in `app/core.ts`, which additionally filter by the app's whitelist. */ @@ -31,9 +32,19 @@ const memo = (factory: () => T): (() => T) => { // ============= Pure workspace-scoped operations ============= -/** List all datatables configured in the workspace, with their schema/table names. */ -export async function listDatatables(workspace: string): Promise { - return await WorkspaceService.listDataTableTables({ workspace }) +/** List the datatables configured in the workspace, with their schema/table names: all of them as + * their default role, or only `datatableName`, as `role` when one is given. */ +export async function listDatatables( + workspace: string, + datatableName?: string, + role?: string +): Promise { + if (datatableName === undefined) return await WorkspaceService.listDataTableTables({ workspace }) + return await WorkspaceService.listDataTableTables({ + workspace, + datatableName, + ...(role !== undefined && { roleFor: datatableName, role }) + }) } /** Get the columns (column_name -> compact_type) of one datatable table. */ @@ -41,13 +52,15 @@ export async function getDatatableColumns( workspace: string, datatableName: string, schemaName: string, - tableName: string + tableName: string, + role?: string ): Promise> { const schema = await WorkspaceService.getDataTableTableSchema({ workspace, datatableName, schemaName, - tableName + tableName, + role }) return schema.columns } @@ -81,7 +94,26 @@ const NO_DATATABLES_CONFIGURED_MESSAGE = // ============= Tool definitions ============= -const getListDatatablesSchema = memo(() => z.object({})) +// The same rule the server applies to `-- role `; a name it would refuse fails here instead. +const getRoleSchema = memo(() => + z + .string() + .regex(/^[A-Za-z0-9_-]{1,63}$/) + .optional() + .describe( + "The datatable role to connect as, when the code you are working on uses one (an app's `data.roles` entry, or the `role` it passes to wmill.datatable). Omit for the datatable's default role." + ) +) + +const getListDatatablesSchema = memo(() => + z.object({ + datatable_name: z + .string() + .optional() + .describe('List only this datatable. Required with `role`.'), + role: getRoleSchema() + }) +) const getListDatatablesToolDef = memo(() => createToolDef( getListDatatablesSchema(), @@ -94,7 +126,8 @@ const getGetDatatableTableSchemaSchema = memo(() => z.object({ datatable_name: z.string().describe('The datatable name to inspect, e.g. "main".'), schema_name: z.string().describe('The schema name, e.g. "public".'), - table_name: z.string().describe('The table name to inspect.') + table_name: z.string().describe('The table name to inspect.'), + role: getRoleSchema() }) ) const getGetDatatableTableSchemaToolDef = memo(() => @@ -117,6 +150,7 @@ const getExecDatatableSqlSchema = memo(() => .describe( 'The SQL query to execute. Supports SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, ALTER TABLE, DROP TABLE, etc. For SELECT queries, results are returned as an array of objects. A newly created table will appear in list_datatables automatically.' ), + role: getRoleSchema(), background: z .boolean() .optional() @@ -217,10 +251,18 @@ export function getDatatableTools(): Tool<{}>[] { { def: getListDatatablesToolDef(), planModeSafe: true, - fn: async ({ workspace, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing datatables...' }) try { - const metadata = await listDatatables(workspace) + const parsedArgs = getListDatatablesSchema().parse(args ?? {}) + if (parsedArgs.role !== undefined && parsedArgs.datatable_name === undefined) { + throw new Error('`role` needs `datatable_name`, the datatable it is a role of') + } + const metadata = await listDatatables( + workspace, + parsedArgs.datatable_name, + parsedArgs.role + ) if (metadata.length === 0) { toolCallbacks.setToolStatus(toolId, { content: 'No datatables configured — set one up in workspace settings' @@ -236,7 +278,18 @@ export function getDatatableTools(): Tool<{}>[] { toolCallbacks.setToolStatus(toolId, { content: `Listed ${metadata.length} datatable(s) with ${totalTables} table(s)` }) - return JSON.stringify(metadata, null, 2) + // Only what the model acts on: the roles it may pass, not the creation privileges + // the manager's UI gates on. + return JSON.stringify( + metadata.map((d) => ({ + datatable_name: d.datatable_name, + schemas: d.schemas, + ...(d.error && { error: d.error }), + ...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role }) + })), + null, + 2 + ) } catch (e) { const errorMsg = `Error listing datatables: ${e instanceof Error ? e.message : String(e)}` toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) @@ -257,7 +310,8 @@ export function getDatatableTools(): Tool<{}>[] { workspace, parsedArgs.datatable_name, parsedArgs.schema_name, - parsedArgs.table_name + parsedArgs.table_name, + parsedArgs.role ) toolCallbacks.setToolStatus(toolId, { content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` @@ -300,7 +354,7 @@ export function getDatatableTools(): Tool<{}>[] { requestBody: { language: 'postgresql', content: parsedArgs.sql, - args: { database: `datatable://${name}` } + args: { database: datatableReference(name, parsedArgs.role) } } }), workspace, diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 2e68b74fe1..6c85a75a4e 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1434,7 +1434,12 @@ Data Tables: - Datatables are workspace-scoped managed PostgreSQL databases, shared across the workspace (not owned by any single app). They must be configured by the user in their workspace settings (Workspace settings → Data Tables); they cannot be created via SQL. - Use list_datatables to discover the available datatables and their tables. Reuse an existing table rather than creating a duplicate. If list_datatables reports none, this is a blocking prerequisite — tell the user to set up a datatable in their workspace settings and stop; do not assume a "main" datatable exists or call exec_datatable_sql. - Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries. -- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step. +- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.${ + isCloudHosted() + ? '' + : ` +- A raw app may use a datatable through a role (\`data.roles\` in its raw_app.yaml). When working on such an app, pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can.` + } - When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.${ skills.length > 0 ? ` diff --git a/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte b/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte deleted file mode 100644 index cbfb43ad39..0000000000 --- a/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte +++ /dev/null @@ -1,50 +0,0 @@ - - -
              - ({ value: t, label: t }))} - bind:value={table} - placeholder="The whole schema" - clearable - size="sm" - class="w-56" - /> - {/if} -
              diff --git a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte index e0ae596a3d..a3a765014a 100644 --- a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte +++ b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte @@ -25,30 +25,24 @@ let { workspace, datatable, - target, - onLoaded + target }: { workspace: string datatable: string /** What owner and grants are read and written for. */ target: AclTarget - /** Each read, with the target it was made for: it also lists what the target holds. */ - onLoaded?: (target: AclTarget, info: DatatableAclInfo) => void } = $props() const acl = resource( () => [workspace, datatable, target] as const, - async ([ws, dt, t]) => { - const loaded = await WorkspaceService.getDatatableAcl({ + async ([ws, dt, t]) => + await WorkspaceService.getDatatableAcl({ workspace: ws, datatableName: dt, kind: t.kind, schema: t.kind === 'database' ? undefined : t.schema, table: t.kind === 'table' ? t.table : undefined }) - onLoaded?.(t, loaded) - return loaded - } ) // Nothing is written before its SQL has been shown, and the apply runs exactly that SQL: the @@ -120,7 +114,12 @@ Loading… {:else}
              - {#if !info.editable} + {#if info.clone} + + Read only: this data table is a clone, and its owners and grants stay as they were copied + from the data table it was cloned from. + + {:else if !info.editable} Read only: access is changed by the admins of the workspace that governs this data table, on Windmill Enterprise Edition. diff --git a/frontend/src/lib/components/datatableMigrationRole.test.ts b/frontend/src/lib/components/datatableMigrationRole.test.ts new file mode 100644 index 0000000000..6d136b8371 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.test.ts @@ -0,0 +1,60 @@ +import { describe, test, expect } from 'vitest' +import { parseMigrationRole, withMigrationRole } from './datatableMigrationRole' + +describe('parseMigrationRole', () => { + test('reads every spelling the server accepts from the leading comment block', () => { + for (const line of [ + '-- role analyst', + '-- Role: analyst', + '-- role=analyst', + '-- role analyst;' + ]) { + expect(parseMigrationRole(`\n${line}\nBEGIN;\nEND;`)).toEqual({ + kind: 'role', + role: 'analyst' + }) + } + }) + + test('an annotation below BEGIN is not one', () => { + expect(parseMigrationRole('BEGIN;\n-- role analyst\nEND;')).toEqual({ kind: 'none' }) + }) + + test('a malformed attempt is an error, not the default', () => { + for (const line of [ + '-- role based access below', + '-- role', + '-- role:', + '-- role an;alytics' + ]) { + expect(parseMigrationRole(`${line}\nBEGIN;`)).toEqual({ kind: 'malformed', line }) + } + }) + + test('comments that do not start with the word role are ignored', () => { + expect(parseMigrationRole('-- roles analyst\n-- rolex\nBEGIN;')).toEqual({ kind: 'none' }) + }) +}) + +describe('withMigrationRole', () => { + test('leads above BEGIN, so the server reads it', () => { + const out = withMigrationRole('BEGIN;\nSELECT 1;\nEND;', 'analyst') + expect(out).toBe('-- role analyst\nBEGIN;\nSELECT 1;\nEND;') + }) + + test('replaces any attempt rather than stacking, malformed ones included', () => { + const out = withMigrationRole( + '-- Role: auditor\n-- role oops no\n-- keep me\nBEGIN;', + 'analyst' + ) + expect(out).toBe('-- role analyst\n-- keep me\nBEGIN;') + }) + + test('undefined strips the annotation, so it runs as admin', () => { + expect(withMigrationRole('-- role analyst\n\nBEGIN;\nEND;', undefined)).toBe('BEGIN;\nEND;') + }) + + test('refuses a name the server would refuse', () => { + expect(() => withMigrationRole('BEGIN;', 'bad;name')).toThrow() + }) +}) diff --git a/frontend/src/lib/components/datatableMigrationRole.ts b/frontend/src/lib/components/datatableMigrationRole.ts new file mode 100644 index 0000000000..5475bf2fd7 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.ts @@ -0,0 +1,70 @@ +import { isDatatableRoleName } from './dbTypes' + +/** + * A migration carries the data table role it runs as in its own SQL, as a `-- role ` + * annotation. There is no separate field: the annotation is what the server reads, and keeping + * it in the SQL is what lets it survive a `wmill sync` round-trip. + * + * Mirrors `SqlAnnotations::datatable_role` on the backend. It is only read from the leading + * comment block, so an annotation below `BEGIN;` is ignored and the migration runs as admin. A + * leading comment whose first word is `role` is an annotation attempt, and a malformed one is an + * error there, so it is one here too. + */ + +export type MigrationRole = + | { kind: 'none' } + | { kind: 'role'; role: string } + | { kind: 'malformed'; line: string } + +/** The body of a leading comment line that attempts a role annotation, or undefined. */ +function roleAttempt(line: string): string | undefined { + if (!line.startsWith('--')) return undefined + const body = line.slice(2).trimStart() + if (body.slice(0, 4).toLowerCase() !== 'role') return undefined + const after = body.slice(4) + if (after !== '' && !/^[\s:=]/.test(after)) return undefined + return after +} + +function parseAttempt(after: string): string | undefined { + let rest = after.trimStart() + if (rest.startsWith(':') || rest.startsWith('=')) rest = rest.slice(1) + const tokens = rest.split(/\s+/).filter((t) => t !== '') + if (tokens.length !== 1) return undefined + const role = tokens[0].endsWith(';') ? tokens[0].slice(0, -1) : tokens[0] + return isDatatableRoleName(role) ? role : undefined +} + +export function parseMigrationRole(sql: string): MigrationRole { + for (const raw of sql.split('\n')) { + const line = raw.trim() + if (line === '') continue + if (!line.startsWith('--')) break + const after = roleAttempt(line) + if (after === undefined) continue + const role = parseAttempt(after) + return role === undefined ? { kind: 'malformed', line } : { kind: 'role', role } + } + return { kind: 'none' } +} + +/** + * `sql` declaring `role`: any role annotation attempt in the leading comment block is removed, + * and `-- role ` is prepended above everything, or nothing when `role` is undefined. + */ +export function withMigrationRole(sql: string, role: string | undefined): string { + if (role !== undefined && !isDatatableRoleName(role)) { + throw new Error(`Invalid data table role '${role}'`) + } + const lines = sql.split('\n') + const kept: string[] = [] + let i = 0 + for (; i < lines.length; i++) { + const line = lines[i].trim() + if (line !== '' && !line.startsWith('--')) break + if (roleAttempt(line) === undefined) kept.push(lines[i]) + } + const rest = [...kept, ...lines.slice(i)] + while (rest.length > 0 && rest[0].trim() === '') rest.shift() + return role === undefined ? rest.join('\n') : [`-- role ${role}`, ...rest].join('\n') +} diff --git a/frontend/src/lib/components/datatableUsableRoles.test.ts b/frontend/src/lib/components/datatableUsableRoles.test.ts new file mode 100644 index 0000000000..cca4ab7607 --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from 'vitest' + +const request = vi.fn() +vi.mock('$lib/gen/core/request', () => ({ request: (...args: unknown[]) => request(...args) })) +vi.mock('$lib/gen', () => ({ OpenAPI: {} })) +vi.mock('$lib/cloud', () => ({ isCloudHosted: () => false })) + +import { listUsableDatatableRoles } from './datatableUsableRoles' + +const refuse = (body: string) => () => + Promise.reject(Object.assign(new Error('Bad Request'), { body })) + +describe('listUsableDatatableRoles', () => { + // The refusal is recognised by its sentence, so rewording it on one side alone would turn every + // data table on a build without the Enterprise Edition into a failed lookup rather than one + // that is not under roles. + it('reads the enterprise refusal as not under roles, and rethrows anything else', async () => { + request.mockImplementation(refuse('Data table roles are a Windmill Enterprise Edition feature')) + expect(await listUsableDatatableRoles('ws', 'main')).toEqual({ + permissioned: false, + roles: [], + default_role: 'admin' + }) + + request.mockImplementation(refuse('Data table not found')) + const rethrown = await listUsableDatatableRoles('ws', 'main').catch((e) => e) + expect(rethrown).toBeInstanceOf(Error) + }) +}) diff --git a/frontend/src/lib/components/datatableUsableRoles.ts b/frontend/src/lib/components/datatableUsableRoles.ts new file mode 100644 index 0000000000..84e52f29a8 --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.ts @@ -0,0 +1,44 @@ +import { OpenAPI, type ListUsableDatatableRolesResponse } from '$lib/gen' +import { request } from '$lib/gen/core/request' +import { isCloudHosted } from '$lib/cloud' +import { ADMIN_DATATABLE_ROLE } from './dbTypes' + +// `datatable_roles_unavailable` on the server, which is a plain 400: rewording it there without +// here makes every role picker on a non-Enterprise build fail instead of reading "not under roles". +const ROLES_UNAVAILABLE = 'Data table roles are a Windmill Enterprise Edition feature' + +const NOT_UNDER_ROLES: ListUsableDatatableRolesResponse = { + permissioned: false, + roles: [], + default_role: ADMIN_DATATABLE_ROLE +} + +/** + * The roles the caller may connect as on a data table. Cloud has no instance database, so no data + * table there is under roles, and none of the role pickers show. Without the Enterprise Edition + * every roles route refuses, which reads the same way: the data table is then used the way it was + * before roles, and one that is under roles is refused when something connects to it. + */ +export async function listUsableDatatableRoles( + workspace: string, + datatableName: string +): Promise { + if (isCloudHosted()) return NOT_UNDER_ROLES + try { + // The generated client encodes path params with `encodeURI`, which leaves a '?' in a + // data table name created before names were restricted to cut the path short. + return await request( + { ...OpenAPI, ENCODE_PATH: encodeURIComponent }, + { + method: 'GET', + url: '/w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}', + path: { workspace, datatable_name: datatableName } + } + ) + } catch (e) { + const body = (e as { body?: unknown })?.body + const detail = `${typeof body === 'string' ? body : JSON.stringify(body ?? '')} ${(e as Error)?.message ?? e}` + if (detail.includes(ROLES_UNAVAILABLE)) return NOT_UNDER_ROLES + throw e + } +} diff --git a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts index 59345c6ae7..890635dcf8 100644 --- a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts +++ b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts @@ -5,7 +5,7 @@ import { isDbType } from './dbTypes' /** * Single URL param `dbm` encodes the full DB manager state: - * firstSegment~path~schema.table + * firstSegment~path~schema.table~role=name * * firstSegment: * datatable – database with datatable:// resource (resourceType always postgresql) @@ -26,28 +26,46 @@ import { isDbType } from './dbTypes' * datatable~main~.customers (schema "public" implied) * ducklake~main~.orders (schema "main" implied) * postgresql~$res:u/user/my_pg~public.customers + * datatable~main~.customers~role=analyst + * datatable~main~role=analyst (no schema/table selected) + * + * role=name (last segment, optional, data tables only): the data table role to connect as. + * Omitted means the data table's default role. A trailing segment starting with `role=` is always + * the role, whatever follows. The name is kept as written, even when invalid (a `.` included), so + * the connection refuses it visibly instead of falling back to the default. */ const dbManagerSchema = z.object({ dbm: z.string().nullable() }) -interface ParsedDbm { +export interface ParsedDbm { type: 'database' | 'datatable' | 'ducklake' path: string resType?: string schema?: string table?: string + role?: string } -function parseDbm(raw: unknown): ParsedDbm | null { +const ROLE_SEGMENT_PREFIX = 'role=' + +function isRoleSegment(segment: string | undefined): segment is string { + return !!segment && segment.startsWith(ROLE_SEGMENT_PREFIX) +} + +export function parseDbm(raw: unknown): ParsedDbm | null { if (!raw || typeof raw !== 'string') return null const parts = raw.split('~') if (parts.length < 2 || !parts[1]) return null const firstSeg = parts[0] const path = parts[1] - const schemaTable = parts[2] ?? '' + const rest = parts.slice(2) + const role = isRoleSegment(rest.at(-1)) + ? rest.pop()!.slice(ROLE_SEGMENT_PREFIX.length) + : undefined + const schemaTable = rest[0] ?? '' let type: ParsedDbm['type'] let resType: string | undefined @@ -81,12 +99,12 @@ function parseDbm(raw: unknown): ParsedDbm | null { schema = defaultSchemas[type] } - return { type, path, resType, schema, table } + return { type, path, resType, schema, table, role: type === 'datatable' ? role : undefined } } const defaultSchemas: Record = { datatable: 'public', ducklake: 'main' } -function buildDbm(p: ParsedDbm): string { +export function buildDbm(p: ParsedDbm): string { const firstSeg = p.type === 'database' ? p.resType! : p.type const schema = p.schema === defaultSchemas[p.type] ? undefined : p.schema let schemaTable = '' @@ -97,7 +115,12 @@ function buildDbm(p: ParsedDbm): string { } else if (schema) { schemaTable = `${schema}.` } - return schemaTable ? `${firstSeg}~${p.path}~${schemaTable}` : `${firstSeg}~${p.path}` + const segments = [firstSeg, p.path] + if (schemaTable) segments.push(schemaTable) + if (p.type === 'datatable' && p.role !== undefined) { + segments.push(`${ROLE_SEGMENT_PREFIX}${p.role}`) + } + return segments.join('~') } export interface DbManagerUriState { @@ -105,6 +128,8 @@ export interface DbManagerUriState { readonly effectiveInput: DbInput | undefined readonly isDatatableInput: boolean selectedDatatable: string | undefined + /** The data table role the drawer connects as; undefined means its default. */ + selectedRole: string | undefined selectedSchema: string | undefined selectedTable: string | undefined readonly open: boolean @@ -137,6 +162,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: 'database' as const, resourceType: resType as DbType, resourcePath: parsed.type === 'datatable' ? `datatable://${parsed.path}` : parsed.path, + role: parsed.role, specificSchema: parsed.schema, specificTable: parsed.table } @@ -163,6 +189,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: isDatatable ? 'datatable' : 'database', path: isDatatable ? nInput.resourcePath.slice('datatable://'.length) : nInput.resourcePath, resType: isDatatable ? undefined : nInput.resourceType, + role: isDatatable ? nInput.role : undefined, schema: nInput.specificSchema, table: nInput.specificTable }) @@ -194,7 +221,14 @@ export function useDbManagerUriState(): DbManagerUriState { return parsed?.type === 'datatable' ? parsed.path : undefined }, set selectedDatatable(v: string | undefined) { - if (v) updateField({ path: v }) + // A role belongs to one data table, so it cannot carry over to another. + if (v) updateField({ path: v, role: undefined }) + }, + get selectedRole() { + return parsed?.role + }, + set selectedRole(v: string | undefined) { + updateField({ role: v }) }, get selectedSchema() { return parsed?.schema diff --git a/frontend/src/lib/components/dbManagerRole.test.ts b/frontend/src/lib/components/dbManagerRole.test.ts new file mode 100644 index 0000000000..6c45d0ed30 --- /dev/null +++ b/frontend/src/lib/components/dbManagerRole.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { buildDbm, parseDbm } from './dbManagerDrawerModel.svelte' +import { schemaCacheKey } from './dbSchemaCache' +import { datatableReference, type DbInput } from './dbTypes' + +describe('dbm role segment', () => { + it('round-trips a role, with and without a table', () => { + for (const dbm of ['datatable~main~.orders~role=p4_analytics', 'datatable~main~role=p4-op']) { + expect(buildDbm(parseDbm(dbm)!)).toBe(dbm) + } + expect(parseDbm('datatable~main~sales.orders~role=analyst')).toMatchObject({ + path: 'main', + schema: 'sales', + table: 'orders', + role: 'analyst' + }) + }) + + it('reads a link without a role as the default role', () => { + const parsed = parseDbm('datatable~main~.orders')! + expect(parsed.role).toBeUndefined() + expect(parsed).toMatchObject({ schema: 'public', table: 'orders' }) + expect(buildDbm(parsed)).toBe('datatable~main~.orders') + }) + + it('keeps an invalid role as written, so the connection refuses it', () => { + expect(parseDbm('datatable~main~role=a;b')?.role).toBe('a;b') + // A dot does not turn it into a schema.table selection read as the default role. + expect(parseDbm('datatable~main~role=bad.name')).toMatchObject({ + role: 'bad.name', + schema: undefined, + table: undefined + }) + }) +}) + +describe('connecting as a role', () => { + const input = (role?: string): DbInput => ({ + type: 'database', + resourceType: 'postgresql', + resourcePath: 'datatable://main', + role + }) + + // What `getDatabaseArg` builds every DB manager connection from. + it('appends the role to the data table reference', () => { + expect(datatableReference('main', 'p4_analytics')).toBe('datatable://main?role=p4_analytics') + expect(datatableReference('main', undefined)).toBe('datatable://main') + }) + + it('never appends a role to a name containing ?', () => { + // The server reads such a whole reference as the stored name first, so `?role=` would + // be taken as part of the name or refused instead of picking the role. + expect(datatableReference('legacy?x', undefined)).toBe('datatable://legacy?x') + expect(() => datatableReference('sales?role=analytics', 'admin')).toThrow(/'\?' in its name/) + }) + + it('refuses a role name the server would not accept', () => { + expect(() => datatableReference('main', 'a&role=admin')).toThrow(/Invalid data table role/) + expect(() => datatableReference('main', '')).toThrow(/Invalid data table role/) + }) + + it('keys the schema cache by role', () => { + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input('b'))) + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input())) + }) +}) diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index 5005cc931c..e239a9e957 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -8,7 +8,8 @@ import { runScriptAndPollResult } from './jobs/utils' import { writingJobOptions } from './jobs/writingJob' import type { DBSchema, SQLSchema } from '$lib/stores' import { stringifySchema } from './copilot/lib' -import type { DbInput, DbType } from './dbTypes' +import { datatableReference, type DbInput, type DbType } from './dbTypes' +import { withMigrationRole } from './datatableMigrationRole' import { assert } from '$lib/utils' import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' @@ -70,7 +71,9 @@ export function dbTableOpsWithPreviewScripts({ }): IDbTableOps { const dbType = getDbType(input) const language = getLanguageByResourceType(dbType) - const dbArg = getDatabaseArg(input) + // Built per call: an invalid role throws there, as that operation's error, rather than while + // the manager renders. + const dbArg = () => getDatabaseArg(input) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined function makeMarker(op: string, payload: Record): string { @@ -91,7 +94,7 @@ export function dbTableOpsWithPreviewScripts({ }) const result = await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, quicksearch }, language, content, tag } + requestBody: { args: { ...dbArg(), quicksearch }, language, content, tag } }) const count = result?.[0].count as number return count @@ -106,7 +109,7 @@ export function dbTableOpsWithPreviewScripts({ }) let items = (await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, ...params }, language, content, tag } + requestBody: { args: { ...dbArg(), ...params }, language, content, tag } })) as unknown[] if (!items || !Array.isArray(items)) { throw 'items is not an array' @@ -123,7 +126,7 @@ export function dbTableOpsWithPreviewScripts({ { workspace, requestBody: { - args: { ...dbArg, value_to_update: newValue, ...values }, + args: { ...dbArg(), value_to_update: newValue, ...values }, language, content, tag @@ -135,14 +138,14 @@ export function dbTableOpsWithPreviewScripts({ onDelete: async ({ values }) => { const content = makeMarker('DELETE', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) }, onInsert: async ({ values }) => { const content = makeMarker('INSERT', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) } @@ -246,6 +249,7 @@ export type IDbSchemaOps = { previewAlterSql: (params: { values: AlterTableValues; schema?: string }) => Promise onCreateSchema: (params: { schema: string }) => Promise onDeleteSchema: (params: { schema: string }) => Promise + onRenameSchema: (params: { schema: string; newSchema: string }) => Promise onFetchTableEditorDefinition: (params: { table: string schema?: string @@ -283,7 +287,8 @@ export function dbSchemaOpsWithPreviewScripts({ tag?: string }): IDbSchemaOps { const dbType = getDbType(input) - const dbArg = getDatabaseArg(input) + // Built per call, for the same reason as in the table ops above. + const dbArg = () => getDatabaseArg(input) const language = getLanguageByResourceType(dbType) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined @@ -293,6 +298,8 @@ export function dbSchemaOpsWithPreviewScripts({ input.type === 'database' && input.resourcePath.startsWith('datatable://') ? input.resourcePath.slice('datatable://'.length) : undefined + // A migration declaring no role runs as admin, whatever role the manager connects as. + const migrationRole = input.type === 'database' ? (input.role ?? input.migrationRole) : undefined function makeMarker(op: string, payload: Record): string { if (ducklake) payload.ducklake = ducklake @@ -359,7 +366,7 @@ export function dbSchemaOpsWithPreviewScripts({ : undefined if (!datatableName || !status?.enabled) { await runScriptAndPollResult( - { workspace, requestBody: { args: dbArg, content, language, tag } }, + { workspace, requestBody: { args: dbArg(), content, language, tag } }, writingJobOptions ) return @@ -373,12 +380,16 @@ export function dbSchemaOpsWithPreviewScripts({ throw new MigrationRunCancelled() } } - const codeUp = wrapMigration(await expandMarker(workspace, language, content)) + // Wrapped before annotating: the annotation must lead, above `BEGIN;`. + const codeUp = withMigrationRole( + wrapMigration(await expandMarker(workspace, language, content)), + migrationRole + ) // Down migrations are only generated for Postgres for now. let codeDown: string | undefined if (downContent && dbType === 'postgresql') { const downSql = (await expandMarker(workspace, language, downContent)).trim() - if (downSql) codeDown = wrapMigration(downSql) + if (downSql) codeDown = withMigrationRole(wrapMigration(downSql), migrationRole) } const created = await WorkspaceService.createDatatableMigration({ workspace, @@ -415,7 +426,7 @@ export function dbSchemaOpsWithPreviewScripts({ const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) const fkResult = await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } + requestBody: { args: dbArg(), content: fkContent, language, tag } }) let rawForeignKeys: RawForeignKey[] @@ -501,6 +512,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onRenameSchema: async ({ schema, newSchema }) => { + const content = makeMarker('RENAME_SCHEMA', { schema, new_schema: newSchema }) + const downContent = makeMarker('RENAME_SCHEMA', { schema: newSchema, new_schema: schema }) + await applyDdl(migrationName('rename_schema', schema), content, downContent) + }, onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { const foreignKeys = await fetchForeignKeys({ table, schema }) @@ -512,7 +528,7 @@ export function dbSchemaOpsWithPreviewScripts({ const pkContent = makeMarker('PRIMARY_KEY_CONSTRAINT', { table, schema }) const pkResult = (await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: pkContent, language, tag } + requestBody: { args: dbArg(), content: pkContent, language, tag } })) as { constraint_name?: string; CONSTRAINT_NAME?: string }[] if (pkResult && Array.isArray(pkResult) && pkResult.length > 0) { @@ -611,7 +627,9 @@ export function getDefaultDbTag(input: DbInput): string { export function getDatabaseArg(input: DbInput | undefined) { if (input?.type === 'database') { if (input.resourcePath.startsWith('datatable://')) { - return { database: input.resourcePath } + return { + database: datatableReference(input.resourcePath.slice('datatable://'.length), input.role) + } } else { return { database: '$res:' + input.resourcePath } } diff --git a/frontend/src/lib/components/dbSchemaCache.ts b/frontend/src/lib/components/dbSchemaCache.ts new file mode 100644 index 0000000000..0421c0c033 --- /dev/null +++ b/frontend/src/lib/components/dbSchemaCache.ts @@ -0,0 +1,21 @@ +import type { DbInput } from './dbTypes' + +/** What identifies a database's schema, role included: two roles on one data table may reach + * different schemas, so they cannot share a cache entry. Never throws, since it keys derived + * state; the connection itself is what refuses an invalid role. */ +export function getDbSchemasPath(input: DbInput): string { + switch (input.type) { + case 'database': + return input.role !== undefined && input.resourcePath.startsWith('datatable://') + ? `${input.resourcePath}?role=${input.role}` + : input.resourcePath + case 'ducklake': + return 'ducklake://' + input.ducklake + } +} + +/** Scoped by the acting workspace: a data table of the same name can exist in both the nav and + * the acting workspace, and one's schema must not be reused for the other. */ +export function schemaCacheKey(workspace: string | undefined, input: DbInput): string { + return `${workspace}:${getDbSchemasPath(input)}` +} diff --git a/frontend/src/lib/components/dbTypes.ts b/frontend/src/lib/components/dbTypes.ts index 6a85617110..e6d3af12e8 100644 --- a/frontend/src/lib/components/dbTypes.ts +++ b/frontend/src/lib/components/dbTypes.ts @@ -3,6 +3,12 @@ export type DbInput = type: 'database' resourceType: DbType resourcePath: string + /** The data table role to connect as; the data table's default when unset. Only + * meaningful for a `datatable://` path. */ + role?: string + /** The role migrations written through this input declare when `role` is unset. A + * migration declaring none runs as admin, not as the role the manager connects as. */ + migrationRole?: string specificSchema?: string specificTable?: string } @@ -23,3 +29,47 @@ export const dbTypes = [ 'duckdb' ] as const export const isDbType = (str?: string): str is DbType => !!str && dbTypes.includes(str as DbType) + +/** The role every data table has: the one it connects as when it is not under roles. */ +export const ADMIN_DATATABLE_ROLE = 'admin' + +/** What the server accepts in `-- role ` and `?role=`. */ +export function isDatatableRoleName(name: string): boolean { + return /^[A-Za-z0-9_-]{1,63}$/.test(name) +} + +/** Whether a role can be named in a reference to this data table. A name stored before names were + * restricted may contain `?`, and the server reads such a whole reference as that name first, so + * `?role=` after it would be taken as part of the name or refused. */ +export function datatableNameTakesRole(name: string): boolean { + return !name.includes('?') +} + +/** The `migrationRole` of a data table that cannot name a role in its reference: it connects as + * its default role, which its migrations must then declare. */ +export function defaultMigrationRole( + name: string, + permissioned: boolean | undefined, + defaultRole: string | undefined +): string | undefined { + return permissioned && !datatableNameTakesRole(name) ? defaultRole : undefined +} + +/** `datatable://`, with `?role=` when a role is named. Throws rather than build a + * reference the executor would refuse, or one that would silently mean another role. */ +export function datatableReference(name: string, role: string | undefined): string { + if (role === undefined) return `datatable://${name}` + if (!isDatatableRoleName(role)) { + throw new Error( + `Invalid data table role '${role}': only letters, digits, '_' and '-' are allowed` + ) + } + if (!datatableNameTakesRole(name)) { + throw new Error( + `Data table '${name}' has a '?' in its name, so it can only be used here as its default role. Rename it to connect as role '${role}'.` + ) + } + return `datatable://${name}?role=${role}` +} + +export type DatatableRowAction = 'migrations' | 'roles' | 'export' | 'import' diff --git a/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte b/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte index 57f01dddc1..c4d6d47269 100644 --- a/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte +++ b/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte @@ -3,13 +3,14 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import Select from '$lib/components/select/Select.svelte' import { + createDatatableAccessResource, createDatatablesResource, - createSchemasResource, toDatatableItems, toSchemaItems } from './datatableUtils.svelte' import { Button } from '../common' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { appDatatableRole } from './dataTableRefUtils' const operatingWorkspace = useOperatingWorkspace() let opWs = $derived($operatingWorkspace) @@ -19,6 +20,8 @@ datatable: string | undefined /** Currently selected schema */ schema: string | undefined + /** The role the app uses each data table through: schemas are listed as that role. */ + roles?: Record /** Callback when either value changes */ onChange?: (datatable: string | undefined, schema: string | undefined) => void /** Description text to show in the popover */ @@ -28,19 +31,31 @@ let { datatable, schema, + roles, onChange, description = 'Set the default datatable and schema for new tables. This is where AI will create new tables when needed.' }: Props = $props() + const role = $derived(datatable ? appDatatableRole(roles, datatable) : undefined) + // Load available datatables and schemas using shared utilities const datatables = createDatatablesResource(() => opWs) - const schemas = createSchemasResource( + const access = createDatatableAccessResource( () => datatable, + () => role, () => opWs ) const datatableItems = $derived(toDatatableItems(datatables.current)) - const schemaItems = $derived(toSchemaItems(schemas.current)) + // Until the answer is for this workspace, data table and role, the schemas in hand belong to + // another. + const schemaItems = $derived( + access.current.workspace === opWs && + access.current.datatable === datatable && + access.current.role === role + ? toSchemaItems(access.current.schemas) + : [] + ) // Track datatable changes to reset schema let previousDatatable = $state(undefined) @@ -81,6 +96,9 @@ placeholder="Select database" size="sm" /> + {#if role} + Used as role {role} + {/if}
              diff --git a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte index 6befd85579..7ccf89a275 100644 --- a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte @@ -1,34 +1,50 @@ - - {/if} -
              - {#if editable && row.id !== ADMIN_ROLE} - removeRole(row.id)} /> - {/if} -
              - -

              - {/each} -
              • - - {#if editable && unusedRoles.length > 0} -
              - - ({ + value: r.id, + label: r.enabled ? r.name : `${r.name} (disabled)` + }))} + placeholder="+ Add a role" + bind:value={ + () => undefined, + (id) => { + if (id) addRole(id) + } + } + onCreateItem={addRoleByName} + class="w-64" + /> +
              + + + {/if} + + {/if}
              + + {#if info?.supported && !hasUnsavedChanges} +
              + +
              + {/if} {/if} + + {#snippet actions()} + {#if editable} + + {/if} + {/snippet} + +{#if $superadmin} + +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte b/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte index d100b1681a..b863dac06b 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte @@ -13,11 +13,22 @@ import { SettingService, type InstanceDatatableRole } from '$lib/gen' import { sendUserToast } from '$lib/toast' + let { + initialName = '', + onChanged + }: { + /** Prefills the name of the role to add. */ + initialName?: string + /** Called after every change to the catalog, whether or not it went through. */ + onChanged?: () => void + } = $props() + let roles = $state([]) let loading = $state(true) let loadError = $state(undefined) let busy = $state(false) - let newName = $state('') + // svelte-ignore state_referenced_locally + let newName = $state(initialName) /** Which role's name is being edited, and to what. */ let renaming = $state<{ id: string; name: string } | undefined>(undefined) @@ -48,6 +59,7 @@ // holds, so a failed flip has to snap back rather than sit there claiming it landed. await load() busy = false + onChanged?.() } } diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 53b4819689..ebc4bd8a7f 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -64,12 +64,11 @@ - +{#if !hideTrigger} + +{/if} Beta {/snippet} - + {#key openCount} + + {/key} diff --git a/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte b/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte index d16f948b88..04cf841b28 100644 --- a/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte +++ b/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte @@ -8,12 +8,17 @@ import TextInput from '../text_input/TextInput.svelte' import SimpleEditor from '../SimpleEditor.svelte' import { WorkspaceService, type DatatableMigration } from '$lib/gen' + import { listUsableDatatableRoles } from '../datatableUsableRoles' import { sendUserToast } from '$lib/toast' import { tick } from 'svelte' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte' import Portal from '$lib/components/Portal.svelte' import { fetchPendingMigrations, outOfOrderRunMessage } from './datatableMigrationUtils' + import Select from '../select/Select.svelte' + import { parseMigrationRole, withMigrationRole } from '../datatableMigrationRole' + import { ADMIN_DATATABLE_ROLE } from '../dbTypes' + import { resource } from 'runed' let { workspace, @@ -50,6 +55,8 @@ let tab = $state('up') let name = $state('') let nameInput = $state() + let upEditor = $state() + let downEditor = $state() // A valid migration name is non-empty and limited to letters, digits, '_' and '-'. const MIGRATION_NAME_RE = /^[a-zA-Z0-9_-]+$/ let nameInvalid = $derived(!MIGRATION_NAME_RE.test(name.trim())) @@ -60,6 +67,96 @@ const confirmationModal = createAsyncConfirmationModal() + // The role lives in the SQL as its `-- role ` annotation, so the code is the single + // source of truth and the Select is a view onto it: reading parses, writing rewrites the + // annotation. + const usableRoles = resource( + () => [workspace, datatable] as const, + async ([ws, dt]) => { + try { + return await listUsableDatatableRoles(ws, dt) + } catch (e) { + console.error('Failed to load data table roles:', e) + return null + } + } + ) + // Not a valid role name, so it cannot collide with one. + const NO_ROLE = '(no role)' + let declaredUp = $derived(parseMigrationRole(codeUp)) + let declaredDown = $derived(enableDown ? parseMigrationRole(codeDown) : undefined) + let malformedLine = $derived( + declaredUp.kind === 'malformed' + ? declaredUp.line + : declaredDown?.kind === 'malformed' + ? declaredDown.line + : undefined + ) + const roleOf = (d: typeof declaredUp) => (d.kind === 'role' ? d.role : undefined) + // A rollback runs as the role its own SQL names: under another role than the up migration it + // typically cannot touch what the up created. + let sqlProblem = $derived( + malformedLine !== undefined + ? malformedMessage(malformedLine) + : declaredDown !== undefined && roleOf(declaredDown) !== roleOf(declaredUp) + ? `The down migration runs as ${roleOf(declaredDown) ?? 'admin (no role)'} but the up migration as ${roleOf(declaredUp) ?? 'admin (no role)'}: make their role annotations match` + : undefined + ) + let selectedRole = $derived(declaredUp.kind === 'role' ? declaredUp.role : NO_ROLE) + let permissioned = $derived(!!usableRoles.current?.permissioned) + // No annotation runs as admin, which the server allows exactly to those who may use `admin`. + let adminUsable = $derived(!!usableRoles.current?.roles.includes(ADMIN_DATATABLE_ROLE)) + let roleItems = $derived.by(() => { + const usable = usableRoles.current + if (!usable?.permissioned) return [] + const names = usable.roles.filter((r) => r !== ADMIN_DATATABLE_ROLE) + // A role the SQL names but the caller cannot use is still shown, or the picker would + // misreport what the migration runs as. + if (declaredUp.kind === 'role' && !names.includes(declaredUp.role)) { + names.push(declaredUp.role) + } + const items = names.map((r) => ({ + value: r, + label: r === usable.default_role ? `${r} (default)` : r + })) + if (adminUsable || declaredUp.kind === 'none') { + items.push({ + value: NO_ROLE, + label: 'No role — runs as admin with full access' + }) + } + return items + }) + + function setRole(value: string | undefined) { + const role = value === NO_ROLE ? undefined : value + codeUp = withMigrationRole(codeUp, role) + // Up and down agree: a rollback run as another role could fail on objects it does not own. + if (enableDown) codeDown = withMigrationRole(codeDown, role) + // Assigning the bound value does not repaint the editor, and its next keystroke would write + // the stale text back. + upEditor?.setCode(codeUp) + if (enableDown) downEditor?.setCode(codeDown) + } + + // Set by `open` when the SQL names no role yet: the data table's default is written once its + // roles are known. + let applyDefaultRole = $state(false) + $effect(() => { + // `undefined` until the first answer lands; `null` when it failed. + const usable = usableRoles.current + if (!applyDefaultRole || !isOpen || usableRoles.loading || usable === undefined) return + applyDefaultRole = false + if (!usable?.permissioned || declaredUp.kind !== 'none') return + const role = + usable.default_role !== ADMIN_DATATABLE_ROLE && usable.roles.includes(usable.default_role) + ? usable.default_role + : usable.default_role === ADMIN_DATATABLE_ROLE && adminUsable + ? undefined + : usable.roles.find((r) => r !== ADMIN_DATATABLE_ROLE) + if (role !== undefined) setRole(role) + }) + // Frame the migration body in an explicit transaction so it applies atomically. function wrapInTransaction(body: string): string { return `BEGIN;\n\n${body}\n\nEND;` @@ -72,15 +169,35 @@ } const PLACEHOLDER = wrapInTransaction('-- Add your migration here') + function malformedMessage(line: string): string { + return `Malformed role annotation \`${line}\`: write it as \`-- role \`, or pick the role above` + } + export function open(prefill?: { name?: string; codeUp?: string; codeDown?: string }) { + // Roles and the default can have changed since the last open (the roles drawer sits next + // to this modal), and the default role is written from this answer. + usableRoles.refetch() name = prefill?.name ?? '' // Start from the transaction template; when prefilled from detected DDL, - // wrap that DDL in the same BEGIN; ... END; frame. - codeUp = prefill?.codeUp - ? wrapInTransaction(ensureTrailingSemicolon(prefill.codeUp)) - : PLACEHOLDER + // wrap that DDL in the same BEGIN; ... END; frame. A role the prefill declares is taken + // out first and put back on top: below `BEGIN;` it would not be read. + const prefillRole = prefill?.codeUp ? parseMigrationRole(prefill.codeUp) : undefined + if (prefill?.codeUp) { + const wrapped = wrapInTransaction( + ensureTrailingSemicolon(withMigrationRole(prefill.codeUp, undefined)) + ) + codeUp = + prefillRole?.kind === 'role' + ? withMigrationRole(wrapped, prefillRole.role) + : prefillRole?.kind === 'malformed' + ? `${prefillRole.line}\n${wrapped}` + : wrapped + } else { + codeUp = PLACEHOLDER + } codeDown = prefill?.codeDown ?? PLACEHOLDER enableDown = (prefill?.codeDown ?? '') !== '' + applyDefaultRole = prefillRole === undefined || prefillRole.kind === 'none' tab = 'up' isOpen = true // Focus the name field once the modal content has rendered. @@ -96,6 +213,10 @@ sendUserToast("Invalid migration name: use only letters, digits, '_' and '-'", true) return } + if (sqlProblem !== undefined) { + sendUserToast(sqlProblem, true) + return + } if (run) { // A new migration gets the highest timestamp, so any still-pending // migration is earlier: running only this one applies it out of order. @@ -176,29 +297,65 @@ closeOnOutsideClick={false} >
              - +
              + + {#if permissioned} + { - const value = parseInt(e.currentTarget.value) - if (!isNaN(value)) { - updateTokensForModel(provider as AIProvider, model, value) - } - }} - class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent" - /> +
              + { + if (e.currentTarget.value === '') { + resetModel(provider as AIProvider, model) + return + } + const value = Number(e.currentTarget.value) + if (!isNaN(value)) { + updateTokens(provider as AIProvider, model, value) + } + }, + onblur: (e: Event & { currentTarget: HTMLInputElement }) => { + // Resync a value the state refused, so what is shown is what is stored. + e.currentTarget.value = String(limits[key] ?? '') + errors[key] = '' + } + }} + /> +
              tokens
              - {#if !isAtDefault} -
              - Default: {defaultTokens} tokens + {#if overridden} +
              + + {fallback.assumed ? 'Assumed' : 'Default'}: {fallback.tokens.toLocaleString()} + tokens +
              - {#if errors[getModelKey(provider as AIProvider, model)]} -
              {errors[getModelKey(provider as AIProvider, model)]}
              - {/if} + {/if} + {#if errors[key]} +
              {errors[key]}
              {/if}
              {/each} diff --git a/frontend/src/lib/components/workspaceSettings/perModelSettings.ts b/frontend/src/lib/components/workspaceSettings/perModelSettings.ts new file mode 100644 index 0000000000..a83da75540 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/perModelSettings.ts @@ -0,0 +1,24 @@ +import type { AIConfig, AIProvider } from '$lib/gen' +import { stripLegacyThinkingSuffix } from '../copilot/reasoningRegistry' + +/** + * The rows of a per-model setting, grouped by provider. Settings are keyed by the id + * the chat reads them under, and `setCopilotInfo` strips the deprecated `/thinking` + * suffix before the chat ever sees a model. A row built from the raw config would + * save a value under a key nothing reads, so it would sit in settings looking applied + * and never take effect. Stripping can collapse two configured slots onto one model, + * hence the dedupe. + */ +export function chatModelsByProvider( + aiProviders: Exclude +): Record> { + return Object.fromEntries( + Object.entries(aiProviders).map(([provider, config]) => [ + provider, + [...new Set(config.models.map(stripLegacyThinkingSuffix))].map((model) => ({ + provider: provider as AIProvider, + model + })) + ]) + ) +} From d340bec9994e12fcb652f17586ceeaec6a5bf49d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:46:03 +0000 Subject: [PATCH 20/53] chore(deps): bump actions/setup-node from 3 to 7 (#11260) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3 to 7. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v3...v7) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ai-agent-tests.yml | 2 +- .github/workflows/ai-evals-test.yml | 2 +- .github/workflows/backend-test-windows.yml | 2 +- .github/workflows/backend-test.yml | 2 +- .github/workflows/check-docs-links.yml | 2 +- .github/workflows/cli-tests.yml | 6 +++--- .github/workflows/codex-pr-review.yml | 2 +- .github/workflows/frontend-check.yml | 2 +- .github/workflows/git-sync-test.yml | 2 +- .github/workflows/npm_on_release.yml | 6 +++--- .github/workflows/pi-pr-review.yml | 2 +- .github/workflows/sdk-tests.yml | 2 +- .github/workflows/validate-openapi.yml | 2 +- .github/workflows/yaml-validator-tests.yml | 2 +- 14 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ai-agent-tests.yml b/.github/workflows/ai-agent-tests.yml index cd07602109..2527339699 100644 --- a/.github/workflows/ai-agent-tests.yml +++ b/.github/workflows/ai-agent-tests.yml @@ -63,7 +63,7 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 87eefab0f3..573bc77df9 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -77,7 +77,7 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: # Node 24 ships npm 11, which frontend/package-lock.json is authored # with; npm 10 rejects it ("Missing: picomatch@4.0.5 from lock file"). diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index f9153c0ce0..e0c731e41f 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -75,7 +75,7 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 9f65c2ae47..d5465d035d 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -59,7 +59,7 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "24" - uses: astral-sh/setup-uv@v6.2.1 diff --git a/.github/workflows/check-docs-links.yml b/.github/workflows/check-docs-links.yml index ec17c06771..e32e976f0c 100644 --- a/.github/workflows/check-docs-links.yml +++ b/.github/workflows/check-docs-links.yml @@ -16,7 +16,7 @@ jobs: sparse-checkout: | frontend/src .github/scripts - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "22.x" - name: Verify docs links are not 404 diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index e26231eda4..19afa777ae 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" @@ -89,7 +89,7 @@ jobs: cache-workspaces: backend - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" @@ -143,7 +143,7 @@ jobs: cache-workspaces: backend - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 7a3882df7c..354e8a60ca 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -213,7 +213,7 @@ jobs: - name: Set up Node.js if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22 diff --git a/.github/workflows/frontend-check.yml b/.github/workflows/frontend-check.yml index 178ed53dc5..6347614e16 100644 --- a/.github/workflows/frontend-check.yml +++ b/.github/workflows/frontend-check.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@v7 with: node-version: 24 cache: "npm" diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index bd15ec8876..b19f899a20 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -141,7 +141,7 @@ jobs: with: deno-version: v2.x - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index 7d30265c74..631ab53153 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" @@ -21,7 +21,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" @@ -32,7 +32,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index 9b536b6ed5..a89275cfb5 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -206,7 +206,7 @@ jobs: - name: Set up Node.js if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22 diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index 6f7d79c6af..c6078a9e7f 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -43,7 +43,7 @@ jobs: with: bun-version: latest - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20.x" diff --git a/.github/workflows/validate-openapi.yml b/.github/workflows/validate-openapi.yml index 5bc59ebc4c..846621b503 100644 --- a/.github/workflows/validate-openapi.yml +++ b/.github/workflows/validate-openapi.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: '20' diff --git a/.github/workflows/yaml-validator-tests.yml b/.github/workflows/yaml-validator-tests.yml index 04d31451d4..c5524312cb 100644 --- a/.github/workflows/yaml-validator-tests.yml +++ b/.github/workflows/yaml-validator-tests.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" From 49e0974c0787c10818b7c7923e09af7234630d01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:46:16 +0000 Subject: [PATCH 21/53] chore(deps): bump astral-sh/setup-uv from 5 to 7 (#11258) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 5 to 7. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/v5...v7) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/backend-test-windows.yml | 2 +- .github/workflows/backend-test.yml | 2 +- .github/workflows/sdk-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index e0c731e41f..d7c29e5d0c 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -79,7 +79,7 @@ jobs: with: node-version: "20" - - uses: astral-sh/setup-uv@v6.2.1 + - uses: astral-sh/setup-uv@v7 with: version: "0.11.24" diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index d5465d035d..00659fef5f 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/setup-node@v7 with: node-version: "24" - - uses: astral-sh/setup-uv@v6.2.1 + - uses: astral-sh/setup-uv@v7 with: version: "0.11.24" - uses: shivammathur/setup-php@v2 diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index c6078a9e7f..260fbb3a02 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -58,7 +58,7 @@ jobs: uses: actions/checkout@v4 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 # The interpreter is named explicitly: on a clean checkout uv picks the # runner's system python and stops with "not compatible with the locked From dbac7cce9921085618961d98fccc682fe01ba876 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:46:50 +0200 Subject: [PATCH 22/53] refactor(ai-evals): drop two unreachable job routes from the fetch stub (#11251) Co-authored-by: Claude Opus 5 (1M context) --- ai_evals/adapters/frontend/mockBackend.ts | 53 --------------- .../adapters/frontend/mockBackendApi.test.ts | 66 ++++--------------- 2 files changed, 12 insertions(+), 107 deletions(-) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 07ea82a58f..90df0cf870 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1178,27 +1178,6 @@ const BENCHMARK_WORKERS = [ } ] -const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/ -const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/ - -/** An intercepted request carries its args as a JSON string; anything else means none - * were supplied. */ -function parseBenchmarkRequestBody( - body: BodyInit | null | undefined -): Record | undefined { - if (typeof body !== 'string') { - return undefined - } - try { - const parsed = JSON.parse(body) - return typeof parsed === 'object' && parsed !== null - ? (parsed as Record) - : undefined - } catch { - return undefined - } -} - /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. * Any other relative fetch must keep its normal (non-benchmark) behavior — * intercepting it with a synthetic 404 sends the model into retry loops. */ @@ -1210,8 +1189,6 @@ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] return ( path === '/api/workers/list' || - BENCHMARK_JOB_GET_PATH.test(path) || - BENCHMARK_RUN_BY_PATH.test(path) || path === '/api/embeddings/query_hub_scripts' || path.startsWith('/api/scripts/hub/get_full/') || BENCHMARK_AI_MODELS_PATH.test(path) @@ -1235,36 +1212,6 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon ?.aiProviders?.find((entry) => entry.path === resourcePath) return Response.json({ data: (seed?.models ?? []).map((id) => ({ id })) }) } - const jobGet = BENCHMARK_JOB_GET_PATH.exec(path) - if (jobGet) { - const id = decodeURIComponent(jobGet[2]) - const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id) - if (!job) { - return Response.json({ error: `Job not found for "${id}"` }, { status: 404 }) - } - // The real endpoint lets a caller drop the bulky fields. Ignoring that here would - // size the model's context off a payload it explicitly asked to shrink. - const query = new URLSearchParams(url.split('?')[1] ?? '') - if (query.get('no_logs') === 'true') { - delete job.logs - } - if (query.get('no_code') === 'true') { - delete job.raw_code - } - return Response.json(job) - } - const runByPath = BENCHMARK_RUN_BY_PATH.exec(path) - if (runByPath) { - const workspace = decodeURIComponent(runByPath[1]) - const runnablePath = decodeURIComponent(runByPath[3]) - const args = parseBenchmarkRequestBody(init?.body) - // The real endpoint answers with the bare job id as text, not JSON. - return new Response( - runByPath[2] === 'f' - ? runBenchmarkFlowByPath({ workspace, path: runnablePath, args }) - : runBenchmarkScriptByPath({ workspace, path: runnablePath, args }) - ) - } if (path === '/api/embeddings/query_hub_scripts') { const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' return Response.json(searchBenchmarkHubScripts(text)) diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts index 0e489a4408..85bd1f3320 100644 --- a/ai_evals/adapters/frontend/mockBackendApi.test.ts +++ b/ai_evals/adapters/frontend/mockBackendApi.test.ts @@ -1,60 +1,18 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test' -import { - createBenchmarkCompletedJob, - getBenchmarkCompletedJob, - handleBenchmarkApiFetch, - hasBenchmarkApiHandler, - resetBenchmarkMockBackend, - registerBenchmarkWorkspaceRunnables -} from './mockBackend' - -const WORKSPACE = 'benchmark-api-ws' +import { describe, expect, it } from 'bun:test' +import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' +// `list_workers`'s client call is not faked, so its request really does leave as a relative +// `/api/...` url that node's fetch cannot resolve. Without this route the tool throws instead +// of answering. describe('benchmark API fetch handlers', () => { - beforeEach(() => resetBenchmarkMockBackend()) - afterEach(() => resetBenchmarkMockBackend()) + it('lists workers, the way list_workers reaches them', async () => { + const url = '/api/workers/list?per_page=100' - it('runs a deployed script by path', async () => { - registerBenchmarkWorkspaceRunnables(WORKSPACE, { - scripts: [ - { - path: 'f/evals/greet', - summary: 'Greet', - language: 'bun', - content: 'export async function main() {}' - } - ] - }) + expect(hasBenchmarkApiHandler(url)).toBe(true) + const body = await handleBenchmarkApiFetch(url).json() - const res = handleBenchmarkApiFetch( - `/api/w/${WORKSPACE}/jobs/run/p/${encodeURIComponent('f/evals/greet')}`, - { method: 'POST', body: JSON.stringify({ name: 'ada' }) } - ) - - expect(res.status).toBe(200) - const job = getBenchmarkCompletedJob(WORKSPACE, (await res.text()).trim()) - expect(job).toMatchObject({ success: true, args: { name: 'ada' } }) - }) - - it('serves a recorded job so a model can check the run it just started', async () => { - const id = createBenchmarkCompletedJob({ - workspace: WORKSPACE, - jobKind: 'preview', - result: 'Hello, World!' - }) - - const res = handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/${id}`) - - expect(res.status).toBe(200) - expect(await res.json()).toMatchObject({ - id, - success: true, - result: 'Hello, World!' - }) - }) - - it('404s an unknown job id instead of letting the fetch fall through', () => { - expect(hasBenchmarkApiHandler(`/api/w/${WORKSPACE}/jobs_u/get/missing`)).toBe(true) - expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404) + // The tool counts the list and maps over it, so it needs a bare array, not an envelope. + expect(Array.isArray(body)).toBe(true) + expect(body[0]).toMatchObject({ worker: expect.any(String), worker_group: expect.any(String) }) }) }) From 6dc78049ba2a29dbb34ff8a231d5742b23308177 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:47:09 +0000 Subject: [PATCH 23/53] chore(deps): bump actions/setup-go from 2 to 7 (#11257) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 2 to 7. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v2...v7) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/backend-test-windows.yml | 2 +- .github/workflows/backend-test.yml | 2 +- .github/workflows/go_on_release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index d7c29e5d0c..54a62ec495 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -67,7 +67,7 @@ jobs: # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v7 with: go-version: 1.21.5 diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 00659fef5f..fea84e7781 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -53,7 +53,7 @@ jobs: # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v7 with: go-version: 1.21.5 - uses: oven-sh/setup-bun@v2 diff --git a/.github/workflows/go_on_release.yml b/.github/workflows/go_on_release.yml index 78ce12c6c3..6e53db09e7 100644 --- a/.github/workflows/go_on_release.yml +++ b/.github/workflows/go_on_release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v7 - name: install_jq run: | sudo apt-get update From f39742ae324a412704f1da84a502a3d21d109203 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:47:20 +0000 Subject: [PATCH 24/53] chore(deps): bump actions/setup-python from 5 to 7 (#11256) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 7. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v7) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ai-agent-tests.yml | 2 +- .github/workflows/check-system-prompts.yml | 2 +- .github/workflows/publish-cli-docs.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ai-agent-tests.yml b/.github/workflows/ai-agent-tests.yml index 2527339699..8a3e9e6dbc 100644 --- a/.github/workflows/ai-agent-tests.yml +++ b/.github/workflows/ai-agent-tests.yml @@ -67,7 +67,7 @@ jobs: with: node-version: "20" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.11" diff --git a/.github/workflows/check-system-prompts.yml b/.github/workflows/check-system-prompts.yml index 3e684c82f9..370a177716 100644 --- a/.github/workflows/check-system-prompts.yml +++ b/.github/workflows/check-system-prompts.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/publish-cli-docs.yml b/.github/workflows/publish-cli-docs.yml index 9e76117eb5..d6a2bdb332 100644 --- a/.github/workflows/publish-cli-docs.yml +++ b/.github/workflows/publish-cli-docs.yml @@ -38,7 +38,7 @@ jobs: ssh-key: ${{ secrets.CLI_DOCS_DEPLOY_KEY }} fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" From b7425443e1e08be029fc676047f727f84b646323 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 21 Sep 2026 16:47:47 +0200 Subject: [PATCH 25/53] fix: guard data table migration routes against operators and unauthorized authors (#11243) * fix: refuse operators and unauthorized authors on data table migration routes Co-Authored-By: Claude Opus 5 (1M context) * fix: check the replaced definition on migration upsert and skip unchanged writes Co-Authored-By: Claude Opus 5 (1M context) * fix: let data table admins edit migrations whose annotation no longer parses Co-Authored-By: Claude Opus 5 (1M context) * fix: keep migration delete idempotent when a concurrent delete wins Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> --- ...ce891a85daa00e037aba4b65b60f87243bd3.json} | 5 +- ...97462276930f0a83ffb458f7cb26035c8c3d.json} | 8 +- backend/tests/datatable_migration_add_down.rs | 32 +++ .../tests/datatable_roles.rs | 152 ++++++++++++ .../src/datatable_migrations.rs | 231 +++++++++++++----- 5 files changed, 363 insertions(+), 65 deletions(-) rename backend/.sqlx/{query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json => query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json} (65%) rename backend/.sqlx/{query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json => query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json} (59%) diff --git a/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json b/backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json similarity index 65% rename from backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json rename to backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json index b82fcf2e17..2ff7820d86 100644 --- a/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json +++ b/backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE NOT $7 OR (datatable_migrations.name = $8::text AND datatable_migrations.code_up = $9::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE datatable_migrations.name = $7::text AND datatable_migrations.code_up = $8::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $9::text", "describe": { "columns": [], "parameters": { @@ -11,7 +11,6 @@ "Varchar", "Text", "Text", - "Bool", "Text", "Text", "Text" @@ -19,5 +18,5 @@ }, "nullable": [] }, - "hash": "2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d" + "hash": "0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3" } diff --git a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json b/backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json similarity index 59% rename from backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json rename to backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json index 3206ba4433..57dbc41a9f 100644 --- a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json +++ b/backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 RETURNING name", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 AND code_up = $4 AND code_down IS NOT DISTINCT FROM $5 RETURNING name", "describe": { "columns": [ { @@ -13,12 +13,14 @@ "Left": [ "Text", "Text", - "Int8" + "Int8", + "Text", + "Text" ] }, "nullable": [ false ] }, - "hash": "3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8" + "hash": "91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d" } diff --git a/backend/tests/datatable_migration_add_down.rs b/backend/tests/datatable_migration_add_down.rs index 3cd39e0fa3..47ea4c253b 100644 --- a/backend/tests/datatable_migration_add_down.rs +++ b/backend/tests/datatable_migration_add_down.rs @@ -142,3 +142,35 @@ async fn test_add_down_to_an_applied_migration(db: Pool) -> anyhow::Re Ok(()) } + +/// A definition saved before `-- role` annotations were parsed can open with a comment that now +/// reads as a malformed one. It never gets through a role check again, so an admin must still be +/// able to finish it rather than be refused on its annotation. +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_add_down_to_a_migration_whose_comment_predates_role_annotations( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + setup_applied_migration_without_down(&db).await?; + let code_up = format!("-- Role table for widgets\n{CODE_UP}"); + sqlx::query("UPDATE datatable_migrations SET code_up = $1 WHERE workspace_id = 'dtmig-ws'") + .bind(&code_up) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = authed(reqwest::Client::new().post(format!( + "http://localhost:{}/api/w/dtmig-ws/workspaces/upsert_datatable_migration/main", + server.addr.port() + ))) + .json(&json!({ + "timestamp": VERSION, + "name": "create_widgets", + "code_up": code_up, + "code_down": CODE_DOWN, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 3d2853284d..4948a0098a 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -1414,6 +1414,158 @@ async fn a_fork_waits_for_a_rename_of_the_data_table_it_keeps( Ok(()) } +async fn enable_migrations_on_main(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main,migrations_enabled}', 'true') + WHERE workspace_id = 'test-workspace'", + ) + .execute(db) + .await?; + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_migration_is_written_only_by_who_may_run_it(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + enable_migrations_on_main(&db).await?; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let create = |token: &str, body: Value| { + authed( + client() + .post(format!("{base}/create_datatable_migration/main")) + .json(&body), + token, + ) + .send() + }; + let upsert = |token: &str, code_up: &str| { + authed( + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&json!({"timestamp": 20260101000000i64, "name": "m", "code_up": code_up})), + token, + ) + .send() + }; + + // `test-user-2` holds `analytics` and nothing else, so of a migration they may store only + // what they could run: SQL that declares no role runs as admin, in the up or the down. + for body in [ + json!({"name": "m", "code_up": "DROP TABLE t"}), + json!({"name": "m", "code_up": "-- role analytics\nSELECT 1", "code_down": "DROP TABLE t"}), + ] { + let resp = create("SECRET_TOKEN_2", body.clone()).await?; + assert_eq!(resp.status(), 401, "{body}: {}", resp.text().await?); + } + let resp = create( + "SECRET_TOKEN_2", + json!({"name": "m", "code_up": "-- role analytics\nSELECT 1"}), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An admin's migration re-pushed as stored writes nothing, so it passes. Replacing it removes + // SQL they could not run, even with SQL they could, so it is refused like deleting it. + let resp = upsert("SECRET_TOKEN", "DROP TABLE t").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let resp = upsert("SECRET_TOKEN_2", "DROP TABLE t").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + for code_up in ["DROP TABLE u", "-- role analytics\nSELECT 1"] { + let resp = upsert("SECRET_TOKEN_2", code_up).await?; + assert_eq!(resp.status(), 401, "{code_up}: {}", resp.text().await?); + } + let resp = authed( + client().delete(format!( + "{base}/delete_datatable_migration/main/20260101000000" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + + let code_up: String = sqlx::query_scalar( + "SELECT code_up FROM datatable_migrations + WHERE workspace_id = 'test-workspace' AND timestamp = 20260101000000", + ) + .fetch_one(&db) + .await?; + assert_eq!(code_up, "DROP TABLE t"); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_operator_neither_writes_nor_runs_a_migration(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + enable_migrations_on_main(&db).await?; + // Off roles, where every role check passes everyone, so only the operator guard is left to + // refuse. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + // A token of its own: the auth cache is process-wide, and the other tests here authenticate + // `SECRET_TOKEN_3` in this workspace as a non-operator. + sqlx::raw_sql( + "UPDATE usr SET operator = true, role = 'Operator' + WHERE workspace_id = 'test-workspace' AND username = 'test-user-3'; + INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('DT_OPERATOR_TOKEN'::bytea), 'hex'), 'DT_OPERATO', + 'DT_OPERATOR_TOKEN', 'test3@windmill.dev', 'operator', false);", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let migration = json!({"timestamp": 20260101000000i64, "name": "m", "code_up": "SELECT 1"}); + + for request in [ + client() + .post(format!("{base}/create_datatable_migration/main")) + .json(&migration), + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&migration), + client().delete(format!( + "{base}/delete_datatable_migration/main/20260101000000" + )), + client().post(format!("{base}/run_datatable_migrations/main")), + client().post(format!("{base}/rollback_datatable_migrations/main")), + client().post(format!("{base}/generate_initial_datatable_migration/main")), + ] { + let resp = authed(request, "DT_OPERATOR_TOKEN").send().await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + status == 401 && text.contains("Operators cannot"), + "{status}: {text}" + ); + } + + // The same write from a developer is accepted: what refused the operator is theirs alone. + let resp = authed( + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&migration), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + #[cfg(not(all(feature = "private", feature = "enterprise")))] const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index fa433b2cbc..df1fac203b 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -107,26 +107,63 @@ async fn ensure_migration_role_allowed( datatable_name: &str, authed: &ApiAuthed, sql: &str, - timestamp: i64, - name: &str, + migration: &str, ) -> Result<()> { - let context = format!("Migration {timestamp} ({name})"); let access = DatatableAccess::Authed(authed.to_authed_ref()); match SqlAnnotations::datatable_role(sql)? { Some(role) => { - ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, &context) + ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, migration) .await } None => ensure_datatable_admin_access(db, w_id, datatable_name, &access) .await .map_err(|e| { Error::NotAuthorized(format!( - "{context} declares no role, so it would run as admin. {e}" + "{migration} declares no role, so it would run as admin. {e}" )) }), } } +/// Refuse writing or deleting a migration definition unless this caller may run both its up and its +/// down. A definition is run later by whoever applies or rolls back pending migrations, as the role +/// it declares, so gating only the run would let anyone plant SQL for a data table's admin to run. +async fn ensure_may_edit_migration( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, + migration: &str, + code_up: &str, + code_down: Option<&str>, +) -> Result<()> { + // Admin access reaches everything a role can, so it edits any definition without reading its + // annotation, which one saved before annotations were parsed may no longer pass. A refusal + // there would leave such a definition impossible to delete or fix. + let access = DatatableAccess::Authed(authed.to_authed_ref()); + if ensure_datatable_admin_access(db, w_id, datatable_name, &access) + .await + .is_ok() + { + return Ok(()); + } + for sql in std::iter::once(code_up).chain(code_down) { + ensure_migration_role_allowed(db, w_id, datatable_name, authed, sql, migration).await?; + } + Ok(()) +} + +/// Operators run deployed code only, and a migration is SQL written or run by hand. Checked before +/// anything else, including the role checks: on a data table not under roles those pass everyone. +fn ensure_not_operator(authed: &ApiAuthed) -> Result<()> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot manage or run data table migrations".to_string(), + )); + } + Ok(()) +} + #[derive(Serialize)] struct AppliedMigration { version: i64, @@ -436,6 +473,7 @@ async fn run_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + ensure_not_operator(&authed)?; // Before the admin connection is opened at all: the bookkeeping below is created and read // through it, so a caller no role covers must be refused here rather than after the fact. crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) @@ -503,8 +541,7 @@ async fn run_datatable_migrations( &datatable_name, &authed, &m.code_up, - m.timestamp, - &m.name, + &format!("Migration {} ({})", m.timestamp, m.name), ) .await?; run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) @@ -573,6 +610,7 @@ async fn rollback_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + ensure_not_operator(&authed)?; // Before the admin connection is opened at all: the bookkeeping below is created and read // through it, so a caller no role covers must be refused here rather than after the fact. crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) @@ -666,8 +704,7 @@ async fn rollback_datatable_migrations( &datatable_name, &authed, &code_down, - version, - &definition.name, + &format!("Migration {} ({})", version, definition.name), ) .await?; @@ -1213,9 +1250,20 @@ async fn create_datatable_migration( Path((w_id, datatable_name)): Path<(String, String)>, Json(payload): Json, ) -> JsonResult { + ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration '{}'", payload.name), + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; let mut tx = db.begin().await?; let timestamp = insert_datatable_migration_def( @@ -1267,6 +1315,34 @@ async fn delete_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, ) -> Result { + ensure_not_operator(&authed)?; + let deleted_message = format!("Deleted migration {} from {}", timestamp, datatable_name); + + // A definition already gone is not an error: `wmill sync push` deletes whatever the repo + // dropped, which may no longer exist here. + let Some(definition) = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_optional(&db) + .await? + else { + return Ok(deleted_message); + }; + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", timestamp, definition.name), + &definition.code_up, + definition.code_down.as_deref(), + ) + .await?; + // Hold the run-serialization lock across the applied-check and the delete: a // run snapshots a migration's SQL before recording its version, so an // unserialized delete could race it and leave `_wm_migrations` pointing at a @@ -1294,19 +1370,46 @@ async fn delete_datatable_migration( ))); } + // Delete only the SQL the role check above cleared: it read the definition without the lock, + // and a rewrite that landed since was never checked. let deleted_name = sqlx::query_scalar!( "DELETE FROM datatable_migrations \ WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 \ + AND code_up = $4 AND code_down IS NOT DISTINCT FROM $5 \ RETURNING name", &w_id, &datatable_name, timestamp, + &definition.code_up, + definition.code_down.as_deref(), ) .fetch_optional(&db) .await?; // The definition is removed; runs may resume (audit/deploy metadata below // don't need the lock). drop(lock_client); + let Some(name) = deleted_name else { + // Nothing matched either because another delete got there first, which is this one's + // outcome too, or because the definition was rewritten. + let still_defined = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3)", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !still_defined { + return Ok(deleted_message); + } + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' changed while it was being deleted. Reload it and \ + retry.", + timestamp, datatable_name + ))); + }; audit_log( &db, @@ -1319,23 +1422,10 @@ async fn delete_datatable_migration( ) .await?; - // Only tally a change if a migration was actually deleted. - if let Some(name) = deleted_name { - record_datatable_migration_deployment( - &authed, - &db, - &w_id, - &datatable_name, - timestamp, - &name, - ) + record_datatable_migration_deployment(&authed, &db, &w_id, &datatable_name, timestamp, &name) .await?; - } - Ok(format!( - "Deleted migration {} from {}", - timestamp, datatable_name - )) + Ok(deleted_message) } #[derive(Deserialize)] @@ -1356,6 +1446,7 @@ async fn upsert_datatable_migration( Path((w_id, datatable_name)): Path<(String, String)>, Json(payload): Json, ) -> Result { + ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; @@ -1381,9 +1472,9 @@ async fn upsert_datatable_migration( // run-serialization lock across the applied-check and the write below so an // in-flight run can't record a version for the SQL we're about to overwrite. // Held until the end of the handler (well past the write). The exempt - // upserts need no lock: a new or unchanged one overwrites nothing, and one - // that only adds a down leaves the `code_up` a concurrent run is recording - // a version for untouched. + // upserts need no lock: a new one overwrites nothing, and one that only adds + // a down leaves the `code_up` a concurrent run is recording a version for + // untouched. let only_adds_down = existing.as_ref().is_some_and(|existing| { existing.name == payload.name && existing.code_up == payload.code_up @@ -1395,8 +1486,40 @@ async fn upsert_datatable_migration( && existing.code_up == payload.code_up && existing.code_down == payload.code_down }); + let upserted_message = format!( + "Upserted migration {} in {}", + payload.timestamp, datatable_name + ); + // A re-push of the definition as stored changes nothing, so it is not checked, written, audited, + // synced or counted. Writing it anyway would re-create a definition deleted since the read above. + if unchanged { + return Ok(upserted_message); + } + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", payload.timestamp, payload.name), + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; + // Replacing a definition removes the one stored, so it answers to the same check as a delete. + if let Some(existing) = existing.as_ref() { + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", payload.timestamp, existing.name), + &existing.code_up, + existing.code_down.as_deref(), + ) + .await?; + } let _run_lock = match existing.as_ref() { - Some(_) if !only_adds_down && !unchanged => { + Some(_) if !only_adds_down => { // Fail closed: if we can't lock/read the applied set (e.g. the // data-table database is temporarily unreachable), refuse the change // rather than risk overwriting a migration that has already run. @@ -1426,31 +1549,27 @@ async fn upsert_datatable_migration( _ => None, }; - // An exempt upsert judged the row from an unlocked read and then writes - // without the lock, so that whole read is re-tested here, where `ON CONFLICT - // DO UPDATE` re-reads the row under a row lock. Otherwise a request working - // from a stale definition silently reverts whatever changed in between — a - // second addition's down, or a locked rewrite of the up whose new SQL a run - // may already have recorded a version for. Reading no row at all is part of - // the premise: the equality is NULL when `$8` is, so a version created in the - // meantime is refused rather than overwritten. - let recheck_observed = _run_lock.is_none(); + // The role checks, and the exemptions from the lock, all judged the row from the unlocked read + // above, so that whole read is re-tested here, where `ON CONFLICT DO UPDATE` re-reads the row + // under a row lock. Otherwise a request working from a stale definition silently replaces + // whatever changed in between: SQL the role checks never saw, or a locked rewrite of the up + // whose new SQL a run may already have recorded a version for. Reading no row at all is part of + // the premise: the equality is NULL when `$7` is, so a version created in the meantime is + // refused rather than overwritten. let written = sqlx::query!( "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ VALUES ($1, $2, $3, $4, $5, $6) \ ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down \ - WHERE NOT $7 \ - OR (datatable_migrations.name = $8::text \ - AND datatable_migrations.code_up = $9::text \ - AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", + WHERE datatable_migrations.name = $7::text \ + AND datatable_migrations.code_up = $8::text \ + AND datatable_migrations.code_down IS NOT DISTINCT FROM $9::text", &w_id, &datatable_name, payload.timestamp, &payload.name, &payload.code_up, payload.code_down.as_deref(), - recheck_observed, existing.as_ref().map(|e| e.name.as_str()), existing.as_ref().map(|e| e.code_up.as_str()), existing.as_ref().and_then(|e| e.code_down.as_deref()), @@ -1489,24 +1608,17 @@ async fn upsert_datatable_migration( ) .await?; - // An unchanged re-push is not counted: `wmill sync push` sends every migration - // on every sync, so counting those would swamp the definitions people write. - if !unchanged { - windmill_common::feature_usage::log_feature_usage( - "datatable", - "migration_created", - if existing.is_none() { - "synced" - } else { - "edited" - }, - ); - } + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_created", + if existing.is_none() { + "synced" + } else { + "edited" + }, + ); - Ok(format!( - "Upserted migration {} in {}", - payload.timestamp, datatable_name - )) + Ok(upserted_message) } /// Generate the first migration for a data table by snapshotting its current @@ -1519,6 +1631,7 @@ async fn generate_initial_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { + ensure_not_operator(&authed)?; // Returns a `pg_dump` of the whole schema and writes into the data table's own bookkeeping, so // it answers to the workspace that governs it rather than to whoever is asking. ensure_datatable_admin_access( From 4cbc944c0bcb938300c964437d745d15a3ea608f Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 21 Sep 2026 17:04:56 +0200 Subject: [PATCH 26/53] perf: keep the sessions page fast with many ai sessions (#11224) * perf: skip whole-document menu lookup on pointerdown when closed Co-Authored-By: Claude Opus 5 (1M context) * perf: load only a session's own chats in its chat history Co-Authored-By: Claude Opus 5 (1M context) * perf: stop mounting a chat per listed session on the sessions page Co-Authored-By: Claude Opus 5 (1M context) * fix: keep unsent input, init and chat rotation safe from runtime eviction Co-Authored-By: Claude Opus 5 (1M context) * refactor: cap mounted chat panes instead of evicting session runtimes Co-Authored-By: Claude Opus 5 (1M context) * refactor: read only message count and error for sessions without a runtime Co-Authored-By: Claude Opus 5 (1M context) * fix: keep a chat pane mounted while its composer reads attachments Co-Authored-By: Claude Opus 5 (1M context) * fix: hold a session's first send until its chat has been restored Co-Authored-By: Claude Opus 5 (1M context) * fix: queue a second send behind one waiting on the chat restore Co-Authored-By: Claude Opus 5 (1M context) * fix: queue raw-app inline prompts behind a send waiting on the restore Co-Authored-By: Claude Opus 5 (1M context) * fix: keep a chat pane mounted while its send waits on the restore Co-Authored-By: Claude Opus 5 (1M context) * fix: preserve session chat drafts across async updates --------- Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/components/DropdownV2.svelte | 3 + .../copilot/chat/AIChatInput.svelte | 12 +- .../copilot/chat/AIChatManager.svelte.ts | 51 +++++++- .../copilot/chat/AIChatManager.test.ts | 19 +++ .../copilot/chat/HistoryManager.svelte.ts | 16 ++- .../copilot/chat/HistoryManager.test.ts | 33 +++++ .../components/copilot/chat/chatViewHost.ts | 4 + .../conversations/flowChatViewHost.svelte.ts | 3 + .../lib/components/meltComponents/Menu.svelte | 3 + .../components/meltComponents/Popover.svelte | 3 + .../sessions/RawAppEditorView.svelte | 9 +- .../components/sessions/SessionPicker.svelte | 23 ++-- .../sessions/sessionRuntime.svelte.ts | 115 +++++++++++++++++- .../sessions/sessionUnread.svelte.ts | 17 ++- .../(root)/(logged)/sessions/+page.svelte | 34 ++++-- 15 files changed, 309 insertions(+), 36 deletions(-) diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 6738e7ace8..5061636835 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -145,6 +145,9 @@ } } async function getMenuElements(): Promise { + // Runs on every pointerdown anywhere, for every mounted dropdown. Skip the + // whole-document query when the outside handler below cannot act anyway. + if (!usePointerDownOutside || !open) return [] return Array.from(document.querySelectorAll('[data-menu]')) as HTMLElement[] } diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index d3cc2c8bd9..6f15937cf2 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -378,6 +378,16 @@ ) $effect(() => { chatHost.setComposerStaged(composerKey, editingMessageIndex, stagedBytes) + chatHost.setComposerHasDraft( + composerKey, + !draft.isEmpty || + (chatHost.mode === AIMode.GLOBAL && selectedContext.length > 0) || + pendingFiles > 0 || + pendingFileBytes > 0 || + pendingImages > 0 || + pendingBlobs > 0 || + ingestionHolds > 0 + ) }) $effect(() => () => chatHost.clearComposerStaged(composerKey)) @@ -889,7 +899,7 @@ contextTextareaComponent?.clearForSend() return } - if (chatHost.loading) { + if (chatHost.loading || chatHost.sendInFlight || chatHost.sendPending) { // Queue the message instead of silently discarding it — it is // auto-sent when the streaming turn completes successfully. // Editing-while-loading keeps the old discard behavior. Paste diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 97854a166e..922f64ad93 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -2224,7 +2224,7 @@ export class AIChatManager implements ChatViewHost { * Gated on `sendInFlight` as well as `loading`: `loading` only rises after a * send's attachment upkeep, so between the two a click would slip past. */ sendOrQueue(text: string) { - if (this.loading || this.sendInFlight) { + if (this.loading || this.sendInFlight || this.sendPending) { this.queueMessage(text) return } @@ -2819,6 +2819,23 @@ export class AIChatManager implements ChatViewHost { clearComposerStaged(key: string) { this.#composerStaged.delete(key) + this.#composersWithDraft.delete(key) + } + + // Composers whose draft is non-empty. The draft itself is component-local, so + // this is the only way to know an unmount would lose unsent input. + #composersWithDraft = new SvelteSet() + + setComposerHasDraft(key: string, hasDraft: boolean) { + if (hasDraft) this.#composersWithDraft.add(key) + else this.#composersWithDraft.delete(key) + } + + /** Unsent input an unmount of this chat would lose: a composer draft, or a + * queued message (whose text may be empty when it carries only attachments + * or context). */ + get hasUnsentInput(): boolean { + return this.#composersWithDraft.size > 0 || this.#hasQueuedMessage() } /** Release the outgoing-files reservation identified by `key` (a per-send token). @@ -3174,7 +3191,39 @@ export class AIChatManager implements ChatViewHost { return this.#sendsInFlight > 0 } + // Resolves once this chat's stored transcript has been restored. A session + // runtime hands its manager out before that finishes, and `loadPastChat` + // refuses to swap a transcript under a send, so a turn started first lands in + // a chat of its own and is lost when the session's real one loads. + // + // Cleared once awaited, and only awaited when set: every later send — and + // every send of the chats that have no gate — reaches the `sendInFlight` + // counter below synchronously, which is what makes a second Enter queue + // behind the first instead of starting a turn of its own. + #ready: Promise | undefined + setReadyGate(ready: Promise | undefined) { + this.#ready = ready + } + + #sendsAwaitingReady = $state(0) + /** A send parked on the gate above. It has not reached `sendInFlight` yet — + * and must not, since `loadPastChat` reads that flag and would then skip the + * restore this send is waiting for — so every guard that decides between + * sending and queueing tests this as well. */ + get sendPending(): boolean { + return this.#sendsAwaitingReady > 0 + } + sendRequest = async (options: Parameters[0] = {}) => { + if (this.#ready) { + this.#sendsAwaitingReady++ + try { + await this.#ready + } finally { + this.#sendsAwaitingReady-- + this.#ready = undefined + } + } // A turn with nowhere to render still streams, spends tokens and applies // tool calls — entirely off-screen. Refuse instead. `sendInlineRequest` is // exempt: the ⌘K widget renders its own composer inside Monaco. diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 35ac65253c..ef15df4e8d 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -514,6 +514,25 @@ describe('AIChatManager.sendOrQueue', () => { releaseUpkeep?.() await sending }) + + // A session's chat is restored after its manager is handed out, so the first + // send waits on that restore — before `sendInFlight`, which the restore reads + // to decide whether to bail. + it('queues while a send waits on the chat restore gate', async () => { + const manager = new AIChatManager() + let openGate: (() => void) | undefined + manager.setReadyGate(new Promise((resolve) => (openGate = resolve))) + manager.instructions = 'first turn' + const sending = manager.sendRequest() + await vi.waitFor(() => expect(manager.sendPending).toBe(true)) + expect(manager.sendInFlight).toBe(false) + + manager.sendOrQueue('fix the failing run') + expect(manager.queuedMessage).toBe('fix the failing run') + + openGate?.() + await sending + }) }) describe('AIChatManager request errors', () => { diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index d956b4c0c0..7843169dd9 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -394,7 +394,10 @@ export default class HistoryManager { .sort((a, b) => b.lastModified - a.lastModified) ) - async init() { + /** With `session`, loads only that session's chats plus `session.chatId`, which + * may predate session tagging. Loading every chat instead costs one full copy + * of the user's history per session runtime. */ + async init(session?: { id: string; chatId?: string }) { // (Re)initializing adopts a new identity's history: drop the previous // identity's cached blob ids with it. this.imageIdByUrl.clear() @@ -404,7 +407,16 @@ export default class HistoryManager { const db = await this.dbh.whenReady() if (!db) return try { - const chats = await db.getAll('chats') + let chats: StoredChat[] + if (session) { + chats = await db.getAllFromIndex('chats', 'by-session', session.id) + if (session.chatId && !chats.some((c) => c.id === session.chatId)) { + const current = await db.get('chats', session.chatId) + if (current) chats.push(current) + } + } else { + chats = await db.getAll('chats') + } this.savedChats = chats.reduce( (acc, chat) => { acc[chat.id] = chat diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index cdbf6f2020..3ee9edad39 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -816,3 +816,36 @@ describe('HistoryManager.reloadChat', () => { expect(await hm.reloadChat(hm.getCurrentChatId())).toBe('unavailable') }) }) + +describe('HistoryManager.init for a session', () => { + it("loads the session's chats and its current chat, not other sessions'", async () => { + const setup = new HistoryManager() + await setup.init() + setup.close() + const chat = (id: string, sessionId?: string) => ({ + id, + actualMessages: [], + displayMessages: [], + title: id, + lastModified: 1, + ...(sessionId ? { sessionId } : {}) + }) + const db = await openDB('copilot-chat-history::admin@test') + await db.put('chats' as never, chat('mine', 's1') as never) + await db.put('chats' as never, chat('other-session', 's2') as never) + await db.put('chats' as never, chat('untagged-current') as never) + await db.put('chats' as never, chat('untagged-other') as never) + db.close() + + const hm = new HistoryManager() + await hm.init({ id: 's1', chatId: 'untagged-current' }) + + // The current chat can predate session tagging, so it is read by key. + expect( + hm + .getAllSavedChats() + .map((c) => c.id) + .sort() + ).toEqual(['mine', 'untagged-current']) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/chatViewHost.ts b/frontend/src/lib/components/copilot/chat/chatViewHost.ts index 4af23337e2..5c4dceb7f1 100644 --- a/frontend/src/lib/components/copilot/chat/chatViewHost.ts +++ b/frontend/src/lib/components/copilot/chat/chatViewHost.ts @@ -52,6 +52,8 @@ export interface ChatViewHost { * pins away from the navigated one. */ readonly operatingWorkspace: string | undefined loading: boolean + /** A send waiting for the chat's transcript to be restored; queue behind it. */ + sendPending: boolean /** A turn this tab can neither follow nor stop, held by another tab on the same chat. */ readonly runHeldElsewhere: boolean loadingLabel: string | undefined @@ -85,6 +87,8 @@ export interface ChatViewHost { ) => void dequeueMessage: () => void setComposerStaged: (key: string, editingIndex: number | null, bytes: number) => void + /** Whether this composer holds anything unsent (text, pastes, attachments, reads in flight). */ + setComposerHasDraft: (key: string, hasDraft: boolean) => void clearComposerStaged: (key: string) => void attachmentBytesExcluding: (selfKey: string) => number diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts index 37bb868982..6ef18b7212 100644 --- a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -398,6 +398,8 @@ export class FlowChatViewHost implements ChatViewHost { get loading(): boolean { return isBusy(this.#state.status) } + // No transcript restore to wait for: a flow chat renders from its rows. + sendPending = false runHeldElsewhere = false loadingLabel = undefined compacting = false @@ -580,6 +582,7 @@ export class FlowChatViewHost implements ChatViewHost { return taken } setComposerStaged = () => {} + setComposerHasDraft = () => {} clearComposerStaged = () => {} attachmentBytesExcluding = () => 0 diff --git a/frontend/src/lib/components/meltComponents/Menu.svelte b/frontend/src/lib/components/meltComponents/Menu.svelte index 4a821ceacc..ab6de64123 100644 --- a/frontend/src/lib/components/meltComponents/Menu.svelte +++ b/frontend/src/lib/components/meltComponents/Menu.svelte @@ -176,6 +176,9 @@ } async function getMenuElements(): Promise { + // Runs on every pointerdown anywhere, for every mounted menu. Skip the + // whole-document query when the outside handler below cannot act anyway. + if (!usePointerDownOutside || !open) return [] // Tooltip content counts as menu territory for the same reason as the // onOutsideClick veto above. return Array.from( diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte index 6a6b7d681b..dd69d16917 100644 --- a/frontend/src/lib/components/meltComponents/Popover.svelte +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -194,6 +194,9 @@ } async function getMenuElements(): Promise { + // Runs on every pointerdown anywhere, for every mounted popover. Skip the + // whole-document query when the outside handler below cannot act anyway. + if (!usePointerDownOutside || (!isOpen && !fullScreen)) return [] const selector = excludeSelectors ? `[data-popover], ${excludeSelectors}` : '[data-popover]' return Array.from(document.querySelectorAll(selector)) as HTMLElement[] } diff --git a/frontend/src/lib/components/sessions/RawAppEditorView.svelte b/frontend/src/lib/components/sessions/RawAppEditorView.svelte index 4e2fb2bbd4..e95bc142e5 100644 --- a/frontend/src/lib/components/sessions/RawAppEditorView.svelte +++ b/frontend/src/lib/components/sessions/RawAppEditorView.svelte @@ -172,9 +172,10 @@ // The inline mini-composer over a selected element sends a chat turn; the // element is already an app_dom_selector chip, so it rides along as context. - // Mirror the composer: while a turn is streaming, queue it (a second concurrent - // sendRequest would race the shared abortController / streaming buffers) — it - // auto-sends when the current turn completes. + // Mirror the composer: while another send holds the chat — streaming, in + // preflight, or waiting for the transcript to be restored — queue it (a second + // concurrent sendRequest would race the shared abortController / streaming + // buffers) — it auto-sends when the current turn completes. function onInlinePrompt(selector: string, prompt: string) { // Snapshot the selection synchronously at submit time so a re-selection // during the async send preflight (immediate path) or before the queue @@ -190,7 +191,7 @@ .filter( (c) => c.type !== 'app_dom_selector' || (c.selector === selector && c.appPath === path) ) - if (runtime.manager.loading) { + if (runtime.manager.loading || runtime.manager.sendInFlight || runtime.manager.sendPending) { runtime.manager.queueMessage(prompt, [], snapshot) } else { void runtime.manager.sendRequest({ instructions: prompt, contextOverride: snapshot }) diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index e5b62da0eb..81ce3ff9af 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -44,8 +44,9 @@ import { unreadCountFor } from './sessionUnread.svelte' import Toggle from '$lib/components/Toggle.svelte' import { - getOrCreateRuntime, + ensureSessionChatPeek, getRuntime, + getSessionChatPeek, getSessionChatStatus, removeSession, resetSessionPreviewTabs @@ -87,7 +88,7 @@ // displayMessages array vs. the localStorage-backed lastSeen map; // both are reactive so the badge updates without polling. function unreadFor(session: Session): number { - return unreadCountFor(session.id, getRuntime(session.id)) + return unreadCountFor(session.id, getRuntime(session.id), getSessionChatPeek(session.id)) } // Whether the composer for a session holds non-whitespace text. We @@ -378,14 +379,12 @@ } }) - // Eagerly create a runtime per VISIBLE session so the status dot reflects - // the persisted chat (last message, pending confirmation, etc.) without - // requiring the user to open the session first. Sessions outside the - // current workspace scope are left cold to avoid opening IDB connections - // for unrelated work. + // Read each VISIBLE session's stored chat so the status dot and unread count + // reflect it without the user opening the session. A runtime per listed + // session would hold every listed chat in memory for as long as the page lives. $effect(() => { for (const session of visibleSessions) { - getOrCreateRuntime(session) + if (!getRuntime(session.id)) void ensureSessionChatPeek(session) } }) @@ -768,7 +767,9 @@ {/if} {#each group.sessions as session (session.id)} {@const runtime = getRuntime(session.id)} - {@const status = runtime ? getSessionChatStatus(runtime) : 'idle'} + {@const status = runtime + ? getSessionChatStatus(runtime) + : (getSessionChatPeek(session.id)?.status ?? 'idle')} {@const isSelected = sessionActive && session.id === sessionState.currentSessionId} {@const unread = unreadFor(session)} @@ -986,7 +987,9 @@ {/snippet} {#snippet sessionRow(session, indented, treeDepth)} {@const runtime = getRuntime(session.id)} - {@const status = runtime ? getSessionChatStatus(runtime) : 'idle'} + {@const status = runtime + ? getSessionChatStatus(runtime) + : (getSessionChatPeek(session.id)?.status ?? 'idle')} {@const isSelected = sessionActive && session.id === sessionState.currentSessionId} {@const isEditing = editingId === session.id} {@const unread = unreadFor(session)} diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index 640b484478..fbb84e1431 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -99,7 +99,13 @@ import type { RawAppDomResult } from '$lib/components/raw_apps/rawAppDom' import { getNonStreamingMetadataCompletion } from '$lib/components/copilot/lib' -import { pendingUserAction, type DisplayMessage } from '$lib/components/copilot/chat/shared' +import { + pendingUserAction, + type ChatJob, + type DisplayMessage +} from '$lib/components/copilot/chat/shared' +import { readStoredChat } from '$lib/components/copilot/chat/HistoryManager.svelte' +import { getCurrentUserEmail } from '$lib/userScopedStorage' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' // Per-kind load state for a session's editor target. Pure state container the @@ -985,7 +991,7 @@ function createRuntime(session: Session): SessionRuntime { async function initRuntime(runtime: SessionRuntime, session: Session) { const { manager } = runtime - await manager.historyManager.init() + await manager.historyManager.init(session) manager.historyManager.setSessionId(session.id) // Restore linked files persisted for this session (live handles re-grant on send; // snapshots restore directly). Non-transient sessions persist immediately. @@ -1000,7 +1006,9 @@ async function initRuntime(runtime: SessionRuntime, session: Session) { if (session.chatId) { manager.historyManager.setCurrentChatId(session.chatId) await manager.historyManager.tagChatWithSession(session.chatId, session.id) - await manager.loadPastChat(session.chatId) + // preserveQueue: restoring a session's own chat is not a conversation switch. + // A message queued while the restore ran is unsent user input waiting for it. + await manager.loadPastChat(session.chatId, { preserveQueue: true }) // loadPastChat only seeds the mask when the chat exists in history; a chatId // pointing at a chat not yet persisted (no turn saved) would leave it // undefined, and the Edits surface would then show every workspace draft. @@ -1020,7 +1028,11 @@ export function getOrCreateRuntime(session: Session): SessionRuntime { if (!runtime) { runtime = createRuntime(session) runtimes.set(session.id, runtime) - initRuntime(runtime, session).catch((e) => console.error('Failed to init session runtime', e)) + // The gate opens even when the restore fails: a session that cannot read its + // history is still usable, and a send that waits forever is not. + runtime.manager.setReadyGate( + initRuntime(runtime, session).catch((e) => console.error('Failed to init session runtime', e)) + ) } return runtime } @@ -1031,6 +1043,9 @@ export function disposeRuntime(sessionId: string) { runtime.manager.cancel('runtime disposed') runtime.manager.historyManager.close() runtimes.delete(sessionId) + // The sidebar reads the stored chat again rather than trust a read that + // predates everything this runtime saw. + peeks.delete(sessionId) } export function listRuntimes(): SessionRuntime[] { @@ -1064,7 +1079,13 @@ onRemoteTurnEnd((sessionId, chatId) => { async function applyRemoteTurnEnd(sessionId: string, chatId: string): Promise { const runtime = runtimes.get(sessionId) - if (!runtime) return + if (!runtime) { + // The driving tab may have rotated to a new chat; follow it, or the next + // read and a later runtime would open the old one. + setSessionChatId(sessionId, chatId) + refreshSessionChatPeek(sessionId) + return + } const m = runtime.manager // Two transient states get a short retry rather than a skip, because the // composer unlocks when this promise settles and a skip would unlock it on @@ -1405,3 +1426,87 @@ export function getSessionChatStatus(runtime: SessionRuntime): SessionChatStatus if (last && (last.role === 'assistant' || last.role === 'tool')) return 'awaiting-user' return 'idle' } + +// --------------------------------------------------------------------------- +// Sessions without a runtime +// --------------------------------------------------------------------------- + +// A runtime costs a chat manager and its loaded transcript, and lives until its +// session is deleted, so the sidebar does not create one per listed session. It +// reads the session's one stored chat instead, once. Without a runtime nothing +// runs in this tab, so only two facts of that chat can matter: its length (for +// the unread count) and whether its last message failed. Every other status is +// a live one. +export interface SessionChatPeek { + status: 'error' | 'idle' + messageCount: number +} + +const peeks = new SvelteMap() +const peeksInFlight = new Set() +const peeksPendingRefresh = new Set() + +// Not gated on `detached`: a chat saved mid-wait stores its job undetached, and +// loadPastChat resumes polling every unfinished job either way. +function isLiveJob(j: ChatJob): boolean { + return ( + j.status === 'queued' || + j.status === 'running' || + j.status === 'suspended' || + j.status === 'scheduled' + ) +} + +function sessionById(sessionId: string): Session | undefined { + return sessionState.sessions.find((s) => s.id === sessionId) +} + +function refreshSessionChatPeek(sessionId: string): void { + peeks.delete(sessionId) + if (peeksInFlight.has(sessionId)) { + peeksPendingRefresh.add(sessionId) + return + } + const session = sessionById(sessionId) + if (session) void ensureSessionChatPeek(session) +} + +export function getSessionChatPeek(sessionId: string): SessionChatPeek | undefined { + return peeks.get(sessionId) +} + +/** Read the stored chat of a session that has no runtime. A chat with a detached + * job still running gets its runtime instead: only a runtime polls the job and + * resumes the conversation when it finishes. */ +export async function ensureSessionChatPeek(session: Session): Promise { + const id = session.id + const chatId = session.chatId + if (runtimes.has(id) || peeks.has(id) || peeksInFlight.has(id) || !chatId) return + const email = getCurrentUserEmail() + if (!email) return + peeksInFlight.add(id) + try { + const chat = await readStoredChat(chatId, email) + if (sessionById(id)?.chatId !== chatId) { + refreshSessionChatPeek(id) + return + } + if (!chat || runtimes.has(id)) return + if (chat.backgroundJobs?.some(isLiveJob)) { + getOrCreateRuntime(session) + return + } + const last = chat.displayMessages[chat.displayMessages.length - 1] + peeks.set(id, { + status: last?.role === 'user' && last.error ? 'error' : 'idle', + messageCount: chat.displayMessages.length + }) + } catch (e) { + console.error('Failed to read session chat', e) + } finally { + peeksInFlight.delete(id) + if (peeksPendingRefresh.delete(id)) { + refreshSessionChatPeek(id) + } + } +} diff --git a/frontend/src/lib/components/sessions/sessionUnread.svelte.ts b/frontend/src/lib/components/sessions/sessionUnread.svelte.ts index 281c6571bc..26ce3f205b 100644 --- a/frontend/src/lib/components/sessions/sessionUnread.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionUnread.svelte.ts @@ -1,4 +1,4 @@ -import type { SessionRuntime } from './sessionRuntime.svelte' +import type { SessionChatPeek, SessionRuntime } from './sessionRuntime.svelte' import { sessionState, putSession } from './sessionState.svelte' // Per-session "last seen" watermark — the displayMessages count the last time @@ -20,11 +20,16 @@ export function markSessionSeen(sessionId: string, count: number) { void putSession(s) } -// Number of unread messages for a session. Undefined / unloaded runtime -// returns 0 — until messages are hydrated we don't know what's new. -export function unreadCountFor(sessionId: string, runtime: SessionRuntime | undefined): number { - if (!runtime) return 0 +// Number of unread messages for a session, from its runtime when it has one, else +// from the sidebar's read of its stored chat. Neither loaded returns 0 — until +// messages are read we don't know what's new. +export function unreadCountFor( + sessionId: string, + runtime: SessionRuntime | undefined, + peek: SessionChatPeek | undefined +): number { + const total = runtime ? runtime.manager.displayMessages.length : peek?.messageCount + if (total === undefined) return 0 const seen = sessionState.sessions.find((x) => x.id === sessionId)?.lastSeenCount ?? 0 - const total = runtime.manager.displayMessages.length return Math.max(0, total - seen) } diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index a3712f3f24..e773e7603f 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -220,19 +220,39 @@ // deep links the same way they react to picker clicks. selectSession(session.id) getOrCreateRuntime(session) + mountChat(session.id) }) }) - // Warm = sessions with a live runtime. The picker eagerly creates runtimes - // for its visible sessions, so this tracks whatever it shows. Keeping warm - // chats mounted (stacked, visibility-toggled) preserves their scroll/draft - // state across switches. + // Warm = sessions with a live runtime: the ones visited this page load plus + // any a chat tool or a running job started. const warmSessions = $derived( listRuntimes() .map((r) => sessionState.sessions.find((s) => s.id === r.sessionId)) .filter((s): s is NonNullable => s != null) ) + // Chat columns stay mounted (stacked, visibility-toggled) so switching back + // keeps scroll position, MRU-capped like preview tabs: every mounted column + // is a full transcript in the DOM. Unmounting drops only the column — the + // runtime keeps the chat — so a column is kept while its composer holds + // unsent input (component-local) or its turn is running. + const MAX_MOUNTED_CHATS = 5 + const mountedChatIds = new SvelteSet() + function keepsChatMounted(id: string): boolean { + const m = getRuntime(id)?.manager + return !!m && (m.loading || m.sendInFlight || m.sendPending || m.hasUnsentInput) + } + function mountChat(id: string) { + mountedChatIds.delete(id) + mountedChatIds.add(id) + for (const oldest of [...mountedChatIds]) { + if (mountedChatIds.size <= MAX_MOUNTED_CHATS) break + if (oldest !== id && !keepsChatMounted(oldest)) mountedChatIds.delete(oldest) + } + } + const mountedChatSessions = $derived(warmSessions.filter((s) => mountedChatIds.has(s.id))) + // Mark the active session "seen" up to its current message count: arrive → // clear unread; AI streams a new message while we're here → clear again. The // effect depends only on the length, not the array contents, so token-by-token @@ -915,11 +935,11 @@ class="flex-1 min-h-0 session-splitter {previewCollapsed ? 'splitter-off' : ''}" > {#if !fullscreen} - +
              - {#each warmSessions as s (s.id)} + {#each mountedChatSessions as s (s.id)} diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 7749310b5b..31fd9426f5 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -13,9 +13,10 @@ items?: Item[] | (() => Item[]) | (() => Promise) meltItem: MenubarMenuElements['item'] builders?: ReturnType['builders'] + close?: (afterClose?: () => void) => void } - let { aiId, items = [], meltItem, builders }: Props = $props() + let { aiId, items = [], meltItem, builders, close }: Props = $props() let computedItems: Item[] | undefined = $state(undefined) async function computeItems() { @@ -102,8 +103,8 @@ {#if item.separatorTop}
              {/if} - {#if item.submenuItems && builders} - + {#if (item.submenuItems || item.customSubmenu) && builders} + {})} /> {:else if item.disabled && item.tooltip} diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index cf27a45d89..17d752ae39 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -3,6 +3,7 @@ import AIChatMessage from './AIChatMessage.svelte' import AppAvailableContextList from './AppAvailableContextList.svelte' import ChatContextPicker from './ChatContextPicker.svelte' + import WorkspaceMentionPicker from './WorkspaceMentionPicker.svelte' import { type Snippet } from 'svelte' import { AlertTriangle, @@ -548,6 +549,11 @@ await imageWork } + function mentionWorkspaceItem(element: ContextElement) { + void aiChatInput?.addContextToSelection(element) + aiChatInput?.insertMention(element.title) + } + function onFolderInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement // webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups @@ -1055,6 +1061,19 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> } ] : []), + ...(inGlobal + ? [ + { + displayName: 'Mention file', + icon: AtSign, + customSubmenu: WorkspaceMentionPicker, + customSubmenuProps: { + onSelect: mentionWorkspaceItem, + onAfterClose: () => aiChatInput?.focusInput() + } + } + ] + : []), ...(skillItems ? [ { diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 6f15937cf2..bcea3a26d6 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -14,7 +14,7 @@ import { CHAT_INPUT_PADDING, getAiChatManager } from './aiChatManagerContext' import { getChatViewHost } from './chatViewHost' import { composerBoxClass, COMPOSER_FIELD_RESET } from './composerBox' - import { formatMention } from './mention' + import { formatMention, hasMention, removeMentionFromText } from './mention' import { twMerge } from 'tailwind-merge' import { tick, untrack, type Snippet } from 'svelte' import Portal from '$lib/components/Portal.svelte' @@ -588,40 +588,33 @@ const contextKey = contextElementKey - /** Append `@title` to the textarea so the button-picker path stays in - * sync with the inline `@` mention path — both leave a visible - * token tied to the selectedContext entry, which the textarea diffs on - * to auto-remove items when the user deletes them. No-op when the - * mention is already present so re-picking the same item doesn't - * leave duplicate tokens. */ + /** Insert `@title` so button/menu picker paths stay in sync with + * the inline `@` mention path — both leave a visible token tied + * to the selectedContext entry, which the textarea diffs on to + * auto-remove items when the user deletes them. No-op when the same + * standalone mention is already present, so picking the same item + * again doesn't leave duplicate tokens. */ export function insertMention(title: string) { - const target = `@${title}` - if (draft.text.split(/\s+/).includes(target)) return + if (hasMention(draft.text, title)) return + if (contextTextareaComponent) { + void contextTextareaComponent.insertMention(title) + return + } const sep = draft.text.length === 0 || /\s$/.test(draft.text) ? '' : ' ' - draft.text = `${draft.text}${sep}${target} ` + draft.text = `${draft.text}${sep}${formatMention(title)} ` } /** Strip every `@title` token from the textarea — used when the user * deletes the corresponding badge so the badge X-button mirrors the - * inverse (text-delete-to-badge-remove) sync. Only matches `@title` as a - * standalone token (boundary on both sides) so substring matches don't - * bleed into other words; only the whitespace adjacent to the removed - * mention is collapsed so unrelated double-spaces stay intact. */ + * inverse (text-delete-to-badge-remove) sync. Only matches tokens whose + * `@` starts a word, so embedded text like `owner@app.ts` is left alone + * while punctuation and non-spacing text after the token still work. */ export function removeMention(title: string) { // Pre-zap the textarea's mention diff snapshot so the upcoming strip // doesn't refire the removal effect on a same-title sibling — the host // has already mutated `selectedContext` to drop the targeted entry. contextTextareaComponent?.unsyncMention(title) - const escaped = title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const re = new RegExp(`(^|\\s)@${escaped}(\\s|$)`, 'g') - draft.text = draft.text.replace(re, (_m, lead, trail) => { - // Boundary on at least one side → drop the mention entirely. - if (!lead || !trail) return '' - // Middle of text: keep ONE of the bracketing whitespace chars so - // the surviving tokens are still separated; preserve the leading - // one verbatim so newlines/tabs aren't downgraded to spaces. - return lead - }) + draft.text = removeMentionFromText(draft.text, title) } export function focusInput() { diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 976b969b3f..dd3a011da2 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -10,7 +10,13 @@ import { zIndexes } from '$lib/zIndexes' import { twMerge } from 'tailwind-merge' import { CHAT_INPUT_PADDING, getAiChatManager } from './aiChatManagerContext' - import { MENTION_RE, mentionTitle, formatMention } from './mention' + import { + MENTION_RE, + mentionTitle, + formatMention, + hasMentionLeadingBoundary, + mentionTitlesInText + } from './mention' import { createFloatingActions, createVirtualElement } from 'svelte-floating-ui' import { flip, offset, shift } from 'svelte-floating-ui/dom' import { @@ -65,17 +71,11 @@ const aiChatManager = getAiChatManager() - function extractMentions(text: string): Set { - const out = new Set() - for (const m of text.matchAll(MENTION_RE)) out.add(mentionTitle(m[0])) - return out - } - // Titles currently appearing as `@title` mentions in the textarea. Compared // against the previous snapshot in a $effect (NOT inside handleInput — // the picker mutates `value` programmatically via `updateInstructionsWithContext`, // which doesn't fire `oninput`, so a handleInput-only diff goes stale). - const mentionedTitles = $derived(extractMentions(value)) + const mentionedTitles = $derived(mentionTitlesInText(value)) let prevMentionedTitles = $state>(new Set()) let showContextTooltip = $state(false) @@ -268,7 +268,11 @@ if (!att) return match return `${match}` }) - html = html.replace(MENTION_RE, (match) => { + html = html.replace(MENTION_RE, (match, ...args) => { + const offset = args[args.length - 2] + if (typeof offset !== 'number' || !hasMentionLeadingBoundary(html, offset)) { + return match + } const title = unescapeHtml(mentionTitle(match)) const inContext = availableContext.find((c) => c.title === title) || @@ -566,6 +570,23 @@ showContextTooltip = false } + export async function insertMention(title: string) { + const token = formatMention(title) + const selectionStart = textarea?.selectionStart ?? value.length + const selectionEnd = textarea?.selectionEnd ?? selectionStart + const { from, to, ids } = tokensOverlapping(selectionStart, selectionEnd) + const before = value.slice(0, from) + const after = value.slice(to) + const prefix = before.length === 0 || /\s$/.test(before) ? '' : ' ' + const suffix = after.length === 0 ? ' ' : /^\s/.test(after) ? '' : ' ' + const inserted = `${prefix}${token}${suffix}` + replacePasteRange(from, to, ids, inserted) + await tick() + const pos = from + inserted.length + textarea?.setSelectionRange(pos, pos) + textarea?.focus() + } + function refreshCommandSkills() { if (commandSkillsRefreshInFlight) return commandSkillsRefreshInFlight = true diff --git a/frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte b/frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte new file mode 100644 index 0000000000..0fb21f684f --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte @@ -0,0 +1,148 @@ + + + +{#snippet leafIcon(leaf: DrillLeaf)} + +{/snippet} + +{#snippet branchIcon(branch: DrillBranch)} + {#if branch.icon} + {@const Icon = branch.icon} + + {:else} + + {/if} +{/snippet} + + +
              + relativizeWorkspacePath(workspaceItemDisplayPath(leaf.data), scope)} + onScopeChange={handleScopeChange} + onFilterChange={loader.onFilterChange} + rootLoading={WORKSPACE_KINDS.some((k) => !loader.loaded[k] && loader.loadingKind[k])} + /> +
              diff --git a/frontend/src/lib/components/copilot/chat/mention.test.ts b/frontend/src/lib/components/copilot/chat/mention.test.ts index 8d379ce3b4..ae021bf1de 100644 --- a/frontend/src/lib/components/copilot/chat/mention.test.ts +++ b/frontend/src/lib/components/copilot/chat/mention.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest' -import { MENTION_RE, mentionTitle, formatMention } from './mention' +import { + MENTION_RE, + mentionTitle, + formatMention, + hasMention, + mentionTitlesInText, + removeMentionFromText +} from './mention' describe('formatMention', () => { it('leaves a simple name bare', () => { @@ -52,3 +59,83 @@ describe('MENTION_RE', () => { expect(mentionTitle(m[0])).toBe(name) }) }) + +describe('removeMentionFromText', () => { + it('removes a bare mention and keeps neighboring words separated', () => { + expect(removeMentionFromText('before @app.ts after', 'app.ts')).toBe('before after') + }) + + it('removes a bracketed mention', () => { + expect(removeMentionFromText('before @[my folder/a b.ts] after', 'my folder/a b.ts')).toBe( + 'before after' + ) + }) + + it('removes an escaped bracketed mention', () => { + expect(removeMentionFromText('before @[notes \\] draft.md] after', 'notes ] draft.md')).toBe( + 'before after' + ) + }) + + it('removes edge mentions without leaving extra whitespace', () => { + expect(removeMentionFromText('@app.ts after', 'app.ts')).toBe('after') + expect(removeMentionFromText('before @app.ts', 'app.ts')).toBe('before') + }) + + it('keeps embedded mention-like text intact', () => { + expect(removeMentionFromText('Contact owner@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact owner@app.ts about' + ) + }) + + it('keeps Unicode-prefixed embedded mention-like text intact', () => { + expect(removeMentionFromText('Contact 用户@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact 用户@app.ts about' + ) + expect(removeMentionFromText('Contact हिंदी@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact हिंदी@app.ts about' + ) + expect(removeMentionFromText('Contact 𐐀@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact 𐐀@app.ts about' + ) + }) +}) + +describe('hasMention', () => { + it('does not treat embedded mention-like text as a selected context mention', () => { + expect(hasMention('Contact owner@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact owner@app.ts about @app.ts', 'app.ts')).toBe(true) + }) + + it('does not treat Unicode-prefixed embedded text as a mention', () => { + expect(hasMention('Contact 用户@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact हिंदी@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact 𐐀@app.ts', 'app.ts')).toBe(false) + }) + + it('treats punctuation next to a mention as a boundary', () => { + expect(hasMention('use @app.ts, then compare', 'app.ts')).toBe(true) + expect(hasMention('open (@app.ts)', 'app.ts')).toBe(true) + }) +}) + +describe('mentionTitlesInText', () => { + it('extracts only standalone mentions for context synchronization', () => { + expect([...mentionTitlesInText('Contact owner@app.ts about @app.ts')]).toEqual(['app.ts']) + expect([...mentionTitlesInText('Contact owner@app.ts')]).toEqual([]) + }) + + it('ignores Unicode-prefixed embedded text for context synchronization', () => { + expect([...mentionTitlesInText('Contact 用户@app.ts')]).toEqual([]) + }) + + it('keeps punctuation-adjacent mentions synchronized', () => { + expect([...mentionTitlesInText('use @app.ts, then compare')]).toEqual(['app.ts']) + expect([...mentionTitlesInText('open (@app.ts)')]).toEqual(['app.ts']) + }) + + it('ignores combining-mark and astral-letter embedded text', () => { + expect([...mentionTitlesInText('Contact हिंदी@app.ts')]).toEqual([]) + expect([...mentionTitlesInText('Contact 𐐀@app.ts')]).toEqual([]) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/mention.ts b/frontend/src/lib/components/copilot/chat/mention.ts index 65a2b3b7c6..d1d42696b0 100644 --- a/frontend/src/lib/components/copilot/chat/mention.ts +++ b/frontend/src/lib/components/copilot/chat/mention.ts @@ -29,3 +29,59 @@ const BARE_SAFE = /^[\w/.\-]+$/ export function formatMention(name: string): string { return BARE_SAFE.test(name) ? `@${name}` : `@[${name.replace(/[\\\]]/g, '\\$&')}]` } + +function isWordChar(char: string | undefined): boolean { + return char !== undefined && /[\p{L}\p{N}\p{M}_]/u.test(char) +} + +function codePointBefore(text: string, index: number): string | undefined { + if (index <= 0) return undefined + const prev = text.charCodeAt(index - 1) + if (prev >= 0xdc00 && prev <= 0xdfff && index > 1) return text.slice(index - 2, index) + return text[index - 1] +} + +// A mention is owned by this parser when the `@` starts a token. The trailing side is +// intentionally open: non-spacing scripts often continue immediately after a mention, +// while embedded forms like `owner@app.ts` are rejected by the leading boundary. +export function hasMentionLeadingBoundary(text: string, index: number): boolean { + return !isWordChar(codePointBefore(text, index)) +} + +export function isStandaloneMention(text: string, match: RegExpMatchArray): boolean { + if (match.index === undefined) return false + return hasMentionLeadingBoundary(text, match.index) +} + +export function hasMention(text: string, title: string): boolean { + return [...text.matchAll(MENTION_RE)].some( + (m) => mentionTitle(m[0]) === title && isStandaloneMention(text, m) + ) +} + +export function mentionTitlesInText(text: string): Set { + const out = new Set() + for (const m of text.matchAll(MENTION_RE)) { + if (isStandaloneMention(text, m)) out.add(mentionTitle(m[0])) + } + return out +} + +export function removeMentionFromText(text: string, title: string): string { + let out = '' + let last = 0 + for (const m of text.matchAll(MENTION_RE)) { + if (m.index === undefined || mentionTitle(m[0]) !== title) continue + let start = m.index + let end = m.index + m[0].length + if (!isStandaloneMention(text, m)) continue + const hasLead = start > 0 && /\s/.test(text[start - 1]) + const hasTrail = end < text.length && /\s/.test(text[end]) + if (hasLead && !hasTrail) start -= 1 + if (!hasLead && hasTrail) end += 1 + if (hasLead && hasTrail) end += 1 + out += text.slice(last, start) + last = end + } + return out + text.slice(last) +} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 0f9eb78da4..b90751b9d3 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1677,6 +1677,8 @@ export type Item = { iconColor?: string /** Extra props for `icon`, for an icon that does not take lucide's `size`. */ iconProps?: Record + customSubmenu?: import('svelte').Component + customSubmenuProps?: Record href?: string hrefTarget?: '_blank' | '_self' | '_parent' | '_top' disabled?: boolean From efa7a0a70af43f8827542059dd450326a9b88d62 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 10:48:55 +0200 Subject: [PATCH 45/53] feat: allow workspace-scoped dynamic custom tags (#11283) * feat: check dynamic worker tags on the tag they resolve to Co-Authored-By: Claude Opus 5 (1M context) * test: pin the tag check after a flow preprocessor resolves the tag Co-Authored-By: Claude Opus 5 (1M context) * fix: keep tied tag placeholders exact and drop an unresolvable flow tag Co-Authored-By: Claude Opus 5 (1M context) * fix: describe tied tag placeholders in the custom tags editor Co-Authored-By: Claude Opus 5 (1M context) * fix: check workflow-as-code task tags on the tag they resolve to Co-Authored-By: Claude Opus 5 (1M context) * fix: judge a preprocessed flow's tag as written when it is run Co-Authored-By: Claude Opus 5 (1M context) * fix: check a preprocessed flow's own tag, as written only where it reads args Co-Authored-By: Claude Opus 5 (1M context) * feat: explain dynamic tags in the worker tag picker Co-Authored-By: Claude Opus 5 (1M context) * fix: look up worker availability for typed static tags Co-Authored-By: Claude Opus 5 (1M context) * fix: poll worker availability with the tag list as it is at each run Co-Authored-By: Claude Opus 5 (1M context) * feat: allow workspace-scoped dynamic custom tags Co-Authored-By: Claude Opus 5 (1M context) * test: pin the $workspace lookup for scoped custom tag patterns Co-Authored-By: Claude Opus 5 (1M context) * feat: keep a tag listed with its own scope inside it past matching patterns Co-Authored-By: Claude Opus 5 (1M context) * fix: warn when a scoped custom tag is also listed for every workspace Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/tests/worker.rs | 116 ++++++++++++++++++ backend/windmill-common/src/jobs.rs | 69 ++++++++--- backend/windmill-common/src/worker.rs | 94 +++++++++++--- .../lib/components/AssignableTagsInner.svelte | 113 +++++++++++------ 4 files changed, 322 insertions(+), 70 deletions(-) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 74a10488e6..d7fb992280 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5719,6 +5719,122 @@ async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> a Ok(()) } +/// A workspace-scoped pattern admits the tags in its range only from the workspaces its scope +/// allows, and a tag listed by its own name with a scope stays inside that scope whatever pattern +/// it fits. The fork lineage and `$workspace` are resolved for a matching pattern. +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_scoped_custom_tag_pattern_admission(db: Pool) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + + // For a non-superadmin caller (a superadmin would bypass the scope check entirely). + async fn check(db: &Pool, w_id: &str, tag: &str, size: &str) -> Result<(), String> { + let args = std::collections::HashMap::from([( + "size".to_string(), + serde_json::value::to_raw_value(size).unwrap(), + )]); + let args = windmill_queue::PushArgs::from(&args); + windmill_queue::check_tag_available_for_push(db, w_id, tag, &args, false, None) + .await + .map_err(|e| e.to_string()) + } + async fn allowed(db: &Pool, w_id: &str, tag: &str, size: &str) -> bool { + check(db, w_id, tag, size).await.is_ok() + } + async fn allowed_as_written(db: &Pool, w_id: &str, tag: &str) -> bool { + windmill_common::jobs::check_tag_available_for_workspace_internal( + db, + w_id, + tag, + None, + std::future::ready(w_id.to_string()), + false, + None, + ) + .await + .is_ok() + } + + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. + let fork = "wm-fork-tagpattern"; + sqlx::query!( + "INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ($1, $1, 'test-user', 'test-workspace')", + fork + ) + .execute(&db) + .await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "gpu-$args[size]".to_string(), + "$args[size]-secret(test-workspace)".to_string(), + "gpu-secret(other)".to_string(), + "$workspace-$args[size](test-workspace*)".to_string(), + "urgent".to_string(), + "urgent(other)".to_string(), + ]))); + + let confined = check(&db, "test-workspace", "gpu-$args[size]", "secret").await; + let checks = [ + ( + "scoped pattern from its workspace", + allowed(&db, "test-workspace", "cpu-secret", "").await, + ), + ( + "scoped pattern refused elsewhere", + !allowed(&db, "other", "cpu-secret", "").await, + ), + ("confined tag refused past the patterns it fits", confined.is_err()), + ( + "global pattern for the tags nothing confines", + allowed(&db, "test-workspace", "gpu-large", "").await, + ), + ( + "confined tag from its workspace", + allowed(&db, "other", "gpu-secret", "").await, + ), + // Listing the tag by name without a scope as well keeps it open everywhere. + ( + "tag listed both globally and scoped", + allowed(&db, "test-workspace", "urgent", "").await, + ), + // The fork's `$workspace` is its parent's, whose tags its lineage reaches. Written as the + // resolved tag, not as the entry, so only the pattern can admit it. + ( + "fork-scoped `$workspace` pattern from a fork", + allowed(&db, fork, "test-workspace-large", "").await, + ), + ( + "fork-scoped `$workspace` pattern refused elsewhere", + !allowed(&db, "other", "$workspace-$args[size]", "large").await, + ), + ( + "as written from its workspace", + allowed_as_written(&db, "test-workspace", "$args[size]-secret").await, + ), + ( + "as written refused elsewhere", + !allowed_as_written(&db, "other", "$args[size]-secret").await, + ), + ]; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + for (what, ok) in checks { + assert!(ok, "{what}"); + } + let confined = confined.unwrap_err(); + assert!( + confined.contains("the custom tag gpu-secret(other) restricts it to other workspaces") + && confined.contains("gpu-$args[size]") + && confined.contains("$args[size]-secret(test-workspace)"), + "got {confined}" + ); + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_whileloop_propagates_inner_iterator_eval_failure( diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index f2b0e16dac..796d1c999a 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -363,6 +363,7 @@ pub async fn check_tag_available_for_workspace_internal( && custom_tags_per_w .global .iter() + .chain(custom_tags_per_w.specific.keys()) .any(|t| t.contains("$workspace")) { tag_workspace.await @@ -371,21 +372,35 @@ pub async fn check_tag_available_for_workspace_internal( }; // A job whose tag is written exactly as an entry resolves inside what that entry admits, which // is the only way in for an entry `custom_tag_matches` cannot turn into a pattern. - let mut is_tag_in_workspace_custom_tags = custom_tags_per_w.global.iter().any(|entry| { + let admits = |entry: &str| { entry == tag || resolved_tag.is_some_and(|r| custom_tag_matches(entry, r, &tag_workspace)) - }); - if !is_tag_in_workspace_custom_tags { - if let Some(specific_tag) = custom_tags_per_w.specific.get(resolved_tag.unwrap_or(tag)) { - // Only a fork-scoped tag can match through the lineage, so every other tag keeps the - // ancestor lookup off the push path entirely. - let chain = if specific_tag.is_fork_scoped() { - workspace_with_fork_ancestors(db, w_id).await? - } else { - vec![w_id.to_string()] - }; - is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); - } - } + }; + let scoped = custom_tags_per_w + .specific + .iter() + .filter(|(name, _)| admits(name)) + .collect::>(); + // Only a fork-scoped tag can match through the lineage, so every other tag keeps the + // ancestor lookup off the push path entirely. + let chain = if scoped.iter().any(|(_, t)| t.is_fork_scoped()) { + workspace_with_fork_ancestors(db, w_id).await? + } else { + vec![w_id.to_string()] + }; + // A tag listed by its own name with a scope stays inside that scope: a pattern it fits does + // not open it to other workspaces. Only listing it by name without a scope does. + let lands_on = resolved_tag.unwrap_or(tag); + let confined_by = custom_tags_per_w + .specific + .get_key_value(lands_on) + .filter(|(name, t)| { + !name.contains('$') + && !t.applies_to_workspace(&chain) + && !custom_tags_per_w.global.iter().any(|e| e == lands_on) + }); + let is_tag_in_workspace_custom_tags = confined_by.is_none() + && (custom_tags_per_w.global.iter().any(|entry| admits(entry)) + || scoped.iter().any(|(_, t)| t.applies_to_workspace(&chain))); match is_tag_in_scope_tags { Some(true) | None => { @@ -411,6 +426,32 @@ pub async fn check_tag_available_for_workspace_internal( if *TAGS_ARE_SENSITIVE { return Err(Error::BadRequest(format!("{tag} is not available to you"))); + } else if let Some((name, confining)) = confined_by { + let overridden = custom_tags_per_w + .global + .iter() + .filter(|entry| admits(entry)) + .cloned() + .chain( + scoped + .iter() + .filter(|(_, t)| t.applies_to_workspace(&chain)) + .map(|(n, t)| t.authored(n)), + ) + .collect::>(); + let overridden = if overridden.is_empty() { + String::new() + } else { + format!( + ", which takes precedence over the custom tags it also fits: {}", + overridden.join(", ") + ) + }; + return Err(Error::BadRequest(format!( + "Tag {tag} is not available in workspace {w_id}: the custom tag {} restricts it \ + to other workspaces{overridden}", + confining.authored(name) + ))); } else { return Err(error::Error::BadRequest(format!( "Tag {tag} is not included in the allowed CUSTOM_TAGS, which only super admins can go beyond: {:?}", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 548dbe07b4..ad0a5068e8 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -41,6 +41,8 @@ use crate::{ #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] pub struct CustomTags { pub global: Vec, + /// Keyed by the entry's name, which may hold placeholders like a global entry: only a name + /// without any can be looked up by the tag it admits, the others through [`custom_tag_matches`]. pub specific: HashMap, } @@ -83,20 +85,7 @@ impl CustomTags { } else { self.specific .iter() - .map(|(tag, tag_data)| { - let separator = tag_data.tag_type.corresponding_separator(); - let mut workspaces = tag_data - .workspaces - .iter() - .map(|w| w.to_string()) - .collect::>() - .join(&*separator.to_string()); - if tag_data.tag_type == SpecificTagType::AllExcluding { - // the AllExcluding tag syntax has a leading separator - workspaces.insert(0, separator); - } - format!("{}({})", tag, workspaces) - }) + .map(|(tag, tag_data)| tag_data.authored(tag)) .collect::>() }; let all_tags = self.global.clone(); @@ -261,6 +250,22 @@ impl SpecificTagData { pub fn is_fork_scoped(&self) -> bool { self.workspaces.iter().any(|w| w.include_forks) } + + /// The entry named `name` with this scope, as written in the custom tags: `tag(ws1+ws2)`. + pub fn authored(&self, name: &str) -> String { + let separator = self.tag_type.corresponding_separator(); + let mut workspaces = self + .workspaces + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(&*separator.to_string()); + if self.tag_type == SpecificTagType::AllExcluding { + // the AllExcluding tag syntax has a leading separator + workspaces.insert(0, separator); + } + format!("{}({})", name, workspaces) + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum SpecificTagType { @@ -542,7 +547,9 @@ lazy_static::lazy_static! { - // ^([\w-]+) # Group 1: tag name + // ^( # Group 1: tag name, a pattern when it holds placeholders + // (?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+ + // ) // \( # Literal '(' // ( # Group 2: the full workspace list // (?:[\w-]+\*?\+)*[\w-]+\*? # NoneExcept pattern: ws1+ws2* @@ -551,8 +558,9 @@ lazy_static::lazy_static! { // ) // \)$ # Closing ')' // - // The optional `*` after each workspace id is the fork marker, see [`WorkspaceMatcher`]. - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); + // The placeholders are CUSTOM_TAG_PLACEHOLDER's. The optional `*` after each workspace id is + // the fork marker, see [`WorkspaceMatcher`]. + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^((?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); // The placeholders a job's tag is resolved from when it is pushed, see [`custom_tag_matches`]. static ref CUSTOM_TAG_PLACEHOLDER: Regex = Regex::new(r"\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\]").unwrap(); @@ -3185,6 +3193,58 @@ mod tests { assert!(!matches("$workspace-$args[size]", "ws2-large")); } + #[test] + fn test_scoped_custom_tag_patterns_parse_and_round_trip() { + let input = vec![ + "gpu-$args[size](ws1+ws2)".to_string(), + "cpu-$flow_expr[results.a.size](^ws1)".to_string(), + "$workspace-$args[x](prod*)".to_string(), + "t-$args[id]-$args[id](ws1)".to_string(), + ]; + let tags = CustomTags::from(input.clone()); + + assert!(tags.global.is_empty()); + assert_eq!( + tags.specific["gpu-$args[size]"], + SpecificTagData { + tag_type: SpecificTagType::NoneExcept, + workspaces: vec![matcher("ws1"), matcher("ws2")], + } + ); + assert_eq!( + tags.specific["cpu-$flow_expr[results.a.size]"], + SpecificTagData { + tag_type: SpecificTagType::AllExcluding, + workspaces: vec![matcher("ws1")], + } + ); + assert!(tags.specific["$workspace-$args[x]"].is_fork_scoped()); + assert!(tags.specific.contains_key("t-$args[id]-$args[id]")); + + let mut result = tags.to_string_vec(None); + result.sort(); + let mut expected = input; + expected.sort(); + assert_eq!(result, expected); + + let mut for_ws1 = tags.to_string_vec(Some(&chain(&["ws1"]))); + for_ws1.sort(); + assert_eq!(for_ws1, vec!["gpu-$args[size]", "t-$args[id]-$args[id]"]); + + // Only the placeholders a tag resolves from make a name a pattern: anything else is no + // scoped entry at all. + for literal in [ + "gpu-$foo(ws1)", + "gpu-$args[](ws1)", + "gpu-$args[a.](ws1)", + "gpu.x(ws1)", + ] { + let tags = CustomTags::from(vec![literal.to_string()]); + assert_eq!(tags.global, vec![literal]); + assert!(tags.specific.is_empty()); + } + } + #[test] fn test_fork_marker_parses_and_round_trips() { let tags = CustomTags::from(vec![ diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index f385ff2b49..fd5859b784 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -39,7 +39,8 @@ const dispatch = createEventDispatcher() // Mirrors CUSTOM_TAG_REGEX in backend/windmill-common/src/worker.rs — keep both in sync. - const customTagRegex = /^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$/ + const customTagRegex = + /^((?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$/ // Mirrors RE_ARG_TAG and RE_FLOW_EXPR_TAG in backend/windmill-queue/src/jobs.rs. const dynamicTagRegex = /\$(args|flow_expr)\[((?:\w+\.)*\w+)\]/ @@ -47,10 +48,16 @@ return w.includeForks ? `${w.id} (and its forks)` : w.id } + // What an entry matches job tags against: a workspace specific entry's name, which may be + // dynamic like any other entry. + function nameOf(entry: string) { + return entry.match(customTagRegex)?.[1] ?? entry + } + let tagName = $derived(nameOf(newTag.trim())) + let dynamicTag = $derived.by(() => { - let r = newTag.trim() - if (r == '') return undefined - let matched = r.match(dynamicTagRegex) + if (tagName == '') return undefined + let matched = tagName.match(dynamicTagRegex) return matched ? { kind: matched[1], path: matched[2] } : undefined }) @@ -88,8 +95,8 @@ placeholders[placeholders.length - 1].end == entry.length ) } - let dynamicTagTied = $derived(isTied(newTag.trim())) - let dynamicTagReachesAnyTag = $derived(reachesAnyTag(newTag.trim())) + let dynamicTagTied = $derived(isTied(tagName)) + let dynamicTagReachesAnyTag = $derived(reachesAnyTag(tagName)) let extractedCustomTag = $derived.by(() => { let r = newTag.trim() @@ -164,10 +171,40 @@ -{#snippet reachesAnyTagWarning()} - Nothing fixed at its start or its end, so it allows anyone to reach almost any tag. We highly - recommend a prefix or a suffix, ideally both, to limit its reach: for instance - gpu-$args[size] or gpu-$args[size]-eu. +{#snippet reachesAnyTagWarning(scoped: boolean)} + Nothing fixed at its start or its end, so it allows anyone{scoped ? ' in its workspaces' : ''} to reach + almost any tag. We highly recommend a prefix or a suffix, ideally both, to limit its reach: for instance + gpu-$args[size] + or gpu-$args[size]-eu. +{/snippet} + +{#snippet dynamicTagInfo(scoped: boolean)} + {#if tagName.includes('$workspace')} +
              Interpolated tag based on workspace id the job was created in
              + {/if} + {#if dynamicTag?.kind == 'flow_expr'} +
              + Interpolated tag based on the flow value at {dynamicTag.path}, resolved when the flow + step starts +
              + {:else if dynamicTag} +
              Interpolated tag based on args input of {dynamicTag.path}
              + {/if} + {#if dynamicTagTied} +
              + Its placeholders read the same value, or one reads inside the other, so it allows only jobs + whose tag is written exactly like this +
              + {:else if dynamicTagReachesAnyTag} +
              + {@render reachesAnyTagWarning(scoped)} +
              + {:else if dynamicTag} +
              + Allows any tag it resolves to{scoped ? ' in these workspaces' : ''}, with the text around the + placeholder as written +
              + {/if} {/snippet}
              {#each customTags as customTag} + {@const name = nameOf(customTag)} {customTag} - {#if reachesAnyTag(customTag)} + {#if reachesAnyTag(name)} - {@render reachesAnyTagWarning()} + {@render reachesAnyTagWarning(name != customTag)}
              {/snippet} @@ -229,8 +267,11 @@
              {#if extractedCustomTag} + {@const dynamic = tagName.includes('$workspace') || dynamicTag != undefined}
              -
              Workspace specific tag
              +
              + Workspace specific {#if dynamic}dynamic tag{:else}tag{/if} +
              Tag: {extractedCustomTag.tag} @@ -243,6 +284,20 @@ All workspaces except {extractedCustomTag.workspaces?.map(formatWorkspace).join(', ')} {/if}
              + {@render dynamicTagInfo(true)} + + {#if !dynamic && tagEditor} + {#if (customTags ?? []).includes(tagName)} +
              + {tagName} is also listed without workspaces, which keeps it open to every + workspace +
              + {:else} +
              + Workspaces outside this scope cannot use it, even through a dynamic tag it fits +
              + {/if} + {/if}
              {:else if newTag.trim()} {#if newTag.includes('(') || newTag.includes(')') || newTag.includes('+') || newTag.includes('^') || newTag.includes('*') || ((newTag.includes('.') || newTag.includes('$args[') || newTag.includes('$flow_expr[')) && !dynamicTag)} @@ -266,31 +321,7 @@ Tag: {newTag.trim()}
              - {#if newTag.includes('$workspace') && !dynamicTag} -
              Interpolated tag based on workspace id the job was created in
              - {/if} - {#if dynamicTag?.kind == 'flow_expr'} -
              - Interpolated tag based on the flow value at {dynamicTag.path}, resolved when - the flow step starts -
              - {:else if dynamicTag} -
              Interpolated tag based on args input of {dynamicTag.path}
              - {/if} - {#if dynamicTagTied} -
              - Its placeholders read the same value, or one reads inside the other, so it allows only - jobs whose tag is written exactly like this -
              - {:else if dynamicTagReachesAnyTag} -
              - {@render reachesAnyTagWarning()} -
              - {:else if dynamicTag} -
              - Allows any tag it resolves to, with the text around the placeholder as written -
              - {/if} + {@render dynamicTagInfo(false)}
              {/if} {/if} @@ -348,7 +379,11 @@ A dynamic tag is checked on the tag it resolves to: it is allowed when that tag is listed here, or fits a listed dynamic tag such as 
              gpu-$args[size]
              , which allows any tag starting with - 
              gpu-
              . + 
              gpu-
              . Scope it like any other tag, e.g. + 
              gpu-$args[size](workspace1+workspace2)
              , to allow those + tags only in some workspaces. A tag listed by its own name for some workspaces, e.g. + 
              gpu-secret(workspace1)
              , stays limited to them even + when a dynamic tag fits it, unless it is also listed without workspaces. {/if}
              From 7d0f2ce3bf920a68f1ea22d59481d239947f1b51 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 11:32:11 +0200 Subject: [PATCH 46/53] fix: size the AI chat output budget by model, not provider (#11286) * fix: size the AI chat output budget by model, not provider Co-Authored-By: Claude Opus 5 (1M context) * fix: keep self-hosted open-weight models on the fallback output budget Co-Authored-By: Claude Opus 5 (1M context) * chore: point ee-repo-ref at the free tier output clamp change Co-Authored-By: Claude Opus 5 (1M context) * fix: treat codestral as open-weight for the output budget Co-Authored-By: Claude Opus 5 (1M context) * chore: update ee-repo-ref to 559a9ff78ea03439553cf2b412e73765762e9013 This commit updates the EE repository reference after PR #821 was merged in windmill-ee-private. Previous ee-repo-ref: 953f7c1cc9a70b740d5d1b129cb1401950854b02 New ee-repo-ref: 559a9ff78ea03439553cf2b412e73765762e9013 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../copilot/chat/AIChatManager.svelte.ts | 4 +- .../lib/components/copilot/chat/anthropic.ts | 3 +- .../copilot/chat/openai-responses.ts | 6 +- .../copilot/chat/outputTokenLimit.ts | 5 + .../components/copilot/lib.toolCalls.test.ts | 36 ++++++- frontend/src/lib/components/copilot/lib.ts | 100 ++++++++++-------- .../src/lib/components/copilot/modelConfig.ts | 65 ++++++++++++ .../components/copilot/reasoningRegistry.ts | 17 ++- .../workspaceSettings/AISettings.svelte | 4 +- 10 files changed, 185 insertions(+), 57 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c336c4fdd3..b82e25d93b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5370ae3a95a3dc10171f72da74286a654996eee6 +559a9ff78ea03439553cf2b412e73765762e9013 diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 871dead4bc..27f9bfc43f 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -110,7 +110,7 @@ import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { closeInterruptedToolBatch, runChatLoop, truncateToToolPairedPrefix } from './chatLoop' -import { OutputTokenLimitError } from './outputTokenLimit' +import { FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE, OutputTokenLimitError } from './outputTokenLimit' import { sanitizeToolCallArguments } from './toolCallArguments' import { billedTokens, normalizeContextUsage, type ChatTokenUsage } from './tokenUsage' import { logAiUsage } from '$lib/utils/aiUsageReporter' @@ -378,7 +378,7 @@ function isImageRejection(err: unknown, models: (string | undefined)[] = []): bo function getSendRequestErrorMessage(err: unknown, webSearchUnavailable: boolean): string { // The request went through; only its response was cut short. if (err instanceof OutputTokenLimitError) { - return err.message + return get(copilotInfo).freeTier ? FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE : err.message } const errorMessage = err instanceof Error ? err.message : typeof err === 'string' ? err : undefined diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index a311d16990..122425e264 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -141,7 +141,8 @@ export async function getAnthropicCompletion( const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) let anthropicTools = convertOpenAIToolsToAnthropic(tools) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index b216a08107..0e460f1921 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -272,7 +272,8 @@ export async function getOpenAIResponsesCompletion( messages, stream: true, tools, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { instructions, input } = convertMessagesToResponsesInput(messages) const responsesConfig = applyReasoningToConfig( @@ -331,7 +332,8 @@ export async function* getOpenAIResponsesCompletionStream( messages, stream: true, tools, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { instructions, input } = convertMessagesToResponsesInput(messages) // No prompt cache key here: a rejected key has to be retried without it, and this is diff --git a/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts index b1127a1db7..19b74f0c68 100644 --- a/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts +++ b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts @@ -10,3 +10,8 @@ export class OutputTokenLimitError extends Error { this.name = 'OutputTokenLimitError' } } + +/** Shown instead on Windmill's free tier, whose server clamps the output below any + * workspace setting: pointing at that setting would send the user to a no-op. */ +export const FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE = + "The response was cut off at the free tier's output token limit. Ask it to continue, or add your own API key in the workspace AI settings for a higher limit." diff --git a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts index b011c1f672..6966e018d1 100644 --- a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts +++ b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts @@ -287,10 +287,42 @@ describe('parseOpenAICompletion tool call arguments', () => { }) describe('output token limit', () => { - it("gives DeepSeek DeepSeek's own thinking-mode budget rather than the 8192 fallback", async () => { + it('follows the model through every host that serves it', async () => { const { getModelMaxTokens } = await import('./lib') expect(getModelMaxTokens('deepseek', 'deepseek-flash')).toBe(131072) - expect(getModelMaxTokens('deepseek', 'deepseek-v4-pro')).toBe(131072) + expect(getModelMaxTokens('azure_foundry', 'DeepSeek-V4-Pro')).toBe(131072) + expect(getModelMaxTokens('openrouter', 'deepseek/deepseek-v4-pro')).toBe(131072) + // gpt-oss reasons whether or not the chat sends an effort. + expect(getModelMaxTokens('groq', 'openai/gpt-oss-120b')).toBe(16000) + expect(getModelMaxTokens('openrouter', 'openai/o3')).toBe(100000) + // A legacy `/thinking` selection is its base model, not the `thinking` row. + expect(getModelMaxTokens('anthropic', 'claude-sonnet-4-5/thinking')).toBe(64000) + }) + + it('gives a model outside the table room to think only when the request asks it to', async () => { + const { getModelMaxTokens } = await import('./lib') + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3', 'high')).toBe(32768) + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3', 'none')).toBe(8192) + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3')).toBe(8192) + // The registry does not know Custom AI models, so an effort says nothing about them. + expect(getModelMaxTokens('customai', 'local-model', 'high')).toBe(8192) + // A self-hosted open-weight model runs at its operator's context length, not the + // vendor's, while a Custom AI proxy in front of a hosted model keeps its row. + expect(getModelMaxTokens('customai', 'deepseek-v4-pro', 'high')).toBe(8192) + expect(getModelMaxTokens('customai', 'gpt-5')).toBe(128000) + }) + + it('sizes the budget from the effort the same request carries', async () => { + const { getCompletion } = await import('./lib') + const create = vi.fn() + + await getCompletion([], new AbortController(), undefined, { + forceModelProvider: { provider: 'mistral', model: 'mistral-medium-latest' }, + openaiClient: { chat: { completions: { create } } } as any, + reasoningEffort: 'high' + }) + + expect(create.mock.calls[0][0]).toMatchObject({ max_tokens: 32768, reasoning_effort: 'high' }) }) it('fails a response cut off at the output token limit instead of ending the turn', async () => { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 5094764807..b55aa06c92 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -15,11 +15,17 @@ import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { + getKnownModelMaxOutputTokens, requiresMaxCompletionTokens, usesAnthropicMessagesApi, usesOpenRouterPromptCaching } from './modelConfig' -import { applyReasoningToConfig } from './reasoningRegistry' +import { + applyReasoningToConfig, + requestsReasoning, + stripLegacyThinkingSuffix, + type ReasoningEffort +} from './reasoningRegistry' import { formatResourceTypes } from './utils' import { appendPendingToolImages, @@ -310,45 +316,31 @@ export async function fetchAvailableModels( return data?.data.map((m) => m.id) ?? [] } -export function getModelMaxTokens(provider: AIProvider, model: string) { - if (provider === 'deepseek') { - // DeepSeek thinks by default and counts the thinking toward max_tokens, so - // the 8192 fallback cuts long turns off mid-thought. 131072 is DeepSeek's - // own default at the highest effort (65536 at the default one). - return 131072 - } else if (model.includes('gpt-5')) { - return 128000 - } else if ( - (provider === 'azure_openai' || provider === 'openai' || provider === 'azure_foundry') && - model.startsWith('o') - ) { - return 100000 - } else if ( - // Raising this further would also raise the worst case of the - // non-streaming completion path, which the Anthropic SDK refuses once - // the request could run past ~10 minutes. - model.includes('claude-sonnet') || - model.includes('claude-haiku') || - model.includes('claude-fable') || - model.includes('claude-mythos') || - // Opus only from 4.5 on. Opus 4.1 and older cap at 32K and fall through - // to the row below. Dots are normalized because OpenRouter writes - // `anthropic/claude-opus-4.5` where Anthropic writes `claude-opus-4-5`. - /claude-opus-(4-(5|6|7|8)|5)(?!\d)/.test(model.replace(/\./g, '-')) || - model.includes('gemini-2.5') || - model.includes('gemini-3') - ) { - return 64000 - } else if (model.includes('gpt-4.1')) { - return 32768 - } else if (model.includes('claude-opus')) { - return 32000 - } else if (model.includes('gpt-4o') || model.includes('codestral')) { - return 16384 - } else if (model.includes('gpt-4-turbo') || model.includes('gpt-3.5')) { - return 4096 - } - return 8192 +// Thinking counts toward max_tokens, so a model outside the table that is asked to +// reason gets more room than the plain fallback. A host rejects a budget above the +// model's cap, so the reasoning fallback must stay within the cap of every model it +// reaches (true of every reasoning model OpenRouter serves outside the table). Custom +// AI never gets it: the registry does not know its models, and the per-model override +// is its way to a larger budget. +const FALLBACK_MAX_TOKENS = 8192 +const REASONING_FALLBACK_MAX_TOKENS = 32768 + +/** + * The output budget a request sends when the workspace sets no override. `reasoningEffort` + * is the effort the request carries, as `resolveRequestReasoning` resolves it. + */ +export function getModelMaxTokens( + provider: AIProvider, + model: string, + reasoningEffort?: ReasoningEffort +): number { + const bareModel = stripLegacyThinkingSuffix(model) + return ( + getKnownModelMaxOutputTokens(provider, bareModel) ?? + (requestsReasoning(provider, bareModel, reasoningEffort) + ? REASONING_FALLBACK_MAX_TOKENS + : FALLBACK_MAX_TOKENS) + ) } // Resolves the completion token cap for a model: the workspace's per-model @@ -356,8 +348,16 @@ export function getModelMaxTokens(provider: AIProvider, model: string) { // Anthropic request paths so both honor the same limit. `cap` bounds the result // (used by short metadata completions, see METADATA_MAX_TOKENS) — a hard ceiling // that wins over both the workspace override and the default. -function resolveMaxTokens(modelProvider: AIProviderModel, cap?: number): number { - const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) +function resolveMaxTokens( + modelProvider: AIProviderModel, + cap?: number, + reasoningEffort?: ReasoningEffort +): number { + const defaultMaxTokens = getModelMaxTokens( + modelProvider.provider, + modelProvider.model, + reasoningEffort + ) const modelKey = `${modelProvider.provider}:${modelProvider.model}` let customMaxTokensStore: Record | undefined try { @@ -379,9 +379,10 @@ export const METADATA_MAX_TOKENS = 4096 function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[], - maxTokensCap?: number + maxTokensCap?: number, + reasoningEffort?: ReasoningEffort ) { - const maxTokens = resolveMaxTokens(modelProvider, maxTokensCap) + const maxTokens = resolveMaxTokens(modelProvider, maxTokensCap, reasoningEffort) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai' || @@ -903,7 +904,8 @@ export function getProviderAndCompletionConfig({ tools, forceModelProvider, maxTokensCap, - promptCaching + promptCaching, + reasoningEffort }: { messages: ChatCompletionMessageParam[] stream: K @@ -913,6 +915,9 @@ export function getProviderAndCompletionConfig({ // Opt-in: a cache write costs more than an uncached read, so it only pays off where // the same prefix is sent again. True for the chat loop, false for one-shot calls. promptCaching?: boolean + // The effort the caller will add with `applyReasoningToConfig`. Only sizes the + // output budget here: a request that reasons needs room for the thinking. + reasoningEffort?: ReasoningEffort }): { provider: AIProvider config: K extends true @@ -929,7 +934,7 @@ export function getProviderAndCompletionConfig({ provider: modelProvider.provider, config: { ...providerConfig, - ...getModelSpecificConfig(modelProvider, tools, maxTokensCap), + ...getModelSpecificConfig(modelProvider, tools, maxTokensCap, reasoningEffort), messages: processedMessages, stream } as any @@ -1135,7 +1140,8 @@ export async function getCompletion( stream: true, tools, forceModelProvider: options?.forceModelProvider, - promptCaching: options?.promptCaching + promptCaching: options?.promptCaching, + reasoningEffort: options?.reasoningEffort }) // Use Responses API for OpenAI and Azure OpenAI diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index be49907ed4..77d0cf942a 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -117,6 +117,58 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['codestral', 32_000] ] +// Output token budgets, matched like the context windows above so one row covers +// every host of a model: native API, Azure AI Foundry, OpenRouter, Together, Bedrock. +// A host rejects any request above its own limit, so each value is one every host of +// the model accepts, not the vendor's maximum. Models not listed fall back in +// `getModelMaxTokens`. +const MODEL_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ + // Anthropic — the Anthropic SDK refuses a non-streaming request whose max_tokens + // implies more than ~10 minutes, so these stay below the models' own maximum. + // Opus caps at 32K before 4.5. + ['claude-opus-4.5', 64_000], + ['claude-opus-4.6', 64_000], + ['claude-opus-4.7', 64_000], + ['claude-opus-4.8', 64_000], + ['claude-opus-5', 64_000], + ['claude-opus', 32_000], + ['claude-sonnet', 64_000], + ['claude-haiku', 64_000], + ['claude-fable', 64_000], + ['claude-mythos', 64_000], + // OpenAI + ['gpt-5', 128_000], + ['gpt-4.1', 32_768], + ['gpt-4o', 16_384], + ['gpt-4-turbo', 4_096], + ['gpt-3.5', 4_096], + ['o1', 100_000], + ['o3', 100_000], + ['o4-mini', 100_000], + // Google + ['gemini-3', 64_000], + ['gemini-2.5', 64_000] +] + +// Open-weight models, which a self-hosted server (Custom AI) caps at whatever context +// length its operator set, often below these: vLLM rejects a request whose prompt plus +// max_tokens exceeds it. So these rows only apply to the providers that host them. +const OPEN_WEIGHT_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ + // gpt-oss reasons on every request. Bedrock documents 16K, where Groq takes 65536 + ['gpt-oss', 16_000], + // DeepSeek — thinks by default and counts the thinking toward max_tokens. 131072 + // is DeepSeek's own default at the highest effort. R1 is capped by its OpenRouter + // host. + ['deepseek-v4', 131_072], + ['deepseek-flash', 131_072], + ['deepseek-r1', 16_000], + // Mistral + ['codestral', 16_384], + // Models named for thinking reason on every request, whether or not the chat sends + // an effort. Every one OpenRouter lists takes at least this much. + ['thinking', 32_768] +] + // Version separators differ by route to the same model: Anthropic writes // `claude-opus-4-8`, OpenRouter writes `anthropic/claude-opus-4.8`. Collapsing // dots to dashes on both sides keeps one table entry covering every route — @@ -197,6 +249,19 @@ export function getKnownModelContextWindow(model: string): number | undefined { return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } +const MODEL_MAX_OUTPUT_TOKEN_MATCHERS = buildModelMatchers(MODEL_MAX_OUTPUT_TOKENS) +const OPEN_WEIGHT_MAX_OUTPUT_TOKEN_MATCHERS = buildModelMatchers(OPEN_WEIGHT_MAX_OUTPUT_TOKENS) + +export function getKnownModelMaxOutputTokens( + provider: AIProvider, + model: string +): number | undefined { + return ( + matchModel(MODEL_MAX_OUTPUT_TOKEN_MATCHERS, model) ?? + (provider === 'customai' ? undefined : matchModel(OPEN_WEIGHT_MAX_OUTPUT_TOKEN_MATCHERS, model)) + ) +} + /** Trim/compaction logic needs a number; assume a conservative window when unknown. */ export const ASSUMED_CONTEXT_WINDOW = 128000 diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 242f6cb329..9915e9c42f 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -374,7 +374,8 @@ const ANTHROPIC_ALWAYS_THINKING = /fable|mythos/ /** * Disable token to forward when the user explicitly turns reasoning off on a * model that reasons *by default* — omitting the field would silently keep - * the default-on behavior. Undefined means omission is the correct off. + * the default-on behavior. Undefined means omission is the correct off. Every + * token is `'none'`: `requestsReasoning` reads that value as off. */ export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { switch (reasoningProviderFamily(provider, model)) { @@ -434,6 +435,20 @@ export function resolveRequestReasoning( return undefined } +/** + * Whether a request carrying `effort`, as `resolveRequestReasoning` resolves it, + * has the model reason. An effort sent to a model the registry does not mark as + * reasoning-capable answers no: an explicit choice is always sent, but that does + * not make the model think. + */ +export function requestsReasoning( + provider: AIProvider, + model: string, + effort: ReasoningEffort | undefined +): boolean { + return !!effort && effort !== 'none' && supportsReasoning(provider, model) +} + export type ReasoningApiKind = 'anthropic' | 'responses' | 'completions' | 'deepseek' | 'mistral' /** diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 3957f47b74..05159e3057 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -21,6 +21,7 @@ getKnownModelContextWindow, getModelContextWindowFromTable } from '../copilot/modelConfig' + import { resolveRequestReasoning } from '../copilot/reasoningRegistry' import { supportsAutocomplete } from '../copilot/utils' import TestAiKey from '../copilot/TestAIKey.svelte' import Label from '../Label.svelte' @@ -650,7 +651,8 @@ min={1} max={2_000_000} getDefault={(provider, model) => ({ - tokens: getModelMaxTokens(provider, model), + // What a chat sends at the default effort; the budget depends on it. + tokens: getModelMaxTokens(provider, model, resolveRequestReasoning({ provider, model })), assumed: false })} /> From 02006f364a4217e967435d9e71d8617c03b65413 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 13:03:30 +0200 Subject: [PATCH 47/53] fix: keep a cancel that lands after a worker last read its queue row (#11246) * fix: keep a cancel that lands after a worker last read its queue row Co-Authored-By: Claude Opus 5 * fix: take the cancellation from the queue row the completion deletes Co-Authored-By: Claude Opus 5 * test: wait for the completion to block on the row instead of a timer Co-Authored-By: Claude Opus 5 * refactor: settle a missed cancel from the deleted queue row alone Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- ...dde0be9e0d192e0aee224bff78d8bd2ccbd8b.json | 16 ++ ...99c2e1e9ca0ed33077ade8d7560e7cc21fa06.json | 22 --- ...933eeff7396430607c02cbd0cc682d5c34d61.json | 23 +++ ...f86505bc499320e08d1aad15a9a82c3823c0b.json | 28 +++ backend/tests/perpetual_cancel_race.rs | 183 ++++++++++++++++++ backend/windmill-queue/src/jobs.rs | 76 ++++++-- 6 files changed, 310 insertions(+), 38 deletions(-) create mode 100644 backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json delete mode 100644 backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json create mode 100644 backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json create mode 100644 backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json create mode 100644 backend/tests/perpetual_cancel_race.rs diff --git a/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json b/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json new file mode 100644 index 0000000000..0bc09d9430 --- /dev/null +++ b/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET status = 'canceled'::job_status, canceled_by = $2, canceled_reason = $3 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b" +} diff --git a/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json b/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json deleted file mode 100644 index 8a4b957c2a..0000000000 --- a/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06" -} diff --git a/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json b/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json new file mode 100644 index 0000000000..9930e0494f --- /dev/null +++ b/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61" +} diff --git a/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json b/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json new file mode 100644 index 0000000000..b11e41fe89 --- /dev/null +++ b/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING canceled_by, canceled_reason", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "canceled_reason", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b" +} diff --git a/backend/tests/perpetual_cancel_race.rs b/backend/tests/perpetual_cancel_race.rs new file mode 100644 index 0000000000..df239eb7ab --- /dev/null +++ b/backend/tests/perpetual_cancel_race.rs @@ -0,0 +1,183 @@ +//! A cancel reaches the queue row first and the worker only later: one landing after the worker's +//! last read of that row completes the run carrying no `canceled_by` of its own, and a perpetual +//! script must still stop rather than restart as if nothing had been cancelled. + +use serde_json::json; +use sqlx::{types::Json, Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::{add_completed_job, get_mini_completed_job}; + +const W_ID: &str = "test-workspace"; +const PATH: &str = "u/test-user/loop"; + +async fn start_perpetual_run(db: &Pool) -> anyhow::Result { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, \ + language, lock, restart_unless_cancelled) \ + VALUES ($1, 1, $2, '', '', 'echo', 'test-user', 'bash', '', true)", + ) + .bind(W_ID) + .bind(PATH) + .execute(db) + .await?; + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, runnable_id, runnable_path, script_lang, tag, args, \ + visible_to_owner) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', 'script', 1, \ + $3, 'bash', 'bash', '{\"n\": 1}', true)", + ) + .bind(id) + .bind(W_ID) + .bind(PATH) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, started_at, tag) \ + VALUES ($1, $2, now(), true, now(), 'bash')", + ) + .bind(id) + .bind(W_ID) + .execute(db) + .await?; + Ok(id) +} + +/// Waits for another connection to be stuck on the queue row of `id`, which is where the +/// completion's delete lands while the cancel is uncommitted. +async fn wait_until_blocked_on(db: &Pool, id: Uuid) -> anyhow::Result<()> { + for _ in 0..100 { + let blocked: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_locks l JOIN v2_job_queue q \ + ON q.ctid = ('(' || l.page || ',' || l.tuple || ')')::tid \ + WHERE NOT l.granted AND l.locktype = 'tuple' AND q.id = $1) \ + OR EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE wait_event_type = 'Lock' AND query LIKE '%v2_job_queue%')", + ) + .bind(id) + .fetch_one(db) + .await?; + if blocked { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + anyhow::bail!("the completion never reached the delete, so the race was not exercised") +} + +/// The same cancel, one step earlier: still being written when the completion reads the queue row, +/// which reads it without a lock. Only the delete that follows waits for the writer. +#[sqlx::test(fixtures("base"))] +async fn a_cancel_still_being_written_is_kept(db: Pool) -> anyhow::Result<()> { + let running = start_perpetual_run(&db).await?; + let job = get_mini_completed_job(&running, W_ID, &db).await?.unwrap(); + + let mut cancel = db.begin().await?; + sqlx::query( + "UPDATE v2_job_queue SET canceled_by = 'test-user', canceled_reason = 'stop' WHERE id = $1", + ) + .bind(running) + .execute(&mut *cancel) + .await?; + + let completing = tokio::spawn({ + let db = db.clone(); + async move { + add_completed_job( + &db, + &job, + true, + false, + Json(&json!("done")), + None, + 0, + None, + false, + None, + false, + ) + .await + } + }); + // Committing on a timer would let a slow completion read the cancel through the insert + // instead, leaving the half this test is for unexercised and green. Waiting for the delete to + // block on the row is what puts it there. + wait_until_blocked_on(&db, running).await?; + cancel.commit().await?; + completing.await??; + + let (status, canceled_by): (String, Option) = + sqlx::query_as("SELECT status::text, canceled_by FROM v2_job_completed WHERE id = $1") + .bind(running) + .fetch_one(&db) + .await?; + assert_eq!(status, "canceled", "the run is recorded as canceled"); + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + let queued: Vec = sqlx::query_scalar( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) \ + WHERE j.workspace_id = $1 AND j.runnable_path = $2", + ) + .bind(W_ID) + .bind(PATH) + .fetch_all(&db) + .await?; + assert!( + queued.is_empty(), + "a canceled perpetual run queues no next one: {queued:?}" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn a_cancel_landing_after_the_worker_read_its_row_is_kept( + db: Pool, +) -> anyhow::Result<()> { + let running = start_perpetual_run(&db).await?; + sqlx::query( + "UPDATE v2_job_queue SET canceled_by = 'test-user', canceled_reason = 'stop' WHERE id = $1", + ) + .bind(running) + .execute(&db) + .await?; + + let job = get_mini_completed_job(&running, W_ID, &db).await?.unwrap(); + add_completed_job( + &db, + &job, + true, + false, + Json(&json!("done")), + None, + 0, + // What the worker carries when the cancel landed after it last read the row. + None, + false, + None, + false, + ) + .await?; + + let (status, canceled_by): (String, Option) = + sqlx::query_as("SELECT status::text, canceled_by FROM v2_job_completed WHERE id = $1") + .bind(running) + .fetch_one(&db) + .await?; + assert_eq!(status, "canceled", "the run is recorded as canceled"); + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + let queued: Vec = sqlx::query_scalar( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) \ + WHERE j.workspace_id = $1 AND j.runnable_path = $2", + ) + .bind(W_ID) + .bind(PATH) + .fetch_all(&db) + .await?; + assert!( + queued.is_empty(), + "a canceled perpetual run queues no next one: {queued:?}" + ); + Ok(()) +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 9eae01c476..50eb9c8b1f 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1285,7 +1285,25 @@ async fn commit_completed_job( } let mut _skip_downstream_error_handlers = false; - tx = delete_job(tx, &job_id).warn_after_seconds(10).await?; + let (ntx, canceled_at_delete) = delete_job(tx, &job_id).warn_after_seconds(10).await?; + tx = ntx; + // `canceled_by` is only what the worker last read from the queue row. Deleting the row waits + // for a cancel still being written, so the row it removed is the final word on whether this + // job was canceled. + if canceled_by.is_none() { + if let Some(canceled) = canceled_at_delete { + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled'::job_status, canceled_by = $2, \ + canceled_reason = $3 WHERE id = $1", + job_id, + canceled.username, + canceled.reason, + ) + .execute(&mut *tx) + .warn_after_seconds(10) + .await?; + } + } // tracing::error!("3 {:?}", start.elapsed()); if completed_job.is_flow_step() { @@ -1611,6 +1629,22 @@ async fn restart_job_if_perpetual_inner( }; if restart { + // Not `canceled_by`: a worker reads the queue row on a widening interval, up to every 5s + // once a job has run for a minute, so a cancel landing after its last read reaches a + // completion carrying none. `commit_completed_job` records it from the queue row it + // deletes, so the completed row is what says whether this loop was stopped. + let canceled = sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed \ + WHERE id = $1 AND workspace_id = $2", + queued_job.id, + &queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if canceled { + return Ok(()); + } let tx = PushIsolationLevel::IsolatedRoot(db.clone()); // perpetual jobs can run one job per 10s max. If the job was faster than 10s, schedule the next one with the appropriate delay @@ -5153,34 +5187,44 @@ async fn extract_result_from_job_result( } } +/// Also reports the cancellation the deleted row carried, if any. Unlike a plain read of the queue +/// row, this waits for a cancel that is still being written, so it is the last word on one. pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, job_id: &Uuid, -) -> windmill_common::error::Result> { +) -> windmill_common::error::Result<(Transaction<'c, Postgres>, Option)> { #[cfg(feature = "prometheus")] if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_DELETE_COUNT.inc(); } otel_incr_queue_delete_count(); - let job_removed = - sqlx::query_scalar!("DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", job_id,) - .fetch_optional(&mut *tx) - .await; + let job_removed = sqlx::query!( + "DELETE FROM v2_job_queue WHERE id = $1 RETURNING canceled_by, canceled_reason", + job_id, + ) + .fetch_optional(&mut *tx) + .await; - if let Err(job_removed) = job_removed { - tracing::error!( - "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling" - ); - } else { - let job_removed = job_removed.unwrap().flatten().unwrap_or(0); - if job_removed != 1 { - tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"); + let canceled = match &job_removed { + Err(job_removed) => { + tracing::error!( + "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling" + ); + None } - } + Ok(None) => { + tracing::error!("Job {job_id} could not be deleted, no row was removed. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"); + None + } + Ok(Some(row)) => row.canceled_by.as_ref().map(|username| CanceledBy { + username: Some(username.clone()), + reason: row.canceled_reason.clone(), + }), + }; tracing::debug!("Job {job_id} deleted"); - Ok(tx) + Ok((tx, canceled)) } pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result { From 732214361f36eb07d65854118bd053c96ce681fb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 13:17:58 +0200 Subject: [PATCH 48/53] fix: stop the raw app editor stealing focus in ai sessions (#11287) * fix: preserve raw app editor focus during ai file updates * fix: sync ai file edits through the existing files effect * docs: explain raw app file synchronization dependency * fix: avoid echoing raw app files back into the editor * fix: synchronize raw app file operations through one effect * fix: stop session draft sync from re-applying the editor's own edits Co-Authored-By: Claude Opus 5 (1M context) * fix: open raw app files without taking focus and follow idle ai edits Co-Authored-By: Claude Opus 5 (1M context) * fix: keep selecting raw app documents that live outside the app files Co-Authored-By: Claude Opus 5 (1M context) * fix: tie a pending raw app editor focus request to the picked file Co-Authored-By: Claude Opus 5 (1M context) * chore: pin ui_builder to 60b6904 Co-Authored-By: Claude Opus 5 (1M context) * fix: always send explicit raw app file replacements to the editor Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Guilhem Lemouel Co-authored-by: Claude Opus 5 (1M context) --- frontend/scripts/ui_builder_artifact.json | 4 +- .../components/raw_apps/RawAppEditor.svelte | 106 ++++++++++++------ .../sessions/useUserDraftSync.svelte.ts | 9 +- 3 files changed, 79 insertions(+), 40 deletions(-) diff --git a/frontend/scripts/ui_builder_artifact.json b/frontend/scripts/ui_builder_artifact.json index 87ce066c87..68c1c558b1 100644 --- a/frontend/scripts/ui_builder_artifact.json +++ b/frontend/scripts/ui_builder_artifact.json @@ -1,5 +1,5 @@ { "baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev", - "version": "1522e43", - "sha256": "0c44db3a52e02e6530860d0e9e9031fcdad7a09a98897b6c97e6f522c1d4754c" + "version": "60b6904", + "sha256": "85b68e8418c97d96f3a419b3e345ef8a1987976943b8439a7df841f9217a9b34" } diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 9a5eb001ec..78d3ba8102 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -566,7 +566,10 @@ // The only writer of `activeTabId`, `selectedRunnable` and `selectedDocument`. // Everything switches through here, so the three can't disagree: one left // stale marks two sidebar rows selected, or leaves a tab over an empty pane. - function select(next: EditorSelection, opts?: { force?: boolean; notifyIframe?: boolean }): void { + function select( + next: EditorSelection, + opts?: { force?: boolean; notifyIframe?: boolean; focus?: boolean } + ): void { if (next.kind === 'preview') { // In split mode Preview is always shown on the right, so selecting it is // a no-op (would collapse the left pane). `force` lets closeTab fall back @@ -575,21 +578,23 @@ activeTabId = PREVIEW_TAB_ID selectedRunnable = undefined selectedDocument = undefined + iframeFocusPending = undefined return } if (next.kind === 'file') { activeTabId = ensureFileTab(next.path) selectedRunnable = undefined selectedDocument = next.path - if (opts?.notifyIframe !== false) openInIframe(next.path) + if (opts?.notifyIframe !== false) openInIframe(next.path, opts?.focus) return } activeTabId = ensureRunnableTab(next.key) selectedDocument = undefined selectedRunnable = next.key + iframeFocusPending = undefined } - function activateTab(id: string, opts?: { force?: boolean }) { + function activateTab(id: string, opts?: { force?: boolean; focus?: boolean }) { if (!tabs.some((t) => t.id === id)) return select(selectionOfTab(id), opts) } @@ -602,13 +607,20 @@ closeTab(runnableTabId(key)) } - // Ask the UI Builder iframe to open a document. `populateFiles` replays - // `iframeDocument` on iframe load, so record it even when the iframe isn't - // ready yet (the postMessage is then skipped). - function openInIframe(path: string) { + // Ask the UI Builder iframe to open a document, taking the keyboard only when + // the user picked it. For a file whose current content the iframe doesn't hold + // yet, `populateFiles` opens `iframeDocument` in the same message as that + // content: opening first errors on a new file and races the write on a changed + // one (the iframe applies the edit twice). It also opens it on iframe load. + // Documents outside `files` (wmill.ts, ui/, node_modules/) never go through it. + function openInIframe(path: string, focus = false) { iframeDocument = path - if (iframeLoaded) { - iframe?.contentWindow?.postMessage({ type: 'selectFile', path }, '*') + const content = files?.[path] + if (iframeLoaded && (content === undefined || iframeFiles?.[path] === content)) { + iframeFocusPending = undefined + iframe?.contentWindow?.postMessage({ type: 'selectFile', path, focus }, '*') + } else { + iframeFocusPending = focus ? path : undefined } } @@ -730,6 +742,7 @@ summary = update.summary } if (update.files !== undefined) { + iframeFiles = undefined files = update.files } if (update.runnables !== undefined) { @@ -810,6 +823,17 @@ } let iframeLoaded = $state(false) // @hmr:keep + // The files the iframe holds: last posted to it, or last reported by it. Explicit + // replacements (history, YAML) clear it so they are always sent: the iframe can + // hold edits it hasn't reported yet, which a skip would leave in place. + let iframeFiles: Record | undefined + // The document the user picked that `openInIframe` left for `populateFiles` to + // open. It takes the keyboard only if it is still the document being opened. + let iframeFocusPending: string | undefined + // Last keystroke or click in the editor, or edit reported by the iframe. AI + // edits only move the editor to their file once the user has paused this long. + let lastUserInputAt = 0 + const USER_IDLE_MS = 3000 // Briefly drops the `setActiveDocument` echo VS Code fires while we're // pushing the initial file set — the iframe auto-opens a default editor // during boot which we don't want to treat as a user-driven activation. @@ -857,6 +881,10 @@ function populateFiles() { if (files) { + // `files` is reassigned with unchanged content (the session draft sync, history + // restores, the iframe's own reports). Re-sending it makes the iframe reopen its + // document and rewrite the files under the user's cursor. + if (deepEqual(files, iframeFiles)) return suppressSetActiveDocument = true if (suppressTimer !== undefined) clearTimeout(suppressTimer) suppressTimer = setTimeout(() => { @@ -875,10 +903,14 @@ } } function setFilesInIframe(newFiles: Record) { + const target = iframe?.contentWindow + if (!target) return + iframeFiles = { ...newFiles } + iframeFocusPending = undefined const files = Object.fromEntries( Object.entries(newFiles).filter(([path, _]) => !path.endsWith('/')) ) - iframe?.contentWindow?.postMessage( + target.postMessage( { type: 'setFiles', files: files @@ -889,14 +921,20 @@ function setFilesAndSelectInIframe(newFiles: Record, pathToSelect: string) { iframeDocument = pathToSelect + const target = iframe?.contentWindow + if (!target) return + iframeFiles = { ...newFiles } + const focus = iframeFocusPending === pathToSelect + iframeFocusPending = undefined const files = Object.fromEntries( Object.entries(newFiles).filter(([path, _]) => !path.endsWith('/')) ) - iframe?.contentWindow?.postMessage( + target.postMessage( { type: 'setFilesAndSelect', files: files, - pathToSelect: pathToSelect + pathToSelect: pathToSelect, + focus }, '*' ) @@ -1012,14 +1050,15 @@ return frontendFiles }, setFrontendFile: (path, content): LintResult => { - console.log('setting frontend file', path, content) + console.log('setting frontend file', path, `${content.length} chars`) if (!files) { files = {} } files[path] = content - // Combined setFilesAndSelect avoids a race, so let it do the telling. - select({ kind: 'file', path }, { notifyIframe: false }) - setFilesAndSelectInIframe(files, path) + // Follow the AI to the file it edits, but never pull a user who is editing + // off their file. The files effect sends the content. + if (Date.now() - lastUserInputAt < USER_IDLE_MS) ensureFileTab(path) + else select({ kind: 'file', path }) return lint() }, deleteFrontendFile: (path) => { @@ -1027,7 +1066,6 @@ files = {} } delete files[path] - setFilesInIframe(files) }, listBackendRunnables: () => { return Object.entries(runnables).map(([key, runnable]) => ({ @@ -1399,9 +1437,11 @@ if (e.data.type === 'setFiles') { // Normalize Windows-style path separators to Linux-style const normalizedFiles = normalizeFilePaths(e.data.files) + iframeFiles = { ...normalizedFiles } // Only mark pending changes if files actually changed (ignore echo from setFilesInIframe) if (!deepEqual(files, normalizedFiles)) { files = normalizedFiles + lastUserInputAt = Date.now() historyManager.markPendingChanges() } } else if (e.data.type === 'getBundle') { @@ -2015,6 +2055,7 @@ } $effect(() => { iframe?.addEventListener('load', () => { + iframeFiles = undefined iframeLoaded = true }) }) @@ -2080,6 +2121,7 @@ } }) $effect(() => { + // populateFiles must track file entries so in-place AI edits trigger synchronization. iframe && iframeLoaded && files && populateFiles() }) $effect(() => { @@ -2108,7 +2150,7 @@ // future `FileExplorer` caller feeds folder paths back in. function handleSelectPath(path: string) { if (!path || path.endsWith('/')) return - select({ kind: 'file', path }) + select({ kind: 'file', path }, { focus: true }) } // Track previous values for change detection @@ -2202,19 +2244,12 @@ data: RawAppData }) { try { + iframeFiles = undefined files = structuredClone($state.snapshot(entry.files)) runnables = structuredClone($state.snapshot(entry.runnables)) summary = entry.summary replaceData(structuredClone($state.snapshot(entry.data))) - // If the open document survives into the new files, use the combined message - if (iframeDocument && isOpenableDocument(iframeDocument)) { - // Use combined setFilesAndSelect message to avoid race condition - setFilesAndSelectInIframe(entry.files, iframeDocument) - } else { - // Otherwise just set files normally - setFilesInIframe(entry.files) - } populateRunnables() } catch (error) { console.error('Failed to apply entry:', error) @@ -2351,7 +2386,12 @@ gateJobIds={false} extraSourceWindow={() => externalPreviewWindow} /> -
              +
              (lastUserInputAt = Date.now())} + onpointerdowncapture={() => (lastUserInputAt = Date.now())} + class="max-h-full overflow-hidden h-full min-h-0 flex flex-col" +> files, - (newFiles) => { - files = newFiles - setFilesInIframe(newFiles ?? {}) - } - } + bind:files onSelectPath={handleSelectPath} onSelectRunnable={(key) => select({ kind: 'runnable', key })} onDeleteRunnable={deleteRunnable} @@ -2495,7 +2529,7 @@ activateTab(id)} + onSelect={(id) => activateTab(id, { focus: true })} onClose={(id) => closeTab(id)} onReorder={(next) => reorderTabs(next)} > @@ -2580,7 +2614,7 @@ activateTab(id)} + onSelect={(id) => activateTab(id, { focus: true })} onClose={(id) => closeTab(id)} onReorder={(next) => reorderTabs(next)} > diff --git a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts index 7e89d3b945..64f7f7e6c2 100644 --- a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts +++ b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts @@ -94,7 +94,10 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void // re-apply only advances the sig, never reverts an edit or fires a save. if (sig === lastInboundSig) return lastInboundSig = sig - codec.applyDraftToStore(incoming) + // Copies both ways: a store sharing proxies with the draft makes every local + // edit re-fire this effect as if a draft came in, reassigning the store on + // each keystroke. + codec.applyDraftToStore($state.snapshot(incoming) as Draft) }) }) @@ -125,7 +128,9 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void untrack(() => { const current = UserDraft.get(codec.itemKind, path, { workspace }) if (current && codec.sig(current) === sig) return - const toSave = codec.storeToDraft(current) ?? draft + const toSave = $state.snapshot(codec.storeToDraft(current) ?? draft) as Draft + // Our own write comes back through the handle; the inbound side must skip it. + lastInboundSig = codec.sig(toSave) UserDraft.save(codec.itemKind, path, toSave, { workspace }) }) } From 841f7e1436ca1217ffde16f5d70c9b49ef60be13 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 14:13:11 +0200 Subject: [PATCH 49/53] fix(cli): sync workspace dependency files on Windows and fix CLI tests (#11288) * fix(cli): sync workspace dependency files on Windows Co-Authored-By: Claude Opus 5 (1M context) * test(cli): pin workspace dependency path parsing on both separators Co-Authored-By: Claude Opus 5 (1M context) * fix(cli): classify workspace dependency files on either separator Co-Authored-By: Claude Opus 5 (1M context) * test(cli): check out the fork deploy clone without autocrlf Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/commands/sync/sync.ts | 4 ++-- cli/src/types.ts | 2 +- cli/src/utils/script_common.ts | 4 +++- cli/src/utils/utils.ts | 3 ++- cli/test/gitsync_deploy_branch_config.test.ts | 5 +++- cli/test/sync_pull_push.test.ts | 12 ++++++---- cli/test/workspace_deps_path_unit.test.ts | 23 +++++++++++++++++++ 7 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 cli/test/workspace_deps_path_unit.test.ts diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 32cff4a3cb..6185decc5a 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1153,7 +1153,7 @@ export function ZipFSElement( ? "script" : p.endsWith(".resource.json") ? "resource" - : p.startsWith("dependencies/") + : isWorkspaceDependencies(p) ? "dependencies" : "other"; @@ -2752,7 +2752,7 @@ export async function compareDynFSElement( if (skipMetadata) { continue; } - if (k.startsWith("dependencies/")) { + if (isWorkspaceDependencies(k)) { if (!workspaceDependenciesPathToLanguageAndFilename(k)) { log.warn(`Skipping unrecognized workspace dependencies file: ${k}`); continue; diff --git a/cli/src/types.ts b/cli/src/types.ts index 5713d06390..866b2be21e 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -391,7 +391,7 @@ export function getTypeStrFromPath( if (isSharedLockPath(p)) { return "shared_lock"; } - if (p.startsWith("dependencies" + SEP)) { + if (isWorkspaceDependencies(p)) { return "workspace_dependencies"; } if (isFileResource(p) || isFilesetResource(p)) { diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index d804318491..611268c27c 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -45,7 +45,9 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [ ] as const; export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined { - const relativePath = path.replace("dependencies/", ""); + // Sync paths carry the platform separator, while a set's name is always + // forward-slashed. + const relativePath = path.replaceAll("\\", "/").replace("dependencies/", ""); for (const { filename, language } of workspaceDependenciesLanguages) { if (relativePath.endsWith(filename)) { return { diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 0801c03911..526dffbde4 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -223,8 +223,9 @@ export function isRawAppFile(path: string): boolean { return isRawAppPath(path); } +/** Sync paths carry the platform separator: `dependencies\x` on Windows. */ export function isWorkspaceDependencies(path: string): boolean { - return path.startsWith("dependencies/"); + return path.startsWith("dependencies/") || path.startsWith("dependencies\\"); } export function printSync(input: string | Uint8Array) { diff --git a/cli/test/gitsync_deploy_branch_config.test.ts b/cli/test/gitsync_deploy_branch_config.test.ts index 39c435d872..eeab7a0120 100644 --- a/cli/test/gitsync_deploy_branch_config.test.ts +++ b/cli/test/gitsync_deploy_branch_config.test.ts @@ -63,7 +63,10 @@ test("git-sync fork deploy follows the fork branch's wmill.yaml", async () => { git(seed, "commit", "-m", "dedupe"); git(seed, "push", "origin", forkBranch); - git(work, "clone", `file://${bare}`, "."); + // Git for Windows defaults to autocrlf=true, which checks the pulled LF + // files out as CRLF; the deploy's pull would rewrite every one of them + // and the clean-tree assertion below would read that as a change. + git(work, "clone", "-c", "core.autocrlf=false", `file://${bare}`, "."); const deployed = await backend.runCLICommand( [ "sync", diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 24c4a282a3..fa52510e6c 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2185,18 +2185,20 @@ describe("settings sync", () => { const files = await listFilesRecursive(tempDir); expect(files).toContain("settings.yaml"); - // Read and modify a safe setting (webhook URL) + // Read and modify a safe setting (webhook URL). The server resolves the + // webhook host when it is saved and refuses one that does not resolve to a + // public address, so this must be a real public hostname. + const webhook = "https://example.com/hook"; const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8"); let modifiedSettings: string; if (settingsContent.includes("webhook:")) { modifiedSettings = settingsContent.replace( /webhook:.*/, - 'webhook: "https://test-webhook.example.com/hook"' + `webhook: "${webhook}"` ); } else { - modifiedSettings = - settingsContent + '\nwebhook: "https://test-webhook.example.com/hook"\n'; + modifiedSettings = settingsContent + `\nwebhook: "${webhook}"\n`; } await writeFile(`${tempDir}/settings.yaml`, modifiedSettings, "utf-8"); @@ -2213,7 +2215,7 @@ describe("settings sync", () => { ); expect(apiResp.status).toEqual(200); const settingsData = await apiResp.json(); - expect(settingsData.webhook).toEqual("https://test-webhook.example.com/hook"); + expect(settingsData.webhook).toEqual(webhook); }); }); }); diff --git a/cli/test/workspace_deps_path_unit.test.ts b/cli/test/workspace_deps_path_unit.test.ts new file mode 100644 index 0000000000..45d725c6e5 --- /dev/null +++ b/cli/test/workspace_deps_path_unit.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from "bun:test"; +import { isWorkspaceDependencies } from "../src/utils/utils.ts"; +import { workspaceDependenciesPathToLanguageAndFilename } from "../src/utils/script_common.ts"; + +// Sync paths carry the platform separator. A Windows path parsed to a different +// set name pushes the file to a set the workspace does not have. +test("workspace dependency paths parse the same with either separator", () => { + for (const sep of ["/", "\\"]) { + expect(isWorkspaceDependencies(`dependencies${sep}requirements.in`)).toBe(true); + expect( + workspaceDependenciesPathToLanguageAndFilename(`dependencies${sep}requirements.in`), + ).toEqual({ name: undefined, language: "python3" }); + expect( + workspaceDependenciesPathToLanguageAndFilename(`dependencies${sep}team.package.json`), + ).toEqual({ name: "team", language: "bun" }); + expect( + workspaceDependenciesPathToLanguageAndFilename( + `dependencies${sep}team${sep}python.requirements.in`, + ), + ).toEqual({ name: "team/python", language: "python3" }); + } + expect(isWorkspaceDependencies("f\\dependencies\\requirements.in")).toBe(false); +}); From 25c8bfaab412d440e51be92af61066af025d334b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 14:13:32 +0200 Subject: [PATCH 50/53] chore(main): release 1.817.0 (#11268) * chore(main): release 1.817.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 24 +++ backend/Cargo.lock | 166 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- chat-sdk/package-lock.json | 4 +- chat-sdk/package.json | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 22 files changed, 154 insertions(+), 130 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 438887d2eb..6dc5574962 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.816.0" + ".": "1.817.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e039fb45..595426d619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [1.817.0](https://github.com/windmill-labs/windmill/compare/v1.816.0...v1.817.0) (2026-09-22) + + +### Features + +* add workspace mentions to AI composer ([#11264](https://github.com/windmill-labs/windmill/issues/11264)) ([e7ba595](https://github.com/windmill-labs/windmill/commit/e7ba5954b69514d72c140f9b1e697fb2236df23e)) +* allow workspace-scoped dynamic custom tags ([#11283](https://github.com/windmill-labs/windmill/issues/11283)) ([efa7a0a](https://github.com/windmill-labs/windmill/commit/efa7a0a70af43f8827542059dd450326a9b88d62)) +* check dynamic worker tags on the tag they resolve to ([#11271](https://github.com/windmill-labs/windmill/issues/11271)) ([69fafb9](https://github.com/windmill-labs/windmill/commit/69fafb9262f90184e6c2dc9a84f1c3feeb9d9638)) +* deploy from the UI to a workspace on another instance ([#11245](https://github.com/windmill-labs/windmill/issues/11245)) ([4d12ea4](https://github.com/windmill-labs/windmill/commit/4d12ea461454f89367bd91daa7e8c39a89035507)) + + +### Bug Fixes + +* add missing credential fields to settings redaction lists ([#11272](https://github.com/windmill-labs/windmill/issues/11272)) ([2a40f25](https://github.com/windmill-labs/windmill/commit/2a40f25bda81d41b9f578338c85656744f59ebfc)) +* apply SSRF validation to workspace webhook URLs ([#11285](https://github.com/windmill-labs/windmill/issues/11285)) ([586699c](https://github.com/windmill-labs/windmill/commit/586699c4938b5203920a43e69a706310078b93f5)) +* apply token path scopes to the native trigger list ([#11281](https://github.com/windmill-labs/windmill/issues/11281)) ([faf7b22](https://github.com/windmill-labs/windmill/commit/faf7b22be0827738c90c1f3ed6cd3e214d6c63d3)) +* check workflow-as-code task tags against CUSTOM_TAGS ([#11273](https://github.com/windmill-labs/windmill/issues/11273)) ([2699d50](https://github.com/windmill-labs/windmill/commit/2699d5065ea3dfe985636733fb6022ca23187291)) +* clear the impersonation token from session storage on logout ([#11274](https://github.com/windmill-labs/windmill/issues/11274)) ([9936673](https://github.com/windmill-labs/windmill/commit/9936673ee62fb6dd929a654a6138aeca687c0fb7)) +* enforce token path scopes on GET /raw_apps/list ([#11284](https://github.com/windmill-labs/windmill/issues/11284)) ([9ad2c91](https://github.com/windmill-labs/windmill/commit/9ad2c91ddb661ab60ada9ec4039ef8d025602969)) +* keep a cancel that lands after a worker last read its queue row ([#11246](https://github.com/windmill-labs/windmill/issues/11246)) ([02006f3](https://github.com/windmill-labs/windmill/commit/02006f364a4217e967435d9e71d8617c03b65413)) +* size the AI chat output budget by model, not provider ([#11286](https://github.com/windmill-labs/windmill/issues/11286)) ([7d0f2ce](https://github.com/windmill-labs/windmill/commit/7d0f2ce3bf920a68f1ea22d59481d239947f1b51)) +* stop cutting DeepSeek chat turns off mid-thought ([#11282](https://github.com/windmill-labs/windmill/issues/11282)) ([5821700](https://github.com/windmill-labs/windmill/commit/58217006ece56bbaf0447af67cac487e72b928c7)) +* stop the raw app editor stealing focus in ai sessions ([#11287](https://github.com/windmill-labs/windmill/issues/11287)) ([7322143](https://github.com/windmill-labs/windmill/commit/732214361f36eb07d65854118bd053c96ce681fb)) + ## [1.816.0](https://github.com/windmill-labs/windmill/compare/v1.815.0...v1.816.0) (2026-09-21) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 82c625df67..e0bfce842b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -928,9 +928,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.3" +version = "1.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0baa720ebadea158c5bda642ac444a2af0cdf7bb66b46d1e4533de5d1f449d0" +checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -938,8 +938,8 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -14792,7 +14792,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -14880,7 +14880,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.816.0" +version = "1.817.0" dependencies = [ "async-stream", "async-trait", @@ -14914,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14927,7 +14927,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "argon2", @@ -15067,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15090,7 +15090,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15107,7 +15107,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15133,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.816.0" +version = "1.817.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15143,7 +15143,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15160,7 +15160,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15182,7 +15182,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15205,7 +15205,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15221,7 +15221,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15265,7 +15265,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15279,7 +15279,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -15314,7 +15314,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15339,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15367,7 +15367,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15389,7 +15389,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15409,7 +15409,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15447,7 +15447,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15476,7 +15476,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.816.0" +version = "1.817.0" dependencies = [ "lazy_static", "serde", @@ -15488,7 +15488,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.816.0" +version = "1.817.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15512,7 +15512,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15526,7 +15526,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15564,7 +15564,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.816.0" +version = "1.817.0" dependencies = [ "chrono", "lazy_static", @@ -15578,7 +15578,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15597,7 +15597,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.816.0" +version = "1.817.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15704,7 +15704,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.816.0" +version = "1.817.0" dependencies = [ "chrono", "futures", @@ -15724,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.816.0" +version = "1.817.0" dependencies = [ "regex", "serde", @@ -15741,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15768,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "futures", @@ -15785,7 +15785,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.816.0" +version = "1.817.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15801,7 +15801,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -15822,7 +15822,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -15853,7 +15853,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "arc-swap", @@ -15878,7 +15878,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-stream", @@ -15913,7 +15913,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "futures", @@ -15931,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.816.0" +version = "1.817.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15940,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -15952,7 +15952,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -15964,7 +15964,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "gosyn", @@ -15976,7 +15976,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -15988,7 +15988,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -16000,7 +16000,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "nu-parser", @@ -16011,7 +16011,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16022,7 +16022,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16034,7 +16034,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16045,7 +16045,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16067,7 +16067,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -16079,7 +16079,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16093,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16110,7 +16110,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16123,7 +16123,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde", @@ -16135,7 +16135,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16153,7 +16153,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16169,7 +16169,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16185,7 +16185,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16199,7 +16199,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16238,7 +16238,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "const_format", @@ -16278,7 +16278,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.816.0" +version = "1.817.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16289,7 +16289,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16324,7 +16324,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16348,7 +16348,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16381,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16408,7 +16408,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16441,7 +16441,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16461,7 +16461,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16495,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16531,7 +16531,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16554,7 +16554,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16578,7 +16578,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -16602,7 +16602,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16637,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16665,7 +16665,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16690,7 +16690,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16709,7 +16709,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-once-cell", @@ -16827,7 +16827,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.816.0" +version = "1.817.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a911d57b81..74da8a4244 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.816.0" +version = "1.817.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.816.0" +version = "1.817.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index dcd846c0d8..ccba27bcc5 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.816.0" +version = "1.817.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.816.0" +version = "1.817.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.816.0" +version = "1.817.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1eacbd24d3..0244b7b106 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.816.0" +version = "1.817.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 46c498de4c..0dd9f8620e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.816.0 + version: 1.817.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 7f0649a2ab..1e6cfdd474 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.816.0"; +export const VERSION = "v1.817.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index e657c139d7..ff470d4ffe 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.816.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.816.0", + "version": "1.817.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index a157a5175a..5655adc64e 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.816.0", + "version": "1.817.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 45eb4e941f..9728b6edd7 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.816.0"; +export const VERSION = "1.817.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 072729566c..611cd0c91f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.816.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.816.0", + "version": "1.817.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index f535994371..46db11793f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.816.0", + "version": "1.817.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 418c7a8f8e..79d2672f4a 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.816.0" +wmill = ">=1.817.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index a5c0d33b62..48f573bab1 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.816.0 + version: 1.817.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 9117e1bf7d..ad6c0e8bc1 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.816.0' + ModuleVersion = '1.817.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index b493b81c41..baab12d832 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.816.0" +version = "1.817.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index c72327a9a7..26441041a1 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.816.0", + "version": "1.817.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index afa9e10871..4badfa307b 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.816.0", + "version": "1.817.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 198c13b19c..2edd9ebec3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.816.0 +1.817.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index cb1c272906..e07061650d 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.816.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.816.0", + "version": "1.817.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index f0dffa300a..01bb378dcd 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.816.0", + "version": "1.817.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", From 7dec3d97717406d078499ea6509cdb00c89c3480 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:15:05 +0200 Subject: [PATCH 51/53] feat(db-manager): add a schema diagram view (#11030) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(datatables): put a data table's connection under Postgres roles A data table backed by the instance database resolved to exactly one Postgres connection, `custom_instance_user`, for everyone who could reach it at all. There was no way to say this job reads, that one writes, this one never sees the salaries table. A data table role is now a real Postgres login on the cluster, defined once for the instance by a superadmin and named exactly as they named it. A script that declares `-- role analytics` connects as `analytics`, and Postgres decides what it may touch — grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`. A data table with no `permissions` block behaves exactly as before. Everything that opens a connection on someone's behalf goes through one chokepoint, `get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when there is none. The role logs in as itself — never `SET ROLE`, which a script could `RESET ROLE` its way out of. A fork's data table entry becomes a pointer at the workspace that governs it rather than a copy of it. The settings clone used to hand a fork a byte-identical entry naming the parent's database, which a fork admin could edit to grant themselves `admin` there; a pointer has nothing local to edit, and its tenants are evaluated as a member of the governing workspace, by email. `permissions` is stripped from the workspace export and ignored on import: tenants name principals of one workspace, and a settings push is not where an access decision should be made. Operations that see the whole database whatever the roles grant stay with the governing workspace's admins: editing the roles, a migration that declares none, and opening a replication stream for a Postgres trigger or capture. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): gate the paths that reach a whole database as admin Auditing what still resolved through the unchecked resolver turned up three that act for a caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema export, the full-schema read, database creation, import and the forked-database drop), the connection test, and the schema snapshot a fork clone takes of its parent. On a data table under roles each let any workspace member — or a fork admin who is nobody in the governing workspace — read or copy the whole database whatever its roles grant. All three now require admin reach on the governing workspace. A dump taken under a restricted role would be a silently truncated copy rather than an error, so refusing is the only right answer for the copy paths. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin putting a *resource-backed* data table under roles, at which point the executor dialled the host that resource names — one the admin chose — with the role's real cluster password, and `CONNECT` is granted to every registered instance database. Both ends now refuse: the permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials on a non-instance entry rather than trusting the record it read. Two more places reached the governing database without answering to it. The initial-migration generator returned a `pg_dump` of the whole schema to any member. And the migration rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after which the parent re-runs every migration from zero. The remote half is now skipped when the entry resolves into another workspace, which is also just correct: a fork renaming what it calls a data table changes nothing about the data table. Also: revoking a tenant now bounces the replication streams of every workspace holding an entry that resolves here, not only the governing one, so a fork's trigger stops rather than living on inside its open connection; the instance role catalog and the governing workspace's tenant lists are no longer returned to someone who cannot edit them; and the tenant rename dedup collapses non-adjacent duplicates, per role rather than once any role changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): fail loudly where a role or a pointer can be left half-recorded Three ways the feature could end up in a state nobody could see or undo. Creating a role writes the cluster first and the catalog second, but the catalog write was an `UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a live login with a password nobody recorded: invisible to the catalog, un-recreatable because the name is taken, and un-deletable because there is no entry to delete. It now errors, so the operation is retryable once the row is restored. Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy would hand each fork the database outright — so the delete now names the data tables it stranded, and resolving one says which workspace is missing rather than reporting a data table this workspace never had. `InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now hand-written so `{:?}` on the catalog cannot put a live credential in a log line. Adds the two branches the reviews found unpinned: a caller who is not a member of the governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that predates this and sends no job id. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): unbreak two operator messages and two comments that described other code The two strings this branch added for states an operator hits once — the catalog write that matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence. `list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop it, and the comment now says why that is the right place to stay quiet. The non-superadmin check in `edit_datatable_config` was introduced as also covering references, which it does not and need not: `reference` is overwritten from the stored entry for every caller before the check runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): serialize role catalog mutations, and state each helper's authorization contract The catalog is one JSON document, so create, rename, enable and delete are all read-modify-write. Two concurrent creates read the same snapshot, both succeed in the cluster, and the second write drops the first — leaving a live Postgres login with a password nobody recorded, which is the exact state the delete path exists to prevent. Every mutation now runs in one transaction holding an advisory lock across the read, the cluster DDL and the write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL helpers take that transaction rather than the pool, which is what makes the lock cover them. Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely multi-statement SQL, and its future is not `Send`, which an axum handler holding the transaction requires. Each of these is one statement anyway. The new cross-crate surface now says what callers must do. `read_role_catalog` returns plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and `converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's raw config. All of them are superadmin-gated by their current handlers, but nothing said so at the definition, which is where the next caller looks. Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip that did not land; the rename affordance is the design-system `Button`, not a raw one; and `resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing resolves as the data table's default until the database manager grows a picker. Why role passwords stay a plain `String` while the instance user's password beside them is a `StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an operator supplies it and may want it from their own backend, while these are minted here and never entered by anyone, so there is nothing for a ref to point at. Encrypting generated secrets at rest is a separate change that would take the replication password with it. Now said at the field. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): give the role catalog its own row, out of reach of the config machinery Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways. The catalog serializes a generated Postgres password per role, and that row is the operator-facing instance config, so the passwords reached `get_instance_config` and its YAML editor — a live cluster credential in a response body, a UI field and any log of either. Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of that key writes the row back without it and the catalog is gone, while the cluster keeps every login it described. `custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret, written only by the server, never operator-authored, hidden so the config machinery cannot read, rewrite or drop it. The catalog is the same thing, so it now has the same shape: `datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist. No redaction to keep in step with three code paths, and no way for a neighbouring write to take it out. Two races on the same shared documents. `edit_datatable_config` read the stored data tables outside its transaction and then wrote the whole `datatable` document, so a permissions save committing in between was silently rolled back; it now reads under `FOR UPDATE`. And `set_datatable_permissions` validated role ids against the catalog before opening its transaction, so a deletion in between let it write a deleted role back — including as the default, which every later job then fails on; it now holds the catalog lock and the settings row across validation and write. Completes the authorization contracts the previous commit claimed but did not finish: `read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole job is to answer for a workspace the caller may not belong to, and `converge_connect_grants_with`, which had not inherited its wrapper's. Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but `extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so code written from that page passed a keyword-only argument positionally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all three handlers left it green — it pinned Postgres, not the code it was written for. It now drives `create_datatable_role` twice concurrently and asserts the catalog kept both names. Checked the way the last one should have been: removing the lock from the handler makes it fail with "wmtest_a_… is a live cluster login the catalog forgot". The contracts added last commit were stricter than this PR's own callers, which is worse than none — the next reader sees a rule already broken and learns to ignore it. `read_role_catalog` said superadmin-only while two of its four callers are open to any workspace member, and `converge_connect_grants` said superadmin while `set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the rule that actually holds is about the credential never reaching a response, log, audit record or export, not about who may call. They now say that. `read_datatable_entry` gets the same treatment rather than the one the earlier message claimed for it: it is the primitive every resolution goes through, so it is deliberately open, and what must not escape is `permissions` — it names the governing workspace's users, groups and folders. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): close the last ways a role or a pointer can be left pointing at nothing The raw settings readers hand back whatever is in the row, so moving the catalog into its own `global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles` and the settings listing returning every live password. Both now filter that one key. The neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it predates this and widening the fix to it is a decision about an operator workflow, not a consequence of this change. Three ways a save could leave something resolving to nothing: A permissioned data table could be moved to a PostgreSQL resource. The block was carried across as a server-owned field, the runtime refuses roles on a resource-backed table, so the save succeeded and every job afterwards failed. Refused instead — turning roles off first is one step, and it keeps discarding an access decision something somebody chose. Renaming a governing data table left every fork pointing at the old name: the data table disappears from their pickers and their jobs stop, with nothing in the renaming workspace to suggest why. The rename now follows into the pointers in the same transaction. Deleting one cannot be followed the same way, so it is reported instead — the response names what it stranded, the way deleting a workspace does, and the fork's own error already says which workspace is gone. Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions handler reaches it as a workspace admin (the same class fixed last commit, one instance missed); the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog` was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its roles only on success — a failing run is exactly the one that creates them without recording them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * refactor(datatables): put the role catalog in its own table, not in global_settings Five findings across three rounds were all the same choice. A set of live Postgres credentials was living in `global_settings`, which has generic read, list, write, config-export and CLI round-trip paths that know nothing about what they carry: the passwords reached the instance config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog, `GET /settings/global/{key}` and the settings listing returned them raw, and this round the redaction that fixed the last two turned `wmill instance push` into something that wipes every password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles` could also empty it outside the lock. The approved plan offered a table or `global_settings`, so this is the other option it already allowed rather than a new design. `datatable_role` is a table: no generic settings path can read it, list it, export it, write it or round-trip it, so none of the five needs a guard. The redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it. One row per role also removes the read-modify-write the concurrency work was about: two concurrent creates are two inserts, and the unique index on `name` is what settles a collision. The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to another transaction until commit, so without it both creates pass their `pg_roles` check. Also from this round: rename mappings are checked against the configuration they claim to describe, since fork pointers are rewritten from them — a caller could otherwise submit `main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning naming forks a delete stranded reached the response but not the screen: both the data table settings save and the workspace delete now show it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): validate a rename against the save it describes, and re-check under the locks Three from the round, all about deciding on state that could already have moved. A permission save resolved the data table and checked it was instance-backed before taking any lock, then wrote under one. A config save committing in between could move the table onto a PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in which case the write targeted a key that no longer existed and reported success having changed nothing. It now re-resolves and re-checks on the locked state. Rename validation checked that the source existed before and the target existed after, which still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows onto a different data table, silently, because it keeps resolving. The rule is now the actual old-to-new key transition — a source may only survive if another rename took its name, and a target may only pre-exist if another rename freed it. That also stops two sources sharing one target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so a swap cannot be done one save at a time, and refusing it was a regression against main. The pointer cascade now runs in two passes through a temporary name, the way the migration cascade one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once from what it named before the save. The tenant mutators say what they are for: they write an access decision for any workspace named, with an arbitrary mutation, and exist for the transaction that frees or renames a principal. Editing a decision on purpose belongs in the permissions endpoint. Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole document, so a workspace whose pointers name something else is not rewritten to a byte-identical value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from the JSON document, one of which would emit `pwd`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call `/workspaces/leave` — a different handler in a different crate with the same name — which deleted the membership and left `u/` in the tenant lists. Leaving and rejoining therefore restored the access the leave was supposed to end, and a later account taking the username would have inherited it. The regression test drives the route the client actually calls; without the fix it fails with "leaving kept the tenant". The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's admin connection, created `_wm_migrations` and read it before reaching the per-migration role check — so with nothing pending, nothing was checked at all. Rollback returned before its check when nothing was applied, and the status endpoint had none. All three now ask, before any connection is opened, whether the caller can reach the data table as any role at all; which role a given migration runs as is still decided per migration, and by the executor after that. Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now runs in the same transaction, so the drop, the row and every tenant list commit together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse to copy a data table that is under roles pg_dump carries no roles and the import runs with --no-privileges, so a copied data table arrives owned by the admin connection with no GRANT for any role. The settings clone brings `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were given, and are denied by Postgres on everything: an entry that reads as configured and answers nothing. Refuse the copy — in the import endpoint before any data moves, and in the fork path the CLI takes. Replaying the source's owners and ACLs into the clone is what lifts this, and is a change of its own. Dropping `permissions` from the copy instead would be the unsafe half, since the copy holds the parent's rows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse the clone's database too, not only its data A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only the second refused a data table under roles, so a fork asking to clone one created and registered an empty `wm_fork_…` instance database and then failed — and nothing collects it, since `drop_forked_datatable_databases` only drops entries carrying `forked_from` and no entry names this one. Refuse in both, so the clone stops before a database exists. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * nit worker error msg * fix pg_dump stuck on version 17 on nix * fix(datatables): refuse a malformed role annotation instead of ignoring it `-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed the annotation parser's exact-match rule, so the query fell through to the data table's default role and ran, silently, under a login the author did not choose. Naming a role exists precisely to not do that. A leading comment whose first word is `role` is now an annotation attempt: the keyword matches case-insensitively, one trailing `;` is tolerated, and anything else is an error naming the line. Only callers that already know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its comments. Also bumps the dev shell's postgres client to 18 — it trailed the server the dev database runs, which takes out every data table export, clone and fork-with-data. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse a malformed role query string instead of ignoring it `?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference parser's exact-match rule, so the connection resolved to the data table's default role and ran under a login the caller never asked for — the URI half of the same trap as a malformed `-- role` annotation. The key now matches case-insensitively, and anything else in the query string is an error naming it; `role` is the only parameter a reference takes. Callers that only need the entry keep a lenient `datatable_ref_name`, since they never act on the role. The DuckDB `ATTACH` parser propagates it rather than attaching under the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): carry the role annotation into the row_to_json retry The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment block never reached the second attempt — and with it the `-- role ` line that decides which login the query runs as. The retry connected as the data table's default role instead, so a query the first attempt was denied could succeed on the second, reported as "recovered with the row_to_json fix". Carry the leading comment block over. The retry itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore(datatables): don't mount the roles UI until the ACL editor lands Enforcement ships first. The permissions drawer is what turns roles on, and the catalog section is what creates them — both are only useful once there is a way to grant a role the privileges it needs, which arrives with the ACL editor. Left mounted they would offer a feature whose other half does not exist. The two components are complete and reviewed; only their call sites here are commented out, with a note pointing the follow-up PRs at them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): honour `-- role: x`, and fix the DuckDB attach test Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse to roll back the catalog while roles exist The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * refactor(datatables): settle clonability in one place, before anything is created A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): let a retried clone reclaim its own leftover database A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) * Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bounce the streams reading a data table when it is deleted Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) * refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. * fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): data table roles in the DB manager and raw apps Co-Authored-By: Claude Opus 5 (1M context) * fix: never add a role to the reference of a data table whose name contains '?' Co-Authored-By: Claude Opus 5 (1M context) * fix: read the roles of a data table whose name contains '?' The generated client leaves a '?' in a path param unencoded, so the lookup 404'd and the raw-app picker blocked Start on such a data table. Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): refuse a reference naming both a legacy data table and a role When a workspace stores both `sales` and a legacy `sales?role=analytics`, the reference resolved to the legacy entry without a role, so browsing `sales` as `analytics` reached another data table. Co-Authored-By: Claude Opus 5 (1M context) * fix: declare the default role in migrations written for a data table whose name contains '?' Such a data table connects as its default role without naming it, so the migrations the manager wrote for it declared no role and ran as admin. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * feat(datatables): put a data table's connection under Postgres roles A data table backed by the instance database resolved to exactly one Postgres connection, `custom_instance_user`, for everyone who could reach it at all. There was no way to say this job reads, that one writes, this one never sees the salaries table. A data table role is now a real Postgres login on the cluster, defined once for the instance by a superadmin and named exactly as they named it. A script that declares `-- role analytics` connects as `analytics`, and Postgres decides what it may touch — grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`. A data table with no `permissions` block behaves exactly as before. Everything that opens a connection on someone's behalf goes through one chokepoint, `get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when there is none. The role logs in as itself — never `SET ROLE`, which a script could `RESET ROLE` its way out of. A fork's data table entry becomes a pointer at the workspace that governs it rather than a copy of it. The settings clone used to hand a fork a byte-identical entry naming the parent's database, which a fork admin could edit to grant themselves `admin` there; a pointer has nothing local to edit, and its tenants are evaluated as a member of the governing workspace, by email. `permissions` is stripped from the workspace export and ignored on import: tenants name principals of one workspace, and a settings push is not where an access decision should be made. Operations that see the whole database whatever the roles grant stay with the governing workspace's admins: editing the roles, a migration that declares none, and opening a replication stream for a Postgres trigger or capture. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): gate the paths that reach a whole database as admin Auditing what still resolved through the unchecked resolver turned up three that act for a caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema export, the full-schema read, database creation, import and the forked-database drop), the connection test, and the schema snapshot a fork clone takes of its parent. On a data table under roles each let any workspace member — or a fork admin who is nobody in the governing workspace — read or copy the whole database whatever its roles grant. All three now require admin reach on the governing workspace. A dump taken under a restricted role would be a silently truncated copy rather than an error, so refusing is the only right answer for the copy paths. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin putting a *resource-backed* data table under roles, at which point the executor dialled the host that resource names — one the admin chose — with the role's real cluster password, and `CONNECT` is granted to every registered instance database. Both ends now refuse: the permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials on a non-instance entry rather than trusting the record it read. Two more places reached the governing database without answering to it. The initial-migration generator returned a `pg_dump` of the whole schema to any member. And the migration rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after which the parent re-runs every migration from zero. The remote half is now skipped when the entry resolves into another workspace, which is also just correct: a fork renaming what it calls a data table changes nothing about the data table. Also: revoking a tenant now bounces the replication streams of every workspace holding an entry that resolves here, not only the governing one, so a fork's trigger stops rather than living on inside its open connection; the instance role catalog and the governing workspace's tenant lists are no longer returned to someone who cannot edit them; and the tenant rename dedup collapses non-adjacent duplicates, per role rather than once any role changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): fail loudly where a role or a pointer can be left half-recorded Three ways the feature could end up in a state nobody could see or undo. Creating a role writes the cluster first and the catalog second, but the catalog write was an `UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a live login with a password nobody recorded: invisible to the catalog, un-recreatable because the name is taken, and un-deletable because there is no entry to delete. It now errors, so the operation is retryable once the row is restored. Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy would hand each fork the database outright — so the delete now names the data tables it stranded, and resolving one says which workspace is missing rather than reporting a data table this workspace never had. `InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now hand-written so `{:?}` on the catalog cannot put a live credential in a log line. Adds the two branches the reviews found unpinned: a caller who is not a member of the governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that predates this and sends no job id. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): unbreak two operator messages and two comments that described other code The two strings this branch added for states an operator hits once — the catalog write that matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence. `list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop it, and the comment now says why that is the right place to stay quiet. The non-superadmin check in `edit_datatable_config` was introduced as also covering references, which it does not and need not: `reference` is overwritten from the stored entry for every caller before the check runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): serialize role catalog mutations, and state each helper's authorization contract The catalog is one JSON document, so create, rename, enable and delete are all read-modify-write. Two concurrent creates read the same snapshot, both succeed in the cluster, and the second write drops the first — leaving a live Postgres login with a password nobody recorded, which is the exact state the delete path exists to prevent. Every mutation now runs in one transaction holding an advisory lock across the read, the cluster DDL and the write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL helpers take that transaction rather than the pool, which is what makes the lock cover them. Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely multi-statement SQL, and its future is not `Send`, which an axum handler holding the transaction requires. Each of these is one statement anyway. The new cross-crate surface now says what callers must do. `read_role_catalog` returns plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and `converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's raw config. All of them are superadmin-gated by their current handlers, but nothing said so at the definition, which is where the next caller looks. Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip that did not land; the rename affordance is the design-system `Button`, not a raw one; and `resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing resolves as the data table's default until the database manager grows a picker. Why role passwords stay a plain `String` while the instance user's password beside them is a `StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an operator supplies it and may want it from their own backend, while these are minted here and never entered by anyone, so there is nothing for a ref to point at. Encrypting generated secrets at rest is a separate change that would take the replication password with it. Now said at the field. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): give the role catalog its own row, out of reach of the config machinery Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways. The catalog serializes a generated Postgres password per role, and that row is the operator-facing instance config, so the passwords reached `get_instance_config` and its YAML editor — a live cluster credential in a response body, a UI field and any log of either. Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of that key writes the row back without it and the catalog is gone, while the cluster keeps every login it described. `custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret, written only by the server, never operator-authored, hidden so the config machinery cannot read, rewrite or drop it. The catalog is the same thing, so it now has the same shape: `datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist. No redaction to keep in step with three code paths, and no way for a neighbouring write to take it out. Two races on the same shared documents. `edit_datatable_config` read the stored data tables outside its transaction and then wrote the whole `datatable` document, so a permissions save committing in between was silently rolled back; it now reads under `FOR UPDATE`. And `set_datatable_permissions` validated role ids against the catalog before opening its transaction, so a deletion in between let it write a deleted role back — including as the default, which every later job then fails on; it now holds the catalog lock and the settings row across validation and write. Completes the authorization contracts the previous commit claimed but did not finish: `read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole job is to answer for a workspace the caller may not belong to, and `converge_connect_grants_with`, which had not inherited its wrapper's. Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but `extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so code written from that page passed a keyword-only argument positionally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all three handlers left it green — it pinned Postgres, not the code it was written for. It now drives `create_datatable_role` twice concurrently and asserts the catalog kept both names. Checked the way the last one should have been: removing the lock from the handler makes it fail with "wmtest_a_… is a live cluster login the catalog forgot". The contracts added last commit were stricter than this PR's own callers, which is worse than none — the next reader sees a rule already broken and learns to ignore it. `read_role_catalog` said superadmin-only while two of its four callers are open to any workspace member, and `converge_connect_grants` said superadmin while `set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the rule that actually holds is about the credential never reaching a response, log, audit record or export, not about who may call. They now say that. `read_datatable_entry` gets the same treatment rather than the one the earlier message claimed for it: it is the primitive every resolution goes through, so it is deliberately open, and what must not escape is `permissions` — it names the governing workspace's users, groups and folders. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): close the last ways a role or a pointer can be left pointing at nothing The raw settings readers hand back whatever is in the row, so moving the catalog into its own `global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles` and the settings listing returning every live password. Both now filter that one key. The neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it predates this and widening the fix to it is a decision about an operator workflow, not a consequence of this change. Three ways a save could leave something resolving to nothing: A permissioned data table could be moved to a PostgreSQL resource. The block was carried across as a server-owned field, the runtime refuses roles on a resource-backed table, so the save succeeded and every job afterwards failed. Refused instead — turning roles off first is one step, and it keeps discarding an access decision something somebody chose. Renaming a governing data table left every fork pointing at the old name: the data table disappears from their pickers and their jobs stop, with nothing in the renaming workspace to suggest why. The rename now follows into the pointers in the same transaction. Deleting one cannot be followed the same way, so it is reported instead — the response names what it stranded, the way deleting a workspace does, and the fork's own error already says which workspace is gone. Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions handler reaches it as a workspace admin (the same class fixed last commit, one instance missed); the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog` was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its roles only on success — a failing run is exactly the one that creates them without recording them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * refactor(datatables): put the role catalog in its own table, not in global_settings Five findings across three rounds were all the same choice. A set of live Postgres credentials was living in `global_settings`, which has generic read, list, write, config-export and CLI round-trip paths that know nothing about what they carry: the passwords reached the instance config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog, `GET /settings/global/{key}` and the settings listing returned them raw, and this round the redaction that fixed the last two turned `wmill instance push` into something that wipes every password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles` could also empty it outside the lock. The approved plan offered a table or `global_settings`, so this is the other option it already allowed rather than a new design. `datatable_role` is a table: no generic settings path can read it, list it, export it, write it or round-trip it, so none of the five needs a guard. The redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it. One row per role also removes the read-modify-write the concurrency work was about: two concurrent creates are two inserts, and the unique index on `name` is what settles a collision. The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to another transaction until commit, so without it both creates pass their `pg_roles` check. Also from this round: rename mappings are checked against the configuration they claim to describe, since fork pointers are rewritten from them — a caller could otherwise submit `main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning naming forks a delete stranded reached the response but not the screen: both the data table settings save and the workspace delete now show it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): validate a rename against the save it describes, and re-check under the locks Three from the round, all about deciding on state that could already have moved. A permission save resolved the data table and checked it was instance-backed before taking any lock, then wrote under one. A config save committing in between could move the table onto a PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in which case the write targeted a key that no longer existed and reported success having changed nothing. It now re-resolves and re-checks on the locked state. Rename validation checked that the source existed before and the target existed after, which still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows onto a different data table, silently, because it keeps resolving. The rule is now the actual old-to-new key transition — a source may only survive if another rename took its name, and a target may only pre-exist if another rename freed it. That also stops two sources sharing one target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so a swap cannot be done one save at a time, and refusing it was a regression against main. The pointer cascade now runs in two passes through a temporary name, the way the migration cascade one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once from what it named before the save. The tenant mutators say what they are for: they write an access decision for any workspace named, with an arbitrary mutation, and exist for the transaction that frees or renames a principal. Editing a decision on purpose belongs in the permissions endpoint. Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole document, so a workspace whose pointers name something else is not rewritten to a byte-identical value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from the JSON document, one of which would emit `pwd`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call `/workspaces/leave` — a different handler in a different crate with the same name — which deleted the membership and left `u/` in the tenant lists. Leaving and rejoining therefore restored the access the leave was supposed to end, and a later account taking the username would have inherited it. The regression test drives the route the client actually calls; without the fix it fails with "leaving kept the tenant". The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's admin connection, created `_wm_migrations` and read it before reaching the per-migration role check — so with nothing pending, nothing was checked at all. Rollback returned before its check when nothing was applied, and the status endpoint had none. All three now ask, before any connection is opened, whether the caller can reach the data table as any role at all; which role a given migration runs as is still decided per migration, and by the executor after that. Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now runs in the same transaction, so the drop, the row and every tenant list commit together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse to copy a data table that is under roles pg_dump carries no roles and the import runs with --no-privileges, so a copied data table arrives owned by the admin connection with no GRANT for any role. The settings clone brings `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were given, and are denied by Postgres on everything: an entry that reads as configured and answers nothing. Refuse the copy — in the import endpoint before any data moves, and in the fork path the CLI takes. Replaying the source's owners and ACLs into the clone is what lifts this, and is a change of its own. Dropping `permissions` from the copy instead would be the unsafe half, since the copy holds the parent's rows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse the clone's database too, not only its data A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only the second refused a data table under roles, so a fork asking to clone one created and registered an empty `wm_fork_…` instance database and then failed — and nothing collects it, since `drop_forked_datatable_databases` only drops entries carrying `forked_from` and no entry names this one. Refuse in both, so the clone stops before a database exists. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * nit worker error msg * fix pg_dump stuck on version 17 on nix * fix(datatables): refuse a malformed role annotation instead of ignoring it `-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed the annotation parser's exact-match rule, so the query fell through to the data table's default role and ran, silently, under a login the author did not choose. Naming a role exists precisely to not do that. A leading comment whose first word is `role` is now an annotation attempt: the keyword matches case-insensitively, one trailing `;` is tolerated, and anything else is an error naming the line. Only callers that already know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its comments. Also bumps the dev shell's postgres client to 18 — it trailed the server the dev database runs, which takes out every data table export, clone and fork-with-data. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse a malformed role query string instead of ignoring it `?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference parser's exact-match rule, so the connection resolved to the data table's default role and ran under a login the caller never asked for — the URI half of the same trap as a malformed `-- role` annotation. The key now matches case-insensitively, and anything else in the query string is an error naming it; `role` is the only parameter a reference takes. Callers that only need the entry keep a lenient `datatable_ref_name`, since they never act on the role. The DuckDB `ATTACH` parser propagates it rather than attaching under the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): carry the role annotation into the row_to_json retry The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment block never reached the second attempt — and with it the `-- role ` line that decides which login the query runs as. The retry connected as the data table's default role instead, so a query the first attempt was denied could succeed on the second, reported as "recovered with the row_to_json fix". Carry the leading comment block over. The retry itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore(datatables): don't mount the roles UI until the ACL editor lands Enforcement ships first. The permissions drawer is what turns roles on, and the catalog section is what creates them — both are only useful once there is a way to grant a role the privileges it needs, which arrives with the ACL editor. Left mounted they would offer a feature whose other half does not exist. The two components are complete and reviewed; only their call sites here are commented out, with a note pointing the follow-up PRs at them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): honour `-- role: x`, and fix the DuckDB attach test Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse to roll back the catalog while roles exist The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * refactor(datatables): settle clonability in one place, before anything is created A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): let a retried clone reclaim its own leftover database A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) * Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bounce the streams reading a data table when it is deleted Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) * refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. * fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): drop a DuckDB data table secret once its ATTACH has used it Co-Authored-By: Claude Opus 5 (1M context) * perf(datatables): resolve a workspace's data tables per pointer hop, not per entry Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): hold the parent's settings while a fork points at its data tables Co-Authored-By: Claude Opus 5 (1M context) * feat(db-manager): add a schema diagram view The database manager could only show one table at a time, so nothing in it answered "how do these tables relate". This adds a Diagram mode beside the data view, for PostgreSQL: the tree's checkboxes pick the tables, and the canvas draws them as cards on a pan/zoom surface. There are deliberately no edges. A schema of any size turns an edge-drawn ERD into a hairball, so the relationships surface on hover instead: hovering a table lights every table it is linked to and the columns tying them together, hovering a column lights only the column it is actually paired with. Only the connected database can be drawn, so another data table's rows in the tree get no checkbox in this mode, and a row of the connected one toggles what is drawn rather than selecting it for a data pane that is not shown. Foreign keys come from a new ALL_FOREIGN_KEYS marker that reads the whole database in one query. The existing FOREIGN_KEYS marker is per table, which would be one job per card. It reads pg_constraint rather than information_schema, whose constraint_column_usage loses which column of a composite key pairs with which. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): discard a superseded relations read The manager is not remounted on every database change and a queued job cannot be recalled, so a slow read could resolve after the next database's and decorate its tables with the previous one's relations: runed stores whatever the fetcher returns, and this one ignored its abort signal. It now throws on a superseded run, the way the per-table foreign-key resource beside it already does, and the result carries the database it was read from so it is only ever shown alongside that one. The database is identified by a key string passed in rather than by the objects read from it: those are replaced whenever the manager re-reads, which says nothing about the database having changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): retry a failed relations read on re-entry A failed read was cached like a successful one, so a transient worker or connection failure left the diagram without relations until the schema was reloaded or the manager remounted. The cache now only answers for a read that succeeded, and leaving the diagram and coming back retries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): scope the diagram's selection to its database The manager is not remounted on every database change, so the tables checked for one database's diagram carried into the next: same-named tables were drawn from a database they were never chosen in, and differing names left an empty canvas that the first draw no longer filled, having already run. Both the selection and that first draw are now scoped to the database they belong to. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * refactor(db-manager): hold the diagram's state against its database Three separate pieces of diagram state each had to work out on their own which database they were about, and each got it wrong in a different way: the selection carried across a switch, the first-draw latch counted itself done before the schema it needed had arrived, and the relations read could land under the wrong database. They are now one record carrying the database it describes, read through a single derived where a key that no longer matches reads as a fresh diagram. The first draw waits for the schema rather than marking itself done without one, and any table the user checks counts as that draw, so an emptied canvas is never refilled behind them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): draw a schema the database actually has `selected.schemaKey` follows the tree and is only replaced once it is empty, so it can still name a schema of the database browsed before this one. The diagram's first draw took it on trust, found no tables under it and stood down for good, leaving that database on a blank canvas. It is now taken only when this database has that schema, and otherwise falls back to the same default the tree starts from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * feat(db-manager): move the view toggle into the drawer header The Data/Diagram toggle sat above the tree's search box, where it read as a filter over the tree rather than a choice about the pane beside it. It now sits with the manager's other header controls, left of the worker tag. The mode moves with it: the drawer holds it and the manager is told which view to show, clamping it for a database that has no diagram. Holding it outside the key that remounts the manager also means picking another data table or role no longer drops the user back into the data view. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * chore: update ee-repo-ref to 7e338e4dabf91689bfd7fb0333c6534040b17b59 This commit updates the EE repository reference after PR #787 was merged in windmill-ee-private. Previous ee-repo-ref: 0edd40979cf36bfba59323f3f6a0811ae1369cf5 New ee-repo-ref: 7e338e4dabf91689bfd7fb0333c6534040b17b59 Automated by sync-ee-ref workflow. * feat(datatables): clone a data table under roles with its owners and grants (#11120) Claude-Session: https://claude.ai/code/session_01UbrtwiYNfayrmqouBJHwGV Co-authored-by: Claude Opus 5 (1M context) * fix: open the raw app data table drawer when the workspace has none Selecting the first data table of an empty list passed undefined to the name check, which threw instead of opening the drawer on no data table. Co-Authored-By: Claude Opus 5 (1M context) * fix(db-manager): wait for table metadata before reading relations Returning to a cached database with the diagram selected remounts the manager with its schema ready but its metadata not yet loaded. The relations read started anyway, then ran again once the metadata landed, and the first queued job cannot be recalled by an abort, so both full-database queries took a worker and hit PostgreSQL. The read now waits for the metadata, whose arrival already re-runs it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): keep the relations read across a switch to the data view The relations came from a runed `resource`, which abandons a run whenever any of its inputs changes. A queued job cannot be recalled, so switching to the data view while relations loaded threw away a read that still ran to completion, and coming back queued a second full-database job. A plain read replaces it: at most one per database and metadata, started when the diagram needs it, kept across mode changes, and replaced only by another database, reloaded metadata, or a retry after failing. An answer arriving after its read was replaced is ignored, so it can never decorate another database's tables. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): hand down metadata only for the database it was read from A resource keeps its previous value while it refetches, so after a switch of database the manager was handed the previous database's metadata as the new one's. The diagram started its relations read beside it, then again once the right metadata arrived: two full-database jobs, the first unrecallable. It also drew the previous database's columns meanwhile. The metadata now carries the database it was read for, and nothing is handed down while it belongs to another. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): address the schema diagram's review nits - Clear a pinned highlight once its table is unchecked or its column is folded away by collapsing the card; it otherwise left the canvas dimmed around nothing on screen. - Use the design-system button for a card's expand control. - Describe the view-mode counter as what it records — switches between the data and diagram views, not how the manager was opened. - Give the relation index loop its real reason: hovering either table of a pair lights both of its columns. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): keep schema diagram cards in place and open tables from the header - Cards were draggable, but the canvas is laid out from scratch on every change, so a moved card snapped back at the next one. Dragging is off. - Double-clicking a card anywhere opened its table. Expanding a card moves its rows under the pointer, so a double-click on the expand control landed its second click on a row and left the diagram. Only the header opens the table now; the rows keep hovering and pinning columns. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * chore: update ee-repo-ref to 39f8896379295728a509c4d642574bb8173368e8 This commit updates the EE repository reference after PR #784 was merged in windmill-ee-private. Previous ee-repo-ref: 720f334ee1843878749e9520567a7b657fc2deda New ee-repo-ref: 39f8896379295728a509c4d642574bb8173368e8 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-common/src/query_builders.rs | 62 ++++++ docs/feature-telemetry.md | 8 +- frontend/src/lib/components/DBManager.svelte | 209 ++++++++++++++++-- .../lib/components/DBManagerContent.svelte | 25 ++- .../src/lib/components/DBManagerDrawer.svelte | 42 +++- .../lib/components/InstanceSettings.svelte | 6 +- frontend/src/lib/components/dbOps.ts | 16 ++ .../src/lib/components/dbRelations.test.ts | 81 +++++++ frontend/src/lib/components/dbRelations.ts | 137 ++++++++++++ .../dbdiagram/DbSchemaDiagram.svelte | 171 ++++++++++++++ .../components/dbdiagram/DbTableNode.svelte | 118 ++++++++++ .../dbdiagram/DiagramAutoFit.svelte | 20 ++ .../dbdiagram/dbDiagramHighlight.svelte.ts | 80 +++++++ .../dbdiagram/dbDiagramModel.test.ts | 45 ++++ .../components/dbdiagram/dbDiagramModel.ts | 131 +++++++++++ 16 files changed, 1126 insertions(+), 27 deletions(-) create mode 100644 frontend/src/lib/components/dbRelations.test.ts create mode 100644 frontend/src/lib/components/dbRelations.ts create mode 100644 frontend/src/lib/components/dbdiagram/DbSchemaDiagram.svelte create mode 100644 frontend/src/lib/components/dbdiagram/DbTableNode.svelte create mode 100644 frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte create mode 100644 frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts create mode 100644 frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts create mode 100644 frontend/src/lib/components/dbdiagram/dbDiagramModel.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b82e25d93b..24c90028c4 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -559a9ff78ea03439553cf2b412e73765762e9013 +39f8896379295728a509c4d642574bb8173368e8 diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index 9d85278404..546d2d454b 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -333,6 +333,7 @@ pub fn try_expand_internal_db_query( // Metadata queries "LOAD_TABLE_METADATA" => expand_load_table_metadata(json_str, db_type), "FOREIGN_KEYS" => expand_foreign_keys(json_str, db_type).map(ExpandedQuery::sql), + "ALL_FOREIGN_KEYS" => expand_all_foreign_keys(db_type).map(ExpandedQuery::sql), "PRIMARY_KEY_CONSTRAINT" => { expand_primary_key_constraint(json_str, db_type).map(ExpandedQuery::sql) } @@ -2737,6 +2738,47 @@ fn expand_foreign_keys(json_str: &str, db_type: DbType) -> Result Result { + make_all_foreign_keys_query(db_type) +} + +/// Every foreign key of the database, one row per referencing column. +/// +/// Read from `pg_constraint` rather than `information_schema`: the latter's +/// `constraint_column_usage` loses the pairing of a composite key's columns, +/// which the diagram needs to line a source column up with the column it points +/// at. `unnest(conkey, confkey)` keeps them zipped in declaration order. +fn make_all_foreign_keys_query(db_type: DbType) -> Result { + if db_type != DbType::Postgresql { + return Err(format!( + "The schema diagram is only supported for PostgreSQL, not {:?}", + db_type + )); + } + Ok(String::from( + "SELECT + con.conname as fk_constraint_name, + src_ns.nspname as source_schema, + src.relname as source_table, + src_att.attname as source_column, + tgt_ns.nspname as target_schema, + tgt.relname as target_table, + tgt_att.attname as target_column, + k.ord as ordinal +FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class src ON src.oid = con.conrelid + JOIN pg_catalog.pg_namespace src_ns ON src_ns.oid = src.relnamespace + JOIN pg_catalog.pg_class tgt ON tgt.oid = con.confrelid + JOIN pg_catalog.pg_namespace tgt_ns ON tgt_ns.oid = tgt.relnamespace + JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(src_attnum, tgt_attnum, ord) ON true + JOIN pg_catalog.pg_attribute src_att ON src_att.attrelid = src.oid AND src_att.attnum = k.src_attnum + JOIN pg_catalog.pg_attribute tgt_att ON tgt_att.attrelid = tgt.oid AND tgt_att.attnum = k.tgt_attnum +WHERE con.contype = 'f' + AND src_ns.nspname NOT IN ('pg_catalog', 'information_schema') +ORDER BY src_ns.nspname, src.relname, con.conname, k.ord;", + )) +} + fn expand_primary_key_constraint(json_str: &str, db_type: DbType) -> Result { let p: PrimaryKeyConstraintPayload = serde_json::from_str(json_str) .map_err(|e| format!("Invalid PRIMARY_KEY_CONSTRAINT payload: {}", e))?; @@ -4721,6 +4763,26 @@ mod tests { assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n")); } + // ----------------------------------------------------------------------- + // ALL_FOREIGN_KEYS + // ----------------------------------------------------------------------- + + #[test] + fn test_expand_all_foreign_keys_postgresql() { + let marker = r#"-- WM_INTERNAL_DB_ALL_FOREIGN_KEYS {}"#; + let sql = expand_code(marker, &ScriptLang::Postgresql); + assert!(sql.contains("con.contype = 'f'")); + // Columns of a composite key must stay zipped with the ones they point at. + assert!(sql.contains("unnest(con.conkey, con.confkey) WITH ORDINALITY")); + } + + #[test] + fn test_expand_all_foreign_keys_rejects_non_postgres() { + let marker = r#"-- WM_INTERNAL_DB_ALL_FOREIGN_KEYS {}"#; + let result = try_expand_internal_db_query(marker, &ScriptLang::Mysql); + assert!(result.unwrap().is_err()); + } + // ----------------------------------------------------------------------- // PRIMARY_KEY_CONSTRAINT // ----------------------------------------------------------------------- diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index ae62d46b94..9dacfc1cbd 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 51 registered actions across nineteen features (`ai_session`, `ai_chat`, -`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, -`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, -`usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the +It currently carries 58 registered actions across twenty features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `db_manager`, `flow_editor`, +`flow_run`, `flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, +`hub_script`, `usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 4775409252..16e64dc5a0 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,5 +1,5 @@ + +
              + {#if error} +
              + {error} +
              + {/if} + {#if tables.length === 0} +
              + Check tables in the sidebar to draw them here +
              + {:else} + + highlight.clear()} + > +
              + + + + +               +               + {/if} + {#if loading} +
              + + Loading relationships +
              + {/if} +
              + + diff --git a/frontend/src/lib/components/dbdiagram/DbTableNode.svelte b/frontend/src/lib/components/dbdiagram/DbTableNode.svelte new file mode 100644 index 0000000000..d17faaa006 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DbTableNode.svelte @@ -0,0 +1,118 @@ + + + +
              highlight.hover({ table: table.key })} + onmouseleave={() => highlight.hover(undefined)} + onclick={() => highlight.togglePin({ table: table.key })} + role="button" + tabindex="-1" +> + + +
              + + {table.table} + {table.schema} +
              + + {#each columns as column (column.name)} + {@const columnKey = columnKeyIn(table.key, column.name)} + {@const isLit = highlight.columns?.has(columnKey)} + +
              highlight.hover({ table: table.key, column: column.name })} + onmouseleave={() => highlight.hover({ table: table.key })} + onclick={(e) => { + e.stopPropagation() + highlight.togglePin({ table: table.key, column: column.name }) + }} + > + {#if column.isPrimaryKey} + + {:else if data.index.referencingColumns.has(columnKey)} + + {:else} + + {/if} + {column.name} + {column.datatype} +
              + {/each} + + {#if hasExpandToggle(table)} +
              + +
              + {/if} +
              diff --git a/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte b/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte new file mode 100644 index 0000000000..b271886abd --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte @@ -0,0 +1,20 @@ + diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts b/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts new file mode 100644 index 0000000000..54f52b85ca --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts @@ -0,0 +1,80 @@ +import { getContext, setContext } from 'svelte' +import { columnKeyIn, type RelationIndex } from '../dbRelations' + +/** What the pointer is on: a whole table, or one of its columns. */ +export type HighlightTarget = { table: string; column?: string } + +const EMPTY: ReadonlySet = new Set() + +/** + * The diagram draws no edges, so the relationships only exist while something is + * highlighted. This owns that: what is hovered (or pinned by a click), and which + * tables and columns it lights up. + */ +export class DbDiagramHighlight { + #getIndex: () => RelationIndex + + hovered = $state(undefined) + /** Set by clicking, so the relations can be read without holding the pointer + * still. A hover takes precedence while it lasts. */ + pinned = $state(undefined) + + target = $derived(this.hovered ?? this.pinned) + + /** Tables that stay lit. Undefined means nothing is highlighted and every + * table renders at full strength. */ + tables = $derived.by | undefined>(() => { + const target = this.target + if (!target) return undefined + if (target.column) { + const columnTable = this.#getIndex().columnTable + const tables = new Set([target.table]) + for (const column of this.columns ?? EMPTY) { + const table = columnTable.get(column) + if (table) tables.add(table) + } + return tables + } + return new Set([target.table, ...(this.#getIndex().relatedTables.get(target.table) ?? EMPTY)]) + }) + + /** Columns that stay lit inside the highlighted tables. */ + columns = $derived.by | undefined>(() => { + const target = this.target + if (!target) return undefined + const index = this.#getIndex() + if (target.column) { + const key = columnKeyIn(target.table, target.column) + return new Set([key, ...(index.linkedColumns.get(key) ?? EMPTY)]) + } + return index.relatedColumns.get(target.table) ?? EMPTY + }) + + constructor(getIndex: () => RelationIndex) { + this.#getIndex = getIndex + } + + hover(target: HighlightTarget | undefined) { + this.hovered = target + } + + /** Clicking the already-pinned target unpins it. */ + togglePin(target: HighlightTarget) { + const current = this.pinned + this.pinned = + current && current.table === target.table && current.column === target.column + ? undefined + : target + } + + clear() { + this.hovered = undefined + this.pinned = undefined + } +} + +const contextKey = 'DbDiagramHighlight' + +export const setDbDiagramHighlight = (highlight: DbDiagramHighlight) => + setContext(contextKey, highlight) +export const getDbDiagramHighlight = () => getContext(contextKey) diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts b/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts new file mode 100644 index 0000000000..8b6b584860 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { CARD_WIDTH, cardHeight, layoutTables, type DiagramTable } from './dbDiagramModel' + +function table(name: string, columnCount: number): DiagramTable { + return { + key: `shop.${name}`, + schema: 'shop', + table: name, + columns: Array.from({ length: columnCount }, (_, i) => ({ + name: `c${i}`, + datatype: 'text', + isPrimaryKey: i === 0 + })) + } +} + +describe('layoutTables', () => { + // Runs again on every checkbox click, so cards must not land on top of each + // other for any mix of card heights. + it('never overlaps two cards', () => { + const tables = Array.from({ length: 40 }, (_, i) => table(`t${i}`, 1 + (i % 17))) + const expanded = new Set(['shop.t3', 'shop.t20']) + const positions = layoutTables(tables, expanded) + + const boxes = tables.map((t) => ({ + x: positions[t.key].x, + y: positions[t.key].y, + h: cardHeight(t, expanded.has(t.key)) + })) + for (let i = 0; i < boxes.length; i++) { + for (let j = i + 1; j < boxes.length; j++) { + const a = boxes[i] + const b = boxes[j] + const apart = + a.x + CARD_WIDTH <= b.x || b.x + CARD_WIDTH <= a.x || a.y + a.h <= b.y || b.y + b.h <= a.y + expect(apart, `${tables[i].key} overlaps ${tables[j].key}`).toBe(true) + } + } + }) + + it('places the same tables in the same spots every time', () => { + const tables = Array.from({ length: 12 }, (_, i) => table(`t${i}`, 3 + i)) + expect(layoutTables(tables, new Set())).toEqual(layoutTables(tables, new Set())) + }) +}) diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts b/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts new file mode 100644 index 0000000000..ccd32cdb22 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts @@ -0,0 +1,131 @@ +import type { DBSchema } from '$lib/stores' +import type { ColumnDef } from '../apps/components/display/dbtable/utils' +import type { SelectedTable } from '../DBManager.svelte' +import { tableKeyOf } from '../dbRelations' + +export type DiagramColumn = { + name: string + datatype: string + isPrimaryKey: boolean +} + +export type DiagramTable = { + /** `schema.table`, the key relations and highlights are addressed by. */ + key: string + schema: string + table: string + columns: DiagramColumn[] +} + +/** Card geometry. The node component must render exactly the rows the layout + * budgeted for, or cards overlap. */ +export const CARD_WIDTH = 248 +export const HEADER_HEIGHT = 34 +export const ROW_HEIGHT = 22 +const CARD_PADDING = 6 +const GAP_X = 32 +const GAP_Y = 20 +/** Rows shown before a card collapses into a "+N more" row. */ +export const COLLAPSED_ROW_LIMIT = 12 +/** Cards are packed towards this width/height ratio, so the canvas opens on + * something screen-shaped rather than one very long column. */ +const TARGET_ASPECT = 1.7 + +export function visibleColumns(table: DiagramTable, expanded: boolean): DiagramColumn[] { + return expanded ? table.columns : table.columns.slice(0, COLLAPSED_ROW_LIMIT) +} + +export function hiddenColumnCount(table: DiagramTable, expanded: boolean): number { + return table.columns.length - visibleColumns(table, expanded).length +} + +/** Whether the card carries a row to expand or collapse its column list. */ +export function hasExpandToggle(table: DiagramTable): boolean { + return table.columns.length > COLLAPSED_ROW_LIMIT +} + +export function cardHeight(table: DiagramTable, expanded: boolean): number { + const rows = visibleColumns(table, expanded).length + (hasExpandToggle(table) ? 1 : 0) + return HEADER_HEIGHT + rows * ROW_HEIGHT + CARD_PADDING +} + +/** The tables to draw, in the order they are laid out. */ +export function buildDiagramTables( + dbSchema: DBSchema, + colDefs: Record | undefined, + selected: SelectedTable[] +): DiagramTable[] { + if (dbSchema.lang === 'graphql') return [] + const tables: DiagramTable[] = [] + + for (const { schema, table } of selected) { + const key = tableKeyOf({ schema, table }) + const fromColDefs = colDefs?.[key] + let columns: DiagramColumn[] + if (fromColDefs?.length) { + columns = fromColDefs.map((col) => ({ + name: col.field ?? '', + datatype: col.datatype ?? '', + isPrimaryKey: !!col.isprimarykey + })) + } else { + // The cached schema has no primary-key flag; it only stands in until the + // column metadata query lands. + const schemaColumns = dbSchema.schema[schema]?.[table] + if (!schemaColumns) continue + columns = Object.entries(schemaColumns).map(([name, col]) => ({ + name, + datatype: col.type, + isPrimaryKey: false + })) + } + tables.push({ key, schema, table, columns }) + } + + tables.sort((a, b) => a.schema.localeCompare(b.schema) || a.table.localeCompare(b.table)) + return tables +} + +/** Packs the cards into balanced columns, in reading order. Runs again on every + * change to the selection, so it must be deterministic: the same tables always + * land in the same place. */ +export function layoutTables( + tables: DiagramTable[], + expanded: Set +): Record { + const heights = tables.map((t) => cardHeight(t, expanded.has(t.key))) + const stacked = heights.reduce((a, b) => a + b, 0) + GAP_Y * Math.max(0, tables.length - 1) + + let columnCount = 1 + let bestScore = Infinity + for (let c = 1; c <= tables.length; c++) { + const width = c * CARD_WIDTH + (c - 1) * GAP_X + const height = stacked / c + // Compared in log space so twice-too-wide and twice-too-tall score alike. + const score = Math.abs(Math.log(width / height / TARGET_ASPECT)) + if (score < bestScore) { + bestScore = score + columnCount = c + } + } + + const target = stacked / columnCount + const positions: Record = {} + let column = 0 + let columnHeight = 0 + + tables.forEach((table, i) => { + const height = heights[i] + // Overshooting the target by more than stopping short of it costs less in + // the next column. + const overshoot = columnHeight + height - target + if (column < columnCount - 1 && columnHeight > 0 && overshoot > target - columnHeight) { + column++ + columnHeight = 0 + } + positions[table.key] = { x: column * (CARD_WIDTH + GAP_X), y: columnHeight } + columnHeight += height + GAP_Y + }) + + return positions +} From 592ef736104ff63dd36978a69353a2585636da87 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 17:28:52 +0200 Subject: [PATCH 52/53] fix: keep a watched job's code when its completion update drops it (#11290) Co-authored-by: Claude Opus 5 --- frontend/src/lib/components/JobLoader.svelte | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index 324bdea6c7..0994905121 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -444,6 +444,8 @@ streamedResult.length >= completedResult.length ? streamedResult : completedResult + njob.raw_code ??= job?.raw_code + njob.raw_flow = job?.raw_flow ?? njob.raw_flow job = njob await onJobCompleted(testId, job, callbacks) } @@ -870,6 +872,11 @@ const njob = previewJobUpdates.job as Job & { result_stream?: string } njob.logs = pickMoreCompleteLogs(job?.logs, njob.logs) njob.result_stream = job?.result_stream ?? '' + // The server sends the completed job without its code to keep the + // update small (no raw_code, "..." for a deployed flow's inline + // steps), so keep the code fetched with the initial job. + njob.raw_code ??= job?.raw_code + njob.raw_flow = job?.raw_flow ?? njob.raw_flow job = njob onJobCompleted(id, job, callbacks) } From ea0b947644d4094304ecea26406fa6aac15dcfac Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 22 Sep 2026 17:29:20 +0200 Subject: [PATCH 53/53] fix: stop the chat repeating run results the card already shows (#11293) * fix: tell the chat not to repeat run results the card already shows * fix: add the run result note on completion, background runs included * fix: skip the run result note on failed runs * fix: keep the run result note off background completions --- .../components/copilot/chat/shared.test.ts | 44 +++++++++++++++++++ .../src/lib/components/copilot/chat/shared.ts | 16 ++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 768b6d1b65..005a7666db 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1510,6 +1510,50 @@ describe('pollJobCompletion detach', () => { }) }) +describe('executeTestRun result note', () => { + // A failed run's card lands on the error, with the logs a tab away: the model must stay + // free to quote them, so only a success says the user already sees the run. + it('tells the model not to repeat a successful result, and not a failed one', async () => { + vi.useFakeTimers() + try { + const { executeTestRun } = await import('./shared') + const { JobService } = await import('$lib/gen') + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ completed: true, running: false } as any) + const run = async (success: boolean) => { + vi.mocked(JobService.getJob).mockReset() + vi.mocked(JobService.getJob).mockResolvedValue({ + type: 'CompletedJob', + success, + result: [1, 2], + logs: 'ran' + } as any) + const promise = executeTestRun({ + jobStarter: async () => 'job1', + workspace: 'w', + toolId: 'tool1', + startMessage: 'Starting...', + contextName: 'script', + // The job hooks are what mark the global/sessions chat. + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + onJobStatus: vi.fn(), + onJobStarted: vi.fn() + } as any + }) + await vi.advanceTimersByTimeAsync(1000) + return promise + } + expect(await run(true)).toContain('Do not repeat') + expect(await run(false)).not.toContain('Do not repeat') + } finally { + vi.useRealTimers() + } + }) +}) + describe('deriveChatJobStatus', () => { // CompletedJob is discriminated by the presence of a `success` key; the branch // order deliberately mirrors JobStatusIcon so the badge and scalar never drift. diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index bd347224f9..f58ed3b6d3 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1871,6 +1871,15 @@ export async function buildTestRunArgs( return parsedArgs } +// Said with the result rather than in the system prompt: there, models still copied a +// result's table or JSON into the reply, under a card that already renders it. Only for +// a successful run in the generic card completedJobToolStatus fills: a failed run's card +// lands on the error with the logs a tab away, and a formatCompletion card shows its own. +// Never on a background completion: its card can be far up the chat by then, so the +// reply may be the only place the user reads the result. +const RESULT_SHOWN_NOTE = + "The user already sees this run's result in its card. Do not repeat the result or logs in your reply: say what it shows, quoting only the values your conclusion rests on." + // The string handed back to the model when a job is backgrounded. It carries the // job id so the model can pull status/args/result/logs on demand (get_run / list_runs), // and tells it the completion will be reported later (notify-only wake). @@ -2015,7 +2024,12 @@ export async function executeTestRun(config: TestRunConfig): Promise { ...(job.success ? {} : { error: getErrorMessage(job.result) }) }) - const summary = formatResultSummary(job.result, job.logs, job.success) + // detachEnabled marks the global/sessions chat; the in-editor chats read the summary + // alone. The card opening on the result is up to each tool: a run card does, a + // generic one only with `autoCollapseDetails: false` at its registration. + const summary = + (detachEnabled && job.success ? `${RESULT_SHOWN_NOTE}\n` : '') + + formatResultSummary(job.result, job.logs, job.success) // get_run only exists in the global/sessions chat (the same hosts that wire // the job hooks) — don't advertise it to in-editor chats. if (detachEnabled && config.contextName === 'flow' && !job.success) {