From e47aedac0a4af40dd697d5fc4d54dd3c8efe9ab8 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:25:16 +0200 Subject: [PATCH] feat: add SQL migrations for data tables (#9693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add datatable_migrations table * feat: add route to run datatable migrations * feat: sync datatable migrations as .up.sql/.down.sql files * feat: add datatable migrate up/down commands and post-push run prompt * feat: add datatable migrate new command to scaffold migrations * feat: add datatable migrations management UI * feat: prompt to create migration on DDL in datatable SQL editors * feat: support running a single specific datatable migration * feat: view migration content, run single migration, fix stacked modal * feat: per-row revert button with out-of-order warning * fix: avoid migrations list flicker on refresh after an action * feat: generate initial datatable migration via pg_dump * fix: surface datatable migration API error details in toasts * fix: revert created migration if create-and-run fails to run * fix: include postgres error detail in migration run/rollback failures * feat: sync datatable migrations as files via the workspace export * refactor: move datatable migrations to migrations/datatable/ path * fix: drop redundant datatable_migration label in sync output * fix: exclude datatable migration sql files from script metadata generation * feat: run datatable migrations as user-permissioned labeled jobs * feat: reject invalid datatable migrations on sync push * feat: datatable migrate up/down default to all datatables, --datatable to target one * fix: surface postgres error detail when datatable migrations fail to run * chore: regenerate CLI docs for datatable migrate commands * feat: default new datatable migration to a BEGIN/END transaction template * fix: validate datatable migration name and datatable at the API boundary * fix: ensure detected DDL ends with semicolon when wrapped in transaction * fix: re-prompt instead of stripping DDL when new-migration modal is cancelled * feat: refresh datatable schema after running a migration from the SQL REPL * feat: record db manager DDL on data tables as migrations * feat: make datatable migrations opt-in per data table * fix: make migration view editor read-only so its code can scroll * fix: don't re-prompt DDL guard when creating a migration without running * feat: generate down migrations for db manager DDL (postgres) * fix: correct down migration for db manager alters (no double-wrap, serial) * feat: explain migrations purpose with a tooltip in the migrations modal * compare paeg * feat: add datatable_migration kind to workspace diff pipeline * chore: point ee-repo-ref at datatable_migration git-sync companion * fix: harden datatable migration version allocation and initial-migration bookkeeping, add tests * feat: deploy and run datatable migrations on workspace merge Co-Authored-By: Claude Opus 4.8 (1M context) * Refactor + handle datatable setting delete/rename * refactor: move datatable migration rename/delete cascade into module Co-Authored-By: Claude Opus 4.8 (1M context) * chore(windmill-utils-internal): bump to 1.7.1 for datatable migration deploy provider methods Co-Authored-By: Claude Opus 4.8 (1M context) * feat(db-manager): add Migrations button to top bar, make Refresh icon-only Co-Authored-By: Claude Opus 4.8 (1M context) * BEGIN/END placeholder in down migration * feat: autofocus migration name input and flag it red when empty Co-Authored-By: Claude Opus 4.8 (1M context) * feat(datatable-migrations): allow non-admins to create/run/revert migrations, gate only opt in/out Co-Authored-By: Claude Opus 4.8 (1M context) * border nits * refresh db manager schema on migrations * BEGIN/END scaffold in CLI * feat(cli): push local datatable migrations before running on migrate up Co-Authored-By: Claude Opus 4.8 (1M context) * feat: flag invalid migration name with red border, not just empty Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop random slug from auto-generated migration names Co-Authored-By: Claude Opus 4.8 (1M context) * feat: offer revert-and-delete when deleting an installed migration Co-Authored-By: Claude Opus 4.8 (1M context) * feat: record fork merge as a migration when target datatable opts in * nit * clone migrations on fork * windmill-utils-internal * fix(datatable-migrations): serialize run/rollback with a per-db advisory lock Co-Authored-By: Claude Opus 4.8 (1M context) * fix(db-manager): fail closed when migrations-status check errors on DDL apply Co-Authored-By: Claude Opus 4.8 (1M context) * docs: fix generate_initial migration ordering comment to match code * chore(datatable-migrations): remove unused update_datatable_migrations endpoint Co-Authored-By: Claude Opus 4.8 (1M context) * fix: run DDL migration guard on the script editor Test button Co-Authored-By: Claude Opus 4.8 (1M context) * split * ee-repo-ref * chore(frontend): sync package-lock with package.json (@emnapi deps) Co-Authored-By: Claude Opus 4.8 (1M context) * fix(datatable-migrations): never resolve instance credentials into migration job args datatable_database_arg eagerly resolved instance data-table credentials (including the shared instance-wide Postgres password) and passed them as the migration job's plaintext `database` arg, landing in v2_job.args. Since the run route has no admin gate, a non-admin could run a migration and read args.database to recover the password, granting cross-workspace psql access to all instance data-table DBs. Pass a `datatable://` reference for both resource-backed and instance data tables instead; the pg executor already resolves it to real credentials server-side at run time, so nothing sensitive is ever stored in the job args. Co-Authored-By: Claude Opus 4.8 (1M context) * nit * fix: handle dollar-quoting and comments when splitting SQL statements * feat: deploy datatable migrations on merge with explicit opt-in error * fix(frontend): sync package-lock with npm 11 peer-dep resolution npm ci failed with 'Missing: @emnapi/core@1.11.2 / @emnapi/runtime@1.11.2 from lock file'. @napi-rs/wasm-runtime declares @emnapi/core|runtime ^1.7.1 as peerDependencies while @rolldown/binding-wasm32-wasi pins them to exactly 1.10.0. Newer npm (bundled with node 24 in CI) installs the peer deps at the highest match (1.11.2) alongside rolldown's nested 1.10.0, so the ideal tree needs both versions; the committed lock only had 1.10.0. Regenerate the lock with npm 11.18 so it carries both 1.11.2 (top-level, for the peer deps) and 1.10.0 (nested, for rolldown's pin). Verified npm ci passes. Co-Authored-By: Claude Opus 4.8 (1M context) * nit npm publish * fix: fail closed on migrations-status error in fork schema merge * nit CI emnapi/core version * prevent initial_datatable_migration if migrations already exist * fix(datatable-migrations): validate persisted data table names as path segments edit_datatable_config only validated rename segments, not the actual settings.datatables keys, so a data table could be saved directly under a name like '..' or one containing '/'. Since new tables default to migrations_enabled = true, generate_initial_datatable_migration would then insert a migration row and the sync export would build migrations/datatable//... paths from that name, producing malformed or directory-escaping export paths. Validate every persisted data table name in edit_datatable_config (alongside the existing rename checks) and add validate_datatable_path_segment to generate_initial_datatable_migration for defense in depth. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: scope datatable _wm_migrations by data table and cascade renames/deletes Co-Authored-By: Claude Opus 4.8 (1M context) * fix(system_prompts): resolve nested local command groups in CLI docs generator The CLI docs generator anchored on the first `new Command()` in a file and never resolved locally-defined command groups passed as `.command("name", localCmd)`. For datatable this flattened the nested `migrate` group: it emitted `datatable new/up/down` plus a bare `datatable migrate`, and mislabeled the datatable command with the migrate group's description. jobs was broken the same way (its description was pull's, and pull/push rendered empty). Anchor block extraction on the `export default`ed command, recurse into locally-defined `const x = new Command()` groups mounted as subcommands, and render nested sub-subcommands. Regenerated docs now show `datatable migrate new/up/down` and `jobs pull/push` with their real options. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop unreleased _wm_migrations legacy-upgrade handling Co-Authored-By: Claude Opus 4.8 (1M context) * fix: return datatable migration SQL from getItemValue for the diff drawer Co-Authored-By: Claude Opus 4.8 (1M context) * chore(frontend): use windmill-utils-internal 1.8.2 for migration diff drawer Co-Authored-By: Claude Opus 4.8 (1M context) * nit * nit * fix: handle datatable migration renames on push and dedupe timestamps * fix: reject rewriting an already-applied datatable migration on upsert Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): add missing @emnapi/core and @emnapi/runtime lockfile entries Resolves npm ci EUSAGE failure: the optional cpu:wasm32 @rolldown/binding-wasm32-wasi declares deps on @emnapi/core@1.11.2 and @emnapi/runtime@1.11.2 that had no resolved lockfile entries. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(cli): datatable migrate up/down default to main datatable, not all Co-Authored-By: Claude Opus 4.8 (1M context) * fix: fail closed when applied status unreadable on datatable migration rewrite Co-Authored-By: Claude Opus 4.8 (1M context) * fix: surface full error detail in Database Manager DDL/query errors * "See migration" button in the toast * feat: add Enter shortcut to Create-a-migration in the DDL guard * fix(frontend): warn before running a newly-created datatable migration out of order The row-level Run action warns when earlier migrations are still pending, but the create-and-run paths ran a just-created migration with `only` directly, applying it ahead of older pending migrations without that confirmation. Reuse the same "Run migration out of order" confirmation across all create-and-run paths via a shared helper (datatableMigrationUtils): - NewDataTableMigrationModal "Create and run" (and the DDL guard path) - DatatableSchemaDiff fork→parent merge - dbOps schema ops (DB manager create/alter/drop) — the pure factory throws a MigrationRunCancelled sentinel on decline, which DBTableEditor treats as a silent cancel Co-Authored-By: Claude Opus 4.8 (1M context) * fix: keep renamed datatable migrations visible in compare view * fix: record per-migration deployment on datatable migrations disable * fix(cli): run deployed datatable migrations after workspace merge The merge command upserted datatable_migration definitions into the target workspace and reported the item as successfully deployed, but never ran the migrations. For forked datatables backed by separate databases, this left the target schema unchanged until someone manually ran `wmill datatable migrate up`, while the CLI reported a successful merge. Collect the datatable migrations deployed (not deleted) into the target and, after the deploy loop, offer to run them via the existing offerToRunNewMigrations helper — the same post-deploy run prompt the push/sync path uses (interactive only; `--yes`/non-TTY skip the mutating run, matching push behavior). Export parseDatatableMigrationDeployPath so the merge path can parse the deployed items. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(backend): serialize datatable migration edits/deletes with the run lock A migration run snapshots a migration's code_up from datatable_migrations and only records its version in the data table's _wm_migrations after the job succeeds. upsert_datatable_migration checked _wm_migrations before allowing an edit but took no lock, so a concurrent edit could read "not applied yet", rewrite code_up/code_down, and then the in-flight run would record the version for the old SQL — leaving _wm_migrations pointing at SQL that was never applied (migrate up then skips it; rollback runs a down that doesn't match). Serialize definition rewrites and deletes with the same per-database advisory lock the run/rollback paths use: - Factor the connect+advisory-lock into lock_datatable_migration_runs and the applied-versions read into read_applied_versions_on_client. - run_datatable_migrations now snapshots the definitions AFTER taking the lock, so code_up can't change between snapshot and version-record. - upsert (when changing an existing def) and delete take the lock across the applied-check and the write; delete now rejects deleting an already-applied migration (would orphan its _wm_migrations record), symmetric with upsert. Both fail closed if the data table database is unreachable. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): stack the out-of-order migration confirm above the DB editor preview Creating a table on a migrations-enabled data table opened the DB table editor's "Confirm running the following" preview modal, whose confirm triggers applyDdl, which then asks for out-of-order confirmation. Both are ConfirmationModals with a hardcoded z-[9999]; the out-of-order one lives in DBManagerContent (mounted before the editor), so it rendered behind the still-open preview modal. Add an optional zIndexClass prop to ConfirmationModal (default z-[9999], backward-compatible) and give the DB-manager out-of-order confirm z-[10000] so it stacks on top. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 27672e37df5d9dfde94f19963d5ffcdf8dd5448c This commit updates the EE repository reference after PR #623 was merged in windmill-ee-private. Previous ee-repo-ref: 6c287041cd7edd4a77a4bc07ad0e156cec32cce4 New ee-repo-ref: 27672e37df5d9dfde94f19963d5ffcdf8dd5448c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...04ab3569a0611b33c261602d62e9c847c583e.json | 15 + ...1f7f387f5055c47f493271d26731336257384.json | 10 +- ...4324c47e010e3554c14e755a7e045453745bb.json | 23 + ...974f2ce577b78decd6b821096c9f2f252ae8b.json | 22 - ...741a2e193f18ffa11d08e8ca49cef5c3b850c.json | 16 + ...3c1293c4b72d0b52abeefd1e954617984a2d8.json | 24 + ...d40dfe1ffe51839884f3e9e9360d9e17b5afd.json | 36 + ...95bde7f479877018980cffe1c1e9d34aad59e.json | 29 + ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...7535d4d962c328a060f33c07191ac3e033e82.json | 16 + ...82734e4ca6cad2d849f75a8f8a249f83df83f.json | 23 + ...332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json | 30 + ...030390d9510e73e9b7df347b697d6dc7aefe6.json | 19 + ...9f909bf4babb29910514753cb822dde4e7ca9.json | 23 + ...7a20a5773113e290d69a016363543067baf14.json | 23 + ...c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json | 23 + ...07ba6333ad5feb4557a9341427e9e68608025.json | 46 + ...89730dd215f4e90d105f0e86025f0ac42f020.json | 15 + ...1cf520939fa4fa4b65371d11db7416ba0b14c.json | 23 + ...8d887920beaec97696a17da020a2adfb052f0.json | 17 + ...ae590d0ed2f4b7831956bb429a47c478f7c1f.json | 19 + ...0d503a92e44e291a76a53ef09ded619edfacb.json | 22 + ...0099ee5f46dafba8f323cf002e329ac69d1ac.json | 30 + ...0e74e14afde3c02000a88e696d7b6adcad0d6.json | 41 + ...7a3e43ef12b97d55353f86a9a368617efccca.json | 23 + ...4b94fd84c1bb64e9659f798df2888848894bd.json | 46 + ...358581f5716b3b58dc2e6b9b9282c50b1b66b.json | 15 + ...6f85238cbf189ccf18191493d6357e269d12d.json | 35 + backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- ...260617081932_datatable_migrations.down.sql | 1 + ...20260617081932_datatable_migrations.up.sql | 21 + backend/windmill-api-workspaces/Cargo.toml | 1 + .../src/datatable_migrations.rs | 1712 +++++++++++++++++ backend/windmill-api-workspaces/src/lib.rs | 1 + .../windmill-api-workspaces/src/workspaces.rs | 162 +- backend/windmill-api/openapi.yaml | 358 ++++ backend/windmill-api/src/workspaces_export.rs | 28 + backend/windmill-common/src/workspaces.rs | 6 + backend/windmill-git-sync/src/lib.rs | 123 +- cli/src/commands/datatable/datatable.ts | 71 + cli/src/commands/datatable_migrations.ts | 340 ++++ .../generate-metadata/generate-metadata.ts | 6 +- cli/src/commands/sync/sync.ts | 106 +- cli/src/commands/workspace/merge.ts | 32 + cli/src/guidance/skills.gen.ts | 26 +- cli/src/types.ts | 40 +- cli/test/datatable_migrations_unit.test.ts | 122 ++ cli/windmill-utils-internal/package.json | 2 +- cli/windmill-utils-internal/src/deploy.ts | 124 ++ frontend/package-lock.json | 171 +- frontend/package.json | 2 +- .../lib/components/CompareWorkspaces.svelte | 93 +- frontend/src/lib/components/DBManager.svelte | 10 +- .../lib/components/DBManagerContent.svelte | 21 +- .../src/lib/components/DBManagerDrawer.svelte | 28 +- .../src/lib/components/DBTableEditor.svelte | 13 +- .../lib/components/DatatableSchemaDiff.svelte | 273 ++- .../lib/components/DdlMigrationGuard.svelte | 159 ++ frontend/src/lib/components/Editor.svelte | 40 +- .../src/lib/components/ScriptEditor.svelte | 15 +- .../src/lib/components/SimpleEditor.svelte | 8 + frontend/src/lib/components/SqlRepl.svelte | 83 +- .../ConfirmationModal.svelte | 6 +- .../lib/components/common/table/Row.svelte | 1 + .../components/common/table/RowIcon.svelte | 3 + frontend/src/lib/components/dbOps.ts | 168 +- frontend/src/lib/components/sqlDdl.ts | 142 ++ .../DataTableMigrationsButton.svelte | 596 ++++++ .../DataTableSettings.svelte | 32 +- .../NewDataTableMigrationModal.svelte | 230 +++ .../datatableMigrationUtils.ts | 30 + frontend/src/lib/utils_deployable.ts | 2 + frontend/src/lib/utils_workspace_deploy.ts | 9 +- .../auto-generated/cli/cli-commands.md | 26 +- system_prompts/auto-generated/prompts.ts | 26 +- .../skills/cli-commands/SKILL.md | 26 +- system_prompts/generate.py | 86 +- 78 files changed, 5791 insertions(+), 459 deletions(-) create mode 100644 backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json create mode 100644 backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json delete mode 100644 backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json create mode 100644 backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json create mode 100644 backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json create mode 100644 backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json create mode 100644 backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json create mode 100644 backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json create mode 100644 backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json create mode 100644 backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json create mode 100644 backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json create mode 100644 backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json create mode 100644 backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json create mode 100644 backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json create mode 100644 backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json create mode 100644 backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json create mode 100644 backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json create mode 100644 backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json create mode 100644 backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json create mode 100644 backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json create mode 100644 backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json create mode 100644 backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json create mode 100644 backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json create mode 100644 backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json create mode 100644 backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json create mode 100644 backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json create mode 100644 backend/migrations/20260617081932_datatable_migrations.down.sql create mode 100644 backend/migrations/20260617081932_datatable_migrations.up.sql create mode 100644 backend/windmill-api-workspaces/src/datatable_migrations.rs create mode 100644 cli/src/commands/datatable_migrations.ts create mode 100644 cli/test/datatable_migrations_unit.test.ts create mode 100644 frontend/src/lib/components/DdlMigrationGuard.svelte create mode 100644 frontend/src/lib/components/sqlDdl.ts create mode 100644 frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts diff --git a/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json new file mode 100644 index 0000000000..13536a244e --- /dev/null +++ b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e" +} diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json b/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json new file mode 100644 index 0000000000..47bfec9e5c --- /dev/null +++ b/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb" +} diff --git a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json b/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json deleted file mode 100644 index d099d97bd3..0000000000 --- a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT ws.datatable->'datatables' AS datatable_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "datatable_name", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b" -} diff --git a/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json b/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json new file mode 100644 index 0000000000..2f1ad395b6 --- /dev/null +++ b/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c" +} diff --git a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json b/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json new file mode 100644 index 0000000000..3206ba4433 --- /dev/null +++ b/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8" +} diff --git a/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json b/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json new file mode 100644 index 0000000000..1e1c5f20c3 --- /dev/null +++ b/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd" +} diff --git a/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json b/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json new file mode 100644 index 0000000000..20ac675432 --- /dev/null +++ b/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 RETURNING timestamp, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json new file mode 100644 index 0000000000..12628305cc --- /dev/null +++ b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82" +} diff --git a/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json b/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json new file mode 100644 index 0000000000..14b085db2e --- /dev/null +++ b/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables'->$2 FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f" +} diff --git a/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json b/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json new file mode 100644 index 0000000000..3a1dc9da28 --- /dev/null +++ b/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0" +} diff --git a/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json new file mode 100644 index 0000000000..384686eb35 --- /dev/null +++ b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6" +} diff --git a/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json b/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json new file mode 100644 index 0000000000..f7ff5d527a --- /dev/null +++ b/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bool", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9" +} diff --git a/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json new file mode 100644 index 0000000000..eb94fe6ab5 --- /dev/null +++ b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14" +} diff --git a/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json b/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json new file mode 100644 index 0000000000..5ac3174af6 --- /dev/null +++ b/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'false'::jsonb) WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb" +} diff --git a/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json b/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json new file mode 100644 index 0000000000..c262f3d1f6 --- /dev/null +++ b/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 ORDER BY datatable, timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025" +} diff --git a/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json b/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json new file mode 100644 index 0000000000..2a90d12ad0 --- /dev/null +++ b/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down)\n SELECT $2, datatable, timestamp, name, code_up, code_down\n FROM datatable_migrations WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020" +} diff --git a/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json b/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json new file mode 100644 index 0000000000..f3cc23c5b8 --- /dev/null +++ b/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c" +} diff --git a/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json b/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json new file mode 100644 index 0000000000..b3861c0111 --- /dev/null +++ b/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, 'initial', $4, NULL)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0" +} diff --git a/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json b/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json new file mode 100644 index 0000000000..fa1396235f --- /dev/null +++ b/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f" +} diff --git a/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json b/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json new file mode 100644 index 0000000000..b089e466cf --- /dev/null +++ b/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb" +} diff --git a/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json b/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json new file mode 100644 index 0000000000..83c306f4d8 --- /dev/null +++ b/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2) AND timestamp = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac" +} diff --git a/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json new file mode 100644 index 0000000000..d6c6a5c02f --- /dev/null +++ b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6" +} diff --git a/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json b/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json new file mode 100644 index 0000000000..fbcf0743be --- /dev/null +++ b/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'true'::jsonb) WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca" +} diff --git a/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json b/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json new file mode 100644 index 0000000000..8a6d989157 --- /dev/null +++ b/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 ORDER BY datatable, timestamp", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd" +} diff --git a/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json b/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json new file mode 100644 index 0000000000..1efaaa32e8 --- /dev/null +++ b/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET labels = (\n SELECT array_agg(DISTINCT l)\n FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) l\n ) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b" +} diff --git a/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json b/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json new file mode 100644 index 0000000000..c630f5dd05 --- /dev/null +++ b/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b2acfc2e66..177509c02e 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14473,6 +14473,7 @@ dependencies = [ "sqlx", "strum", "tokio", + "tokio-postgres", "tracing", "uuid", "windmill-api-auth", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 087459205e..1b702a3a54 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0de2412ff0734b11e12ba378c9bcc373ff9ae800 +27672e37df5d9dfde94f19963d5ffcdf8dd5448c diff --git a/backend/migrations/20260617081932_datatable_migrations.down.sql b/backend/migrations/20260617081932_datatable_migrations.down.sql new file mode 100644 index 0000000000..26efc2bdcc --- /dev/null +++ b/backend/migrations/20260617081932_datatable_migrations.down.sql @@ -0,0 +1 @@ +DROP TABLE datatable_migrations; diff --git a/backend/migrations/20260617081932_datatable_migrations.up.sql b/backend/migrations/20260617081932_datatable_migrations.up.sql new file mode 100644 index 0000000000..72c7dc5ec6 --- /dev/null +++ b/backend/migrations/20260617081932_datatable_migrations.up.sql @@ -0,0 +1,21 @@ +-- SQL migrations defined per data table within a workspace. +-- `datatable` is the target data table name, `name` is the migration name +-- (e.g. add_index_to_customers), and `timestamp` is the migration version +-- (YYYYMMDDHHMMSS), recorded as `version` in the data table's `_wm_migrations` +-- table once applied. +CREATE TABLE datatable_migrations ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + datatable VARCHAR(255) NOT NULL, + timestamp BIGINT NOT NULL, + name VARCHAR(255) NOT NULL, + code_up TEXT NOT NULL, + code_down TEXT, + PRIMARY KEY (workspace_id, datatable, timestamp) +); + +-- No standalone index: the (workspace_id, datatable, timestamp) primary-key btree +-- already serves both `WHERE workspace_id = $1` and `WHERE workspace_id = $1 AND +-- datatable = $2` lookups via its leading columns. + +GRANT ALL ON datatable_migrations TO windmill_user; +GRANT ALL ON datatable_migrations TO windmill_admin; diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index c25bb8f42b..52a58aa1a3 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -47,6 +47,7 @@ serde_json.workspace = true sha2.workspace = true sqlx.workspace = true tokio.workspace = true +tokio-postgres.workspace = true tracing.workspace = true uuid.workspace = true strum.workspace = true diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs new file mode 100644 index 0000000000..fef9963933 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -0,0 +1,1712 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Data table SQL migrations: CRUD endpoints, run/rollback execution, opt-in +//! management, and the workspace-merge diff helper. Split out of `workspaces.rs` +//! to keep that file focused on core workspace configuration. + +use crate::workspaces::{pg_dump_database, ItemComparison}; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use sqlx::{Postgres, Transaction}; +use std::collections::{HashMap, HashSet}; + +use windmill_api_auth::{require_super_admin, ApiAuthed}; +use windmill_api_jobs::run_wait_result_internal; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::db::UserDB; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, DebouncingSettings}; +use windmill_common::scripts::ScriptLang; +use windmill_common::users::username_to_permissioned_as; +use windmill_common::worker::to_raw_value; +use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; +use windmill_common::{PgDatabase, DB}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_queue::{push, PushArgs, PushIsolationLevel}; + +pub(crate) fn routes() -> Router { + Router::new() + .route( + "/run_datatable_migrations/{datatable_name}", + post(run_datatable_migrations), + ) + .route( + "/rollback_datatable_migrations/{datatable_name}", + post(rollback_datatable_migrations), + ) + .route("/list_datatable_migrations", get(list_datatable_migrations)) + .route( + "/datatable_migrations_status/{datatable_name}", + get(datatable_migrations_status), + ) + .route( + "/enable_datatable_migrations/{datatable_name}", + post(enable_datatable_migrations), + ) + .route( + "/disable_datatable_migrations/{datatable_name}", + post(disable_datatable_migrations), + ) + .route( + "/create_datatable_migration/{datatable_name}", + post(create_datatable_migration), + ) + .route( + "/delete_datatable_migration/{datatable_name}/{timestamp}", + delete(delete_datatable_migration), + ) + .route( + "/upsert_datatable_migration/{datatable_name}", + post(upsert_datatable_migration), + ) + .route( + "/generate_initial_datatable_migration/{datatable_name}", + post(generate_initial_datatable_migration), + ) +} + +#[derive(Serialize)] +struct AppliedMigration { + version: i64, + name: String, +} + +#[derive(Serialize)] +struct RunDatatableMigrationsResult { + applied: Vec, +} + +#[derive(Deserialize)] +struct RunDatatableMigrationsQuery { + /// When set, only apply pending migrations up to and including this version. + up_to: Option, + /// When set, apply only this specific migration version (if not already + /// applied), ignoring any other pending migrations. Takes precedence over + /// `up_to`. + only: Option, +} + +/// Build the `database` argument for a migration job. Both resource-backed and +/// instance data tables pass a `datatable://` reference; the pg executor +/// resolves it to real credentials server-side at run time. It must never be +/// resolved here: the resolved instance credentials include a single +/// instance-wide Postgres password, and the job's `args` are readable by the — +/// possibly non-admin — user who ran the migration. +async fn datatable_database_arg( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + // Fail fast with a clear error if the data table doesn't exist. + sqlx::query_scalar!( + "SELECT ws.datatable->'datatables'->$2 FROM workspace_settings ws WHERE ws.workspace_id = $1", + w_id, + datatable_name, + ) + .fetch_one(db) + .await? + .ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?; + + Ok(to_raw_value(&format!("datatable://{datatable_name}"))) +} + +/// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as +/// the requesting user and labelled `datatable_migration` for traceability, then +/// wait for it. Errors if the job fails. +async fn run_datatable_migration_job( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + database_arg: &Box, + sql: &str, +) -> Result<()> { + let mut args = HashMap::new(); + args.insert("database".to_string(), database_arg.clone()); + let push_args = PushArgs { extra: None, args: &args }; + + let (uuid, mut tx) = push( + db, + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), + w_id, + JobPayload::Code(RawCode { + content: sql.to_string(), + path: Some("datatable_migration".to_string()), + hash: None, + language: ScriptLang::Postgresql, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + tag: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + modules: None, + }), + push_args, + authed.display_username(), + &authed.email, + username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + None, + ) + .await?; + + // Tag the job so migration runs are easy to find in the run history. + sqlx::query!( + "UPDATE v2_job SET labels = ( + SELECT array_agg(DISTINCT l) + FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) l + ) WHERE id = $1", + uuid, + &vec!["datatable_migration".to_string()], + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + let (result, success) = + run_wait_result_internal(db, uuid, w_id, None, false, &authed.username).await?; + if !success { + // On failure the job result is `{"error": {"name", "message", ...}}`; + // surface the executor's message (the Postgres error, e.g. `relation + // "foo" does not exist`) instead of the raw JSON envelope. + let detail = serde_json::from_str::(result.get()) + .ok() + .as_ref() + .and_then(|v| v.get("error")) + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()) + .map(str::to_string) + .unwrap_or_else(|| result.get().to_string()); + return Err(Error::internal_err(detail)); + } + Ok(()) +} + +/// Ensure the `_wm_migrations` bookkeeping table exists. Migration versions are +/// only unique per data table, but several data-table configs can point at one +/// physical database, so it is keyed by `(datatable, version)` — a version-only +/// key would let one data table's migration mark another's same-version +/// migration as already applied (and rollback could touch the wrong row). +async fn ensure_wm_migrations_schema(client: &tokio_postgres::Client) -> Result<()> { + client + .batch_execute( + "CREATE TABLE IF NOT EXISTS _wm_migrations (\ + datatable TEXT NOT NULL, \ + version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), \ + PRIMARY KEY (datatable, version))", + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to ensure _wm_migrations table: {}", e)) + })?; + Ok(()) +} + +/// Open a connection to a data table's own database and hold the session-level +/// advisory lock that serializes migration runs/rollbacks. The lock is released +/// when the returned client is dropped, so callers must keep it in scope for the +/// whole critical section. +/// +/// Runs, rollbacks *and* definition rewrites/deletes all take this lock: a run +/// snapshots a migration's `code_up` from `datatable_migrations` and only records +/// its version in `_wm_migrations` after the job succeeds, so an unserialized edit +/// could rewrite the definition in that window and leave `_wm_migrations` pointing +/// at SQL that was never applied. `_wm_migrations` is per-database, so a single +/// key is sufficient. +async fn lock_datatable_migration_runs( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + client + .batch_execute("SELECT pg_advisory_lock(hashtext('windmill_datatable_migrations')::int8)") + .await + .map_err(|e| Error::internal_err(format!("Failed to acquire migration lock: {}", e)))?; + Ok(client) +} + +/// Read the versions recorded as applied in a data table's `_wm_migrations`, +/// scoped to that data table, using an existing connection. An absent table +/// (`42P01`) means nothing has been migrated yet. +async fn read_applied_versions_on_client( + client: &tokio_postgres::Client, + datatable_name: &str, +) -> Result> { + match client + .query( + "SELECT version FROM _wm_migrations WHERE datatable = $1", + &[&datatable_name], + ) + .await + { + Ok(rows) => Ok(rows.iter().map(|row| row.get::<_, i64>(0)).collect()), + Err(e) if e.as_db_error().map(|d| d.code().code()) == Some("42P01") => Ok(HashSet::new()), + Err(e) => Err(Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + e + ))), + } +} + +/// Apply the workspace's pending data table migrations to a given data table. +/// Each migration runs as a normal Windmill `postgresql` job (permissioned as +/// the requester, labelled `datatable_migration`); applied versions are then +/// recorded in the data table's own `_wm_migrations` table, so only migrations +/// not recorded there are run, in ascending `timestamp` order. +async fn run_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + audit_log( + &db, + &authed, + "workspaces.run_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; + + // Take the run-serialization lock before snapshotting the migration + // definitions: a concurrent definition rewrite/delete takes the same lock, so + // the `code_up` we read here can't change between now and when we record its + // version below. The lock is held until `client` drops at return. + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name).await?; + + let migrations = sqlx::query!( + "SELECT timestamp, name, code_up FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + ensure_wm_migrations_schema(&client).await?; + + let applied_versions = read_applied_versions_on_client(&client, &datatable_name).await?; + + let mut applied = Vec::new(); + for m in migrations { + if let Some(only) = query.only { + // Run a single specific migration, skipping every other one. + if m.timestamp != only { + continue; + } + } else if query.up_to.is_some_and(|up_to| m.timestamp > up_to) { + // Migrations are ordered ascending, so once we pass `up_to` we're done. + break; + } + if applied_versions.contains(&m.timestamp) { + continue; + } + run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to apply migration {} ({}): {}", + m.timestamp, m.name, e + )) + })?; + // Record the migration as installed once its job has succeeded. + client + .execute( + "INSERT INTO _wm_migrations (datatable, version) VALUES ($1, $2) \ + ON CONFLICT (datatable, version) DO NOTHING", + &[&datatable_name, &m.timestamp], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to record migration: {}", e)))?; + applied.push(AppliedMigration { version: m.timestamp, name: m.name }); + } + + Ok(Json(RunDatatableMigrationsResult { applied })) +} + +#[derive(Serialize)] +struct RolledBackMigration { + version: i64, + name: String, +} + +#[derive(Serialize)] +struct RollbackDatatableMigrationsResult { + rolled_back: Vec, +} + +#[derive(Deserialize)] +struct RollbackDatatableMigrationsQuery { + /// When set, roll back this specific applied migration version instead of + /// the most recently applied one. + only: Option, +} + +/// Roll back a migration on a given data table: run its `code_down` as a normal +/// Windmill `postgresql` job (permissioned as the requester, labelled +/// `datatable_migration`) then drop its `_wm_migrations` row. Without `only` this +/// targets the most recently applied migration (one step); with `only` it +/// targets that specific applied version. +async fn rollback_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + audit_log( + &db, + &authed, + "workspaces.rollback_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + // The data table's `_wm_migrations` bookkeeping is read here and the version + // dropped after the job succeeds; the down SQL itself runs in the job. The + // lock is held (until `client` drops at return) so a concurrent run or + // definition rewrite can't interleave with this rollback. + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name).await?; + + ensure_wm_migrations_schema(&client).await?; + + // Resolve which applied version to roll back: a specific one when `only` is + // given (and actually applied), otherwise the most recently applied. Scoped + // to this data table so a shared physical database can't surface another + // data table's version. + let target = match query.only { + Some(only) => client + .query_opt( + "SELECT version FROM _wm_migrations WHERE datatable = $1 AND version = $2", + &[&datatable_name, &only], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + None => client + .query_opt( + "SELECT version FROM _wm_migrations WHERE datatable = $1 \ + ORDER BY version DESC LIMIT 1", + &[&datatable_name], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + }; + + let version: i64 = match target { + Some(row) => row.get::<_, i64>(0), + None => { + return Ok(Json(RollbackDatatableMigrationsResult { + rolled_back: vec![], + })) + } + }; + + let definition = sqlx::query!( + "SELECT name, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + version + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Cannot roll back migration {version}: its definition no longer exists" + )) + })?; + + let code_down = definition.code_down.ok_or_else(|| { + Error::BadRequest(format!( + "Cannot roll back migration {} ({}): it has no down migration", + version, definition.name + )) + })?; + + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; + run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to roll back migration {} ({}): {}", + version, definition.name, e + )) + })?; + + // Forget the version once its down job has succeeded. + client + .execute( + "DELETE FROM _wm_migrations WHERE datatable = $1 AND version = $2", + &[&datatable_name, &version], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to drop migration record: {}", e)))?; + + Ok(Json(RollbackDatatableMigrationsResult { + rolled_back: vec![RolledBackMigration { version, name: definition.name }], + })) +} + +#[derive(Serialize, Deserialize)] +pub struct DatatableMigration { + pub datatable: String, + pub timestamp: i64, + pub name: String, + pub code_up: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_down: Option, +} + +async fn list_datatable_migrations( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let migrations = sqlx::query_as!( + DatatableMigration, + "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 ORDER BY datatable, timestamp ASC", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(migrations)) +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum DatatableMigrationRunStatus { + /// Recorded in the data table's `_wm_migrations` table. + Ran, + /// Defined but not yet applied. + NotRun, + /// Applied status could not be determined (connection failure). + Unknown, +} + +#[derive(Serialize)] +struct DatatableMigrationWithStatus { + timestamp: i64, + name: String, + code_up: String, + #[serde(skip_serializing_if = "Option::is_none")] + code_down: Option, + status: DatatableMigrationRunStatus, +} + +#[derive(Serialize)] +struct DatatableMigrationsStatusResult { + /// Whether the migrations feature is opted in for this data table. + enabled: bool, + migrations: Vec, + /// Set when the applied status couldn't be read from the data table. + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +/// Whether the SQL-migrations feature is enabled for a data table. Honors the +/// explicit `migrations_enabled` flag; when unset (data tables predating the +/// feature) it is considered enabled only if migrations already exist. +async fn datatable_migrations_enabled(db: &DB, w_id: &str, datatable_name: &str) -> Result { + let flag: Option = sqlx::query_scalar!( + "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean \ + FROM workspace_settings ws WHERE ws.workspace_id = $1", + w_id, + datatable_name, + ) + .fetch_optional(db) + .await? + .flatten(); + + match flag { + Some(v) => Ok(v), + None => Ok(sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2)", + w_id, + datatable_name, + ) + .fetch_one(db) + .await? + .unwrap_or(false)), + } +} + +/// Reject the request when migrations are not enabled for the data table. +async fn ensure_datatable_migrations_enabled( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result<()> { + if !datatable_migrations_enabled(db, w_id, datatable_name).await? { + return Err(Error::BadRequest(format!( + "Migrations are not enabled for data table '{}'. Enable them first.", + datatable_name + ))); + } + Ok(()) +} + +/// Read the versions recorded in a data table's `_wm_migrations` table. A +/// missing table means nothing has been applied yet (empty set, not an error). +async fn read_applied_datatable_versions( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + + // Read-only status path: don't create the table here, and don't take the run + // lock — a stale-by-a-moment applied set is fine for display. + read_applied_versions_on_client(&client, datatable_name).await +} + +/// List a data table's migrations annotated with whether each has been applied. +async fn datatable_migrations_status( + _authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + if !enabled { + return Ok(Json(DatatableMigrationsStatusResult { + enabled: false, + migrations: vec![], + error: None, + })); + } + + let defs = sqlx::query!( + "SELECT timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + let (applied, error) = match read_applied_datatable_versions(&db, &w_id, &datatable_name).await + { + Ok(set) => (Some(set), None), + Err(e) => (None, Some(e.to_string())), + }; + + let migrations = defs + .into_iter() + .map(|m| { + let status = match &applied { + Some(set) if set.contains(&m.timestamp) => DatatableMigrationRunStatus::Ran, + Some(_) => DatatableMigrationRunStatus::NotRun, + None => DatatableMigrationRunStatus::Unknown, + }; + DatatableMigrationWithStatus { + timestamp: m.timestamp, + name: m.name, + code_up: m.code_up, + code_down: m.code_down, + status, + } + }) + .collect(); + + Ok(Json(DatatableMigrationsStatusResult { + enabled: true, + migrations, + error, + })) +} + +/// Only workspace admins and super admins may opt a data table in or out of +/// migrations. +async fn require_datatable_migrations_manager(db: &DB, authed: &ApiAuthed) -> Result<()> { + if authed.is_admin || require_super_admin(db, &authed.email).await.is_ok() { + Ok(()) + } else { + Err(Error::BadRequest( + "Only workspace admins and super admins can enable or disable data table migrations" + .to_string(), + )) + } +} + +async fn enable_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> Result { + require_datatable_migrations_manager(&db, &authed).await?; + + let updated = sqlx::query_scalar!( + "UPDATE workspace_settings \ + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'true'::jsonb) \ + WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) \ + RETURNING 1", + &w_id, + &datatable_name, + ) + .fetch_optional(&db) + .await?; + if updated.is_none() { + return Err(Error::NotFound(format!( + "data table {datatable_name} not found" + ))); + } + + audit_log( + &db, + &authed, + "workspaces.enable_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + Ok(format!( + "Enabled migrations for data table {datatable_name}" + )) +} + +/// Opt a data table out of migrations. Deletes ALL of its migration definitions. +async fn disable_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> Result { + require_datatable_migrations_manager(&db, &authed).await?; + + let mut tx = db.begin().await?; + + let updated = sqlx::query_scalar!( + "UPDATE workspace_settings \ + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'false'::jsonb) \ + WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) \ + RETURNING 1", + &w_id, + &datatable_name, + ) + .fetch_optional(&mut *tx) + .await?; + if updated.is_none() { + return Err(Error::NotFound(format!( + "data table {datatable_name} not found" + ))); + } + + // Capture the deleted definitions so each removal is tallied as a deployed + // object (like single-migration deletion), keeping workspace comparison and + // git-sync callbacks in sync when a fork opts back out of migrations. + let deleted = sqlx::query!( + "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 \ + RETURNING timestamp, name", + &w_id, + &datatable_name, + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.disable_datatable_migrations", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + for m in deleted { + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + m.timestamp, + &m.name, + ) + .await?; + } + + Ok(format!( + "Disabled migrations for data table {datatable_name} and deleted its migrations" + )) +} + +#[derive(Deserialize)] +pub struct CreateDatatableMigration { + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Migration names map onto on-disk file names and the `_wm_migrations` record, +/// so keep them to a safe path-segment charset (matches the CLI scaffold). +fn validate_migration_name(name: &str) -> Result<()> { + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(Error::BadRequest(format!( + "Invalid migration name '{name}': use only letters, digits, '_' and '-'" + ))); + } + Ok(()) +} + +/// The data table name becomes a directory segment in the sync export +/// (`migrations/datatable//...`); reject anything that could escape it. +pub(crate) fn validate_datatable_path_segment(datatable: &str) -> Result<()> { + if datatable.is_empty() + || datatable.contains('/') + || datatable.contains('\\') + || datatable.contains("..") + { + return Err(Error::BadRequest(format!( + "Invalid data table name '{datatable}': must not contain '/', '\\' or '..'" + ))); + } + Ok(()) +} + +/// Record a data table migration change as a deployed object so it is tallied +/// into `workspace_diff` and shows up as a `datatable_migration` item in the +/// workspace-merge diff. The diff path is `/_`, +/// matching `parse_datatable_migration_diff_path`. +async fn record_datatable_migration_deployment( + authed: &ApiAuthed, + db: &DB, + w_id: &str, + datatable: &str, + timestamp: i64, + name: &str, +) -> Result<()> { + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + w_id, + DeployedObject::DatatableMigration { path: format!("{datatable}/{timestamp}_{name}") }, + Some(format!( + "Data table migration {name} ({timestamp}) on {datatable}" + )), + false, + None, + ) + .await +} + +/// Allocate the next version for a data table and insert the migration +/// definition, in one transaction. A per-(workspace, data table) advisory lock +/// serializes concurrent version allocation so two creates can't read the same +/// `MAX(timestamp)` and collide on the `(workspace_id, datatable, timestamp)` +/// primary key. The version is the current UTC `YYYYMMDDHHMMSS`, bumped past any +/// existing version to stay unique and monotonically increasing. +async fn insert_datatable_migration_def( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + datatable: &str, + name: &str, + code_up: &str, + code_down: Option<&str>, +) -> Result { + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", + w_id, + datatable, + ) + .execute(&mut **tx) + .await?; + + let now_ts: i64 = Utc::now() + .format("%Y%m%d%H%M%S") + .to_string() + .parse() + .map_err(|e| Error::internal_err(format!("Failed to build migration version: {}", e)))?; + let max_existing: Option = sqlx::query_scalar!( + "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + w_id, + datatable, + ) + .fetch_one(&mut **tx) + .await?; + let timestamp = match max_existing { + Some(m) if m >= now_ts => m + 1, + _ => now_ts, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6)", + w_id, + datatable, + timestamp, + name, + code_up, + code_down, + ) + .execute(&mut **tx) + .await?; + + Ok(timestamp) +} + +/// Mark a version as already installed in a data table's `_wm_migrations` table +/// (ensuring the table exists first). +async fn mark_datatable_version_installed( + db: &DB, + pg_db: &PgDatabase, + datatable: &str, + version: i64, +) -> Result<()> { + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + ensure_wm_migrations_schema(&client).await?; + client + .execute( + "INSERT INTO _wm_migrations (datatable, version) VALUES ($1, $2) \ + ON CONFLICT (datatable, version) DO NOTHING", + &[&datatable, &version], + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to mark initial migration installed: {}", e)) + })?; + Ok(()) +} + +/// Create a single migration for a data table. The version is generated +/// server-side (current UTC `YYYYMMDDHHMMSS`), bumped past any existing version +/// so it stays unique and monotonically increasing. +async fn create_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> JsonResult { + validate_datatable_path_segment(&datatable_name)?; + validate_migration_name(&payload.name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + let mut tx = db.begin().await?; + let timestamp = insert_datatable_migration_def( + &mut tx, + &w_id, + &datatable_name, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; + tx.commit().await?; + + audit_log( + &db, + &authed, + "workspaces.create_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + &payload.name, + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: payload.name, + code_up: payload.code_up, + code_down: payload.code_down, + })) +} + +/// Delete a single migration definition from a data table. +async fn delete_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, +) -> Result { + // Hold the run-serialization lock across the applied-check and the delete: a + // run snapshots a migration's SQL before recording its version, so an + // unserialized delete could race it and leave `_wm_migrations` pointing at a + // definition that no longer exists (breaking rollback and hiding the applied + // version). Held until the handler returns. Fail closed if we can't verify. + let unreachable = |e| { + Error::internal_err(format!( + "Cannot verify whether migration {} on data table '{}' has already been applied \ + (its database is unreachable: {}). Refusing to delete it; retry once the database \ + is reachable.", + timestamp, datatable_name, e + )) + }; + let lock_client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) + .await + .map_err(unreachable)?; + let applied = read_applied_versions_on_client(&lock_client, &datatable_name) + .await + .map_err(unreachable)?; + if applied.contains(×tamp) { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' has already been applied and cannot be deleted. \ + Revert it first.", + timestamp, datatable_name + ))); + } + + let deleted_name = sqlx::query_scalar!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 \ + RETURNING name", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_optional(&db) + .await?; + // The definition is removed; runs may resume (audit/deploy metadata below + // don't need the lock). + drop(lock_client); + + audit_log( + &db, + &authed, + "workspaces.delete_datatable_migration", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + // Only tally a change if a migration was actually deleted. + if let Some(name) = deleted_name { + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + &name, + ) + .await?; + } + + Ok(format!( + "Deleted migration {} from {}", + timestamp, datatable_name + )) +} + +#[derive(Deserialize)] +pub struct UpsertDatatableMigration { + pub timestamp: i64, + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Insert or update a single migration at an explicit version. Used by +/// `wmill sync` to push a `migrations/datatable/
/_.up.sql` +/// (and `.down.sql`) file as the source of truth. +async fn upsert_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + validate_datatable_path_segment(&datatable_name)?; + validate_migration_name(&payload.name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + // Guard against silently rewriting a migration that has already run in the + // data table's database: its `_wm_migrations` record would no longer match + // its SQL, so a later `migrate up` would skip it and a rollback would run a + // `down` that doesn't correspond to what was applied. Only an actual change + // to an existing migration is guarded; unchanged re-pushes (e.g. + // `wmill sync push`) always proceed. + let existing = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + payload.timestamp, + ) + .fetch_optional(&db) + .await?; + // When modifying an existing definition, hold the run-serialization lock + // across the applied-check and the write below so an in-flight run can't + // record a version for the SQL we're about to overwrite. Held until the end + // of the handler (well past the write); a new/unchanged upsert needs no lock. + let _run_lock = match existing { + Some(existing) + if !(existing.name == payload.name + && existing.code_up == payload.code_up + && existing.code_down == payload.code_down) => + { + // Fail closed: if we can't lock/read the applied set (e.g. the + // data-table database is temporarily unreachable), refuse the change + // rather than risk overwriting a migration that has already run. + let unreachable = |e| { + Error::internal_err(format!( + "Cannot verify whether migration {} on data table '{}' has already been \ + applied (its database is unreachable: {}). Refusing to modify it; retry \ + once the database is reachable.", + payload.timestamp, datatable_name, e + )) + }; + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) + .await + .map_err(unreachable)?; + let applied = read_applied_versions_on_client(&client, &datatable_name) + .await + .map_err(unreachable)?; + if applied.contains(&payload.timestamp) { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' has already been applied and cannot be modified. \ + Revert it first, or add a new migration instead.", + payload.timestamp, datatable_name + ))); + } + Some(client) + } + _ => None, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ + SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + &w_id, + &datatable_name, + payload.timestamp, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .execute(&db) + .await?; + // The definition is written; runs may resume (audit/deploy metadata below + // don't need the lock). + drop(_run_lock); + + audit_log( + &db, + &authed, + "workspaces.upsert_datatable_migration", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + payload.timestamp, + &payload.name, + ) + .await?; + + Ok(format!( + "Upserted migration {} in {}", + payload.timestamp, datatable_name + )) +} + +/// Generate the first migration for a data table by snapshotting its current +/// schema with `pg_dump`. The migration is recorded as already installed (the +/// definition is written first, then its version is marked in the data table's +/// `_wm_migrations`, so it ends up considered applied and is never re-run) and +/// has no down migration. +async fn generate_initial_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + validate_datatable_path_segment(&datatable_name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + // The initial snapshot only makes sense on a data table with no migrations + // yet; reject otherwise so repeated calls don't pile up duplicate "initial" + // definitions (each at a distinct timestamp, each marked installed). + let has_migrations: bool = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2)", + &w_id, + &datatable_name, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if has_migrations { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' already has migrations; the initial migration can only be generated when there are none." + ))); + } + + let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + + // Snapshot the schema, excluding Windmill's own migration bookkeeping table. + let dump_file = pg_dump_database(&pg_db, true, &["_wm_migrations"]).await?; + let raw_dump = tokio::fs::read_to_string(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to read schema dump: {}", e)))?; + // pg_dump emits psql meta-commands (\restrict / \unrestrict) that aren't + // valid SQL; drop them so the migration body can run via a plain query. + let code_up: String = raw_dump + .lines() + .filter(|line| !line.trim_start().starts_with('\\')) + .collect::>() + .join("\n"); + + // Record the definition first, then mark it installed. If marking fails we + // delete the definition, so a failure leaves no phantom "initial" (rather + // than a `_wm_migrations` version with no definition that the UI can't + // clear). The narrow window where it briefly shows "not run" is benign: + // running it would just no-op/fail harmlessly against the existing schema. + let mut tx = db.begin().await?; + let timestamp = + insert_datatable_migration_def(&mut tx, &w_id, &datatable_name, "initial", &code_up, None) + .await?; + tx.commit().await?; + + if let Err(e) = mark_datatable_version_installed(&db, &pg_db, &datatable_name, timestamp).await + { + let _ = sqlx::query!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .execute(&db) + .await; + return Err(e); + } + + audit_log( + &db, + &authed, + "workspaces.generate_initial_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + "initial", + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: "initial".to_string(), + code_up, + code_down: None, + })) +} + +/// A datatable migration diff item has path `/_`. +/// Parse out the (datatable, timestamp) needed to look it up. +fn parse_datatable_migration_diff_path(path: &str) -> Option<(String, i64)> { + let (datatable, file) = path.split_once('/')?; + let ts_str: String = file.chars().take_while(|c| c.is_ascii_digit()).collect(); + let timestamp = ts_str.parse::().ok()?; + Some((datatable.to_string(), timestamp)) +} + +pub(crate) async fn compare_two_datatable_migration( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + let (datatable, timestamp) = match parse_datatable_migration_diff_path(path) { + Some(v) => v, + None => { + return Ok(ItemComparison { + has_changes: false, + exists_in_source: false, + exists_in_fork: false, + }) + } + }; + + let source = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + source_workspace_id, + datatable, + timestamp, + ) + .fetch_optional(db) + .await?; + let target = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + fork_workspace_id, + datatable, + timestamp, + ) + .fetch_optional(db) + .await?; + + let has_changes = match (&source, &target) { + (Some(s), Some(t)) => { + s.name != t.name || s.code_up != t.code_up || s.code_down != t.code_down + } + _ => source.is_some() || target.is_some(), + }; + + Ok(ItemComparison { + has_changes, + exists_in_source: source.is_some(), + exists_in_fork: target.is_some(), + }) +} + +#[derive(Deserialize, Debug)] +pub(crate) struct DatatableRename { + pub(crate) from: String, + pub(crate) to: String, +} + +async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?; + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} + +/// Tolerate a data table whose database has no `_wm_migrations` yet (42P01 = +/// undefined_table): it has never run a migration, so nothing to rename or forget. +fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> { + match e.as_db_error().map(|d| d.code().code()) { + Some("42P01") => Ok(()), + _ => Err(Error::internal_err(format!( + "Failed to update _wm_migrations: {}", + e + ))), + } +} + +/// Drop a data table's rows from its own database's `_wm_migrations`. +async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> { + let pg_db = resolve_datatable_pg(db, w_id, datatable).await?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + let _ = connection.await; + }); + client + .execute( + "DELETE FROM _wm_migrations WHERE datatable = $1", + &[&datatable], + ) + .await + .map(|_| ()) + .or_else(ignore_missing_wm_migrations) +} + +/// Relabel a data table's rows in its own database's `_wm_migrations`. `resolve_by` +/// names the config entry used to find the database (the old name, still present +/// pre-commit); `from`/`to` are the `datatable` column values to move between. +async fn remote_rename_datatable_migrations( + db: &DB, + w_id: &str, + resolve_by: &str, + from: &str, + to: &str, +) -> Result<()> { + let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + let _ = connection.await; + }); + client + .execute( + "UPDATE _wm_migrations SET datatable = $2 WHERE datatable = $1", + &[&from, &to], + ) + .await + .map(|_| ()) + .or_else(ignore_missing_wm_migrations) +} + +/// Keep migration bookkeeping in sync when data tables are renamed or deleted in +/// the workspace config: the control table `datatable_migrations` (this database, +/// in `tx`) and each data table's own `_wm_migrations` (its own database, keyed +/// by data table name — see [`ensure_wm_migrations_schema`]). +/// +/// Renames are applied in two phases through a temporary key so that a rename +/// chain or a swap (A->B, B->A) can't transiently collide on the +/// (datatable, ...) uniqueness mid-update. +/// +/// The `_wm_migrations` updates are best-effort: they run just before `tx` +/// commits (resolved via the pool, which still exposes the old names), and a +/// temporarily unreachable data-table database is logged rather than failing the +/// whole config edit. If one is missed, the next run re-applies its migrations +/// against the existing schema. +pub(crate) async fn cascade_datatable_migration_renames_and_deletes( + db: &DB, + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + renames: &[DatatableRename], + deleted_datatables: &[String], +) -> Result<()> { + if !deleted_datatables.is_empty() { + sqlx::query!( + "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", + w_id, + deleted_datatables + ) + .execute(&mut **tx) + .await?; + } + + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + sqlx::query!( + "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + w_id, + &r.from, + &tmp + ) + .execute(&mut **tx) + .await?; + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + sqlx::query!( + "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + w_id, + &tmp, + &r.to + ) + .execute(&mut **tx) + .await?; + } + + for name in deleted_datatables { + if let Err(e) = remote_forget_datatable_migrations(db, w_id, name).await { + tracing::warn!("Failed to clear _wm_migrations for deleted data table {name}: {e}"); + } + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + if let Err(e) = remote_rename_datatable_migrations(db, w_id, &r.from, &r.from, &tmp).await { + tracing::warn!( + "Failed to stage _wm_migrations rename {} -> {}: {e}", + r.from, + r.to + ); + } + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + if let Err(e) = remote_rename_datatable_migrations(db, w_id, &r.from, &tmp, &r.to).await { + tracing::warn!( + "Failed to finish _wm_migrations rename {} -> {}: {e}", + r.from, + r.to + ); + } + } + + Ok(()) +} + +/// Copy a workspace's migration definitions to another workspace, so a fork +/// inherits the same per-data-table migration history as its parent. +pub(crate) async fn clone_datatable_migrations( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) + SELECT $2, datatable, timestamp, name, code_up, code_down + FROM datatable_migrations WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_migration_name_accepts_safe_names() { + for name in ["initial", "add_index_to_customers", "fix-bug_2", "ABC123"] { + assert!( + validate_migration_name(name).is_ok(), + "{name} should be valid" + ); + } + } + + #[test] + fn validate_migration_name_rejects_unsafe_names() { + for name in [ + "", + "add index", + "a/b", + "a\\b", + "a..b", + "a.b", + "naïve", + "a/../b", + ] { + assert!( + validate_migration_name(name).is_err(), + "{name} should be rejected" + ); + } + } + + #[test] + fn validate_datatable_path_segment_accepts_and_rejects() { + for ok in ["mydt", "my-dt", "main", "a b"] { + assert!( + validate_datatable_path_segment(ok).is_ok(), + "{ok} should be ok" + ); + } + for bad in ["", "a/b", "a\\b", "..", "a..b", "../etc", "x/.."] { + assert!( + validate_datatable_path_segment(bad).is_err(), + "{bad} should be rejected" + ); + } + } + + #[test] + fn parse_datatable_migration_diff_path_roundtrips() { + assert_eq!( + parse_datatable_migration_diff_path("mydt/20260101000001_create_users.up.sql"), + Some(("mydt".to_string(), 20260101000001)) + ); + // up and down map to the same (datatable, timestamp) record. + assert_eq!( + parse_datatable_migration_diff_path("mydt/20260101000001_create_users.down.sql"), + Some(("mydt".to_string(), 20260101000001)) + ); + // datatable names may themselves be hyphenated. + assert_eq!( + parse_datatable_migration_diff_path("my-dt/42_x.up.sql"), + Some(("my-dt".to_string(), 42)) + ); + } + + #[test] + fn parse_datatable_migration_diff_path_rejects_malformed() { + // no slash → not a migration path + assert_eq!(parse_datatable_migration_diff_path("nofile"), None); + // filename not starting with digits → no timestamp + assert_eq!( + parse_datatable_migration_diff_path("mydt/create_users.up.sql"), + None + ); + // empty filename + assert_eq!(parse_datatable_migration_diff_path("mydt/"), None); + } + + async fn seed_migration(pool: &DB, w_id: &str, datatable: &str, timestamp: i64, name: &str) { + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) \ + VALUES ($1, $2, $3, $4, 'select 1;')", + ) + .bind(w_id) + .bind(datatable) + .bind(timestamp) + .bind(name) + .execute(pool) + .await + .unwrap(); + } + + async fn migration_keys(pool: &DB, w_id: &str) -> Vec<(String, String)> { + sqlx::query_as::<_, (String, String)>( + "SELECT datatable, name FROM datatable_migrations WHERE workspace_id = $1 \ + ORDER BY datatable, timestamp", + ) + .bind(w_id) + .fetch_all(pool) + .await + .unwrap() + } + + // The cascade keeps each data table's migrations attached to its name when a + // data table is renamed, drops them when it is deleted, and survives a swap + // (A->B, B->A) at a shared timestamp without a primary-key collision. + #[sqlx::test(migrations = "../migrations")] + async fn cascade_renames_and_deletes_datatable_migrations(pool: DB) { + let w_id = format!("dtmig{}", uuid::Uuid::new_v4().simple()); + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')") + .bind(&w_id) + .execute(&pool) + .await + .unwrap(); + + // rename a -> a2, delete d, and swap sa <-> sb (both at timestamp 5000) + seed_migration(&pool, &w_id, "a", 1, "a_mig").await; + seed_migration(&pool, &w_id, "d", 1, "d_mig").await; + seed_migration(&pool, &w_id, "sa", 5000, "sa_mig").await; + seed_migration(&pool, &w_id, "sb", 5000, "sb_mig").await; + + let mut tx = pool.begin().await.unwrap(); + cascade_datatable_migration_renames_and_deletes( + &pool, + &mut tx, + &w_id, + &[ + DatatableRename { from: "a".to_string(), to: "a2".to_string() }, + DatatableRename { from: "sa".to_string(), to: "sb".to_string() }, + DatatableRename { from: "sb".to_string(), to: "sa".to_string() }, + ], + &["d".to_string()], + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!( + migration_keys(&pool, &w_id).await, + vec![ + ("a2".to_string(), "a_mig".to_string()), + ("sa".to_string(), "sb_mig".to_string()), + ("sb".to_string(), "sa_mig".to_string()), + ] + ); + } + + // A fork inherits its parent's migration definitions unchanged. + #[sqlx::test(migrations = "../migrations")] + async fn clone_copies_datatable_migrations_to_target(pool: DB) { + let src = format!("src{}", uuid::Uuid::new_v4().simple()); + let dst = format!("dst{}", uuid::Uuid::new_v4().simple()); + for w in [&src, &dst] { + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')", + ) + .bind(w) + .execute(&pool) + .await + .unwrap(); + } + seed_migration(&pool, &src, "customers", 1, "create_customers").await; + seed_migration(&pool, &src, "customers", 2, "add_index").await; + seed_migration(&pool, &src, "orders", 3, "create_orders").await; + + let mut tx = pool.begin().await.unwrap(); + clone_datatable_migrations(&mut tx, &src, &dst) + .await + .unwrap(); + tx.commit().await.unwrap(); + + // the target ends up with an identical set, and the source is untouched. + let expected = vec![ + ("customers".to_string(), "create_customers".to_string()), + ("customers".to_string(), "add_index".to_string()), + ("orders".to_string(), "create_orders".to_string()), + ]; + assert_eq!(migration_keys(&pool, &dst).await, expected); + assert_eq!(migration_keys(&pool, &src).await, expected); + } +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index b3e9853de9..c2b62d450b 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,3 +1,4 @@ +pub mod datatable_migrations; pub mod deployment_requests; pub mod workspaces; pub mod workspaces_extra; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8d53562635..0b5b600bf1 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -132,6 +132,7 @@ pub fn workspaced_service() -> Router { get(get_datatable_table_schema), ) .route("/edit_datatable_config", post(edit_datatable_config)) + .merge(crate::datatable_migrations::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) @@ -430,6 +431,12 @@ pub struct DucklakeSettings { #[derive(Deserialize, Debug)] struct EditDataTableConfig { settings: DataTableSettings, + // Data table renames (old -> new) and deletions, tracked client-side by a + // stable id, so we can cascade or drop each data table's migrations. + #[serde(default)] + renames: Vec, + #[serde(default)] + deleted_datatables: Vec, } #[derive(Deserialize, Serialize, Debug)] @@ -1973,8 +1980,8 @@ pub(crate) async fn resolve_pg_source_checked( } /// A temporary file for pg_dump output that is automatically deleted when dropped. -struct DumpFile { - path: std::path::PathBuf, +pub(crate) struct DumpFile { + pub(crate) path: std::path::PathBuf, } impl DumpFile { @@ -2021,7 +2028,11 @@ impl Drop for DumpFile { /// Run pg_dump against a PgDatabase, writing output to a temp file on disk. /// Returns a DumpFile handle; the file is deleted when the handle is dropped. -async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result { +pub(crate) async fn pg_dump_database( + pg_db: &PgDatabase, + schema_only: bool, + exclude_tables: &[&str], +) -> Result { let dump_file = DumpFile::new()?; let host = &pg_db.host; @@ -2034,6 +2045,9 @@ async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result, ) -> Result { let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let dump_file = pg_dump_database(&pg, true).await?; + let dump_file = pg_dump_database(&pg, true, &[]).await?; tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) @@ -2416,13 +2430,57 @@ async fn edit_datatable_config( Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, email, .. }: ApiAuthed, - Json(new_config): Json, + Json(mut new_config): Json, ) -> Result { require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &email).await.is_ok(); let mut tx = db.begin().await?; + let old_datatables: HashMap = serde_json::from_value( + sqlx::query_scalar!( + "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null), + ) + .unwrap_or_default(); + + // Validate every persisted data table name and rename segment before + // touching anything, since they become directory segments in migration + // storage/export keys (`migrations/datatable//...`). + for name in new_config.settings.datatables.keys() { + crate::datatable_migrations::validate_datatable_path_segment(name)?; + } + for r in &new_config.renames { + crate::datatable_migrations::validate_datatable_path_segment(&r.from)?; + crate::datatable_migrations::validate_datatable_path_segment(&r.to)?; + } + + // Map new name -> old name so a renamed data table inherits the previous + // flag instead of being treated as brand new. + let rename_src: HashMap<&str, &str> = new_config + .renames + .iter() + .map(|r| (r.to.as_str(), r.from.as_str())) + .collect(); + + // Migrations opt-in is owned by the enable/disable endpoints, not this config + // form: preserve each existing data table's flag, and default brand-new data + // tables to enabled. + for (name, dt) in new_config.settings.datatables.iter_mut() { + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + dt.migrations_enabled = match old_datatables.get(lookup) { + Some(old) => old.migrations_enabled, + None => Some(true), + }; + } + let args_for_audit = format!("{:?}", new_config.settings); audit_log( &mut *tx, @@ -2437,19 +2495,6 @@ async fn edit_datatable_config( // Check that non-superadmins are not abusing Instance databases if !is_superadmin { - let old_datatables = sqlx::query_scalar!( - r#" - SELECT ws.datatable->'datatables' AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(serde_json::Value::Null); - let old_datatables: HashMap = - serde_json::from_value(old_datatables).unwrap_or_default(); for (name, dt) in new_config.settings.datatables.iter() { if dt.database.resource_type == DataTableCatalogResourceType::Instance { let old_dt = old_datatables.get(name); @@ -2478,6 +2523,15 @@ async fn edit_datatable_config( .execute(&mut *tx) .await?; + crate::datatable_migrations::cascade_datatable_migration_renames_and_deletes( + &db, + &mut tx, + &w_id, + &new_config.renames, + &new_config.deleted_datatables, + ) + .await?; + tx.commit().await?; Ok(format!("Edit datatable config for workspace {}", &w_id)) @@ -4068,6 +4122,15 @@ async fn clone_workspace_data( // Clone workspace settings (merge with existing basic settings) update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; + // Clone data table migration definitions (the settings above carry the data + // table config; this carries their migration history). + crate::datatable_migrations::clone_datatable_migrations( + tx, + source_workspace_id, + target_workspace_id, + ) + .await?; + // Clone workspace environment variables clone_workspace_env(tx, source_workspace_id, target_workspace_id).await?; @@ -7418,6 +7481,7 @@ pub struct CompareSummary { pub folders_changed: usize, pub schedules_changed: usize, pub triggers_changed: usize, + pub datatable_migrations_changed: usize, pub conflicts: usize, // Items that are both ahead and behind } @@ -7609,6 +7673,15 @@ async fn compare_workspaces( compare_two_folders(&db, &source_workspace_id, &fork_workspace_id, &item.path) .await?, ), + "datatable_migration" => Some( + crate::datatable_migrations::compare_two_datatable_migration( + &db, + &source_workspace_id, + &fork_workspace_id, + &item.path, + ) + .await?, + ), // Triggers and schedules are diffed against a hardcoded ignore list // (mode/enabled/server_id/last_server_ping/edited_at/by/error/extra_perms/permissioned_as/email) // so that fork-clones — which differ from the parent only in the runtime @@ -7731,6 +7804,10 @@ async fn compare_workspaces( .iter() .filter(|s| s.kind.ends_with("_trigger")) .count(), + datatable_migrations_changed: visible_diffs + .iter() + .filter(|s| s.kind == "datatable_migration") + .count(), conflicts: visible_diffs .iter() .filter(|s| s.ahead > 0 && s.behind > 0) @@ -8036,6 +8113,45 @@ async fn query_visible_items<'c>( .fetch_all(&mut **tx) .await? } + "datatable_migration" => { + // Match by (datatable, timestamp), not the full path: a migration + // keeps its identity across a rename, so the candidate path's + // `name` segment can differ from the stored one. Parse each + // `/_` candidate, probe existence by + // (datatable, timestamp), and return the *original* candidate path + // so the visibility set stays keyed by the diff's path. + let parsed: Vec<(String, i64, String)> = paths_vec + .iter() + .filter_map(|p| { + let (dt, rest) = p.split_once('/')?; + let ts = rest.split_once('_')?.0.parse::().ok()?; + Some((dt.to_string(), ts, p.clone())) + }) + .collect(); + if parsed.is_empty() { + vec![] + } else { + let dts: Vec = parsed.iter().map(|(d, _, _)| d.clone()).collect(); + let tss: Vec = parsed.iter().map(|(_, t, _)| *t).collect(); + let existing: HashSet<(String, i64)> = sqlx::query!( + "SELECT datatable, timestamp FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = ANY($2) AND timestamp = ANY($3)", + workspace_id, + &dts, + &tss, + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|r| (r.datatable, r.timestamp)) + .collect(); + parsed + .into_iter() + .filter(|(d, t, _)| existing.contains(&(d.clone(), *t))) + .map(|(_, _, p)| p) + .collect() + } + } k if TRIGGER_OR_SCHEDULE_TABLES.contains(&k) => { // SAFETY: `kind` comes from a hardcoded allowlist // TRIGGER_OR_SCHEDULE_TABLES, not user input. @@ -8103,10 +8219,10 @@ async fn existing_runnables( } #[derive(Debug)] -struct ItemComparison { - has_changes: bool, - exists_in_source: bool, - exists_in_fork: bool, +pub(crate) struct ItemComparison { + pub(crate) has_changes: bool, + pub(crate) exists_in_source: bool, + pub(crate) exists_in_fork: bool, } async fn compare_two_scripts( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 323de509bd..177b22662d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4631,6 +4631,22 @@ paths: properties: settings: $ref: "#/components/schemas/DataTableSettings" + renames: + description: data tables renamed in this save, so their migrations cascade + type: array + items: + type: object + required: [from, to] + properties: + from: + type: string + to: + type: string + deleted_datatables: + description: data tables removed in this save, so their migrations are deleted + type: array + items: + type: string responses: "200": description: status @@ -4638,6 +4654,307 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}: + post: + summary: run pending datatable migrations against a datatable + operationId: runDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: up_to + in: query + required: false + description: only apply pending migrations up to and including this version + schema: + type: integer + format: int64 + - name: only + in: query + required: false + description: apply only this specific migration version, ignoring others + schema: + type: integer + format: int64 + responses: + "200": + description: applied migrations + content: + application/json: + schema: + type: object + required: [applied] + properties: + applied: + type: array + items: + type: object + required: [version, name] + properties: + version: + type: integer + format: int64 + name: + type: string + + /w/{workspace}/workspaces/rollback_datatable_migrations/{datatable_name}: + post: + summary: roll back the most recently applied migration on a datatable + operationId: rollbackDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: only + in: query + required: false + description: roll back this specific applied migration version instead of the latest + schema: + type: integer + format: int64 + responses: + "200": + description: rolled back migrations + content: + application/json: + schema: + type: object + required: [rolled_back] + properties: + rolled_back: + type: array + items: + type: object + required: [version, name] + properties: + version: + type: integer + format: int64 + name: + type: string + + /w/{workspace}/workspaces/list_datatable_migrations: + get: + summary: list datatable migrations for a workspace + operationId: listDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: datatable migrations + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/datatable_migrations_status/{datatable_name}: + get: + summary: list a datatable's migrations with their applied status + operationId: getDatatableMigrationsStatus + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: migrations with status + content: + application/json: + schema: + type: object + required: [enabled, migrations] + properties: + enabled: + type: boolean + migrations: + type: array + items: + $ref: "#/components/schemas/DatatableMigrationWithStatus" + error: + type: string + + /w/{workspace}/workspaces/enable_datatable_migrations/{datatable_name}: + post: + summary: opt a datatable in to migrations (admins / super admins only) + operationId: enableDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/disable_datatable_migrations/{datatable_name}: + post: + summary: opt a datatable out of migrations, deleting all of them (admins / super admins only) + operationId: disableDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/create_datatable_migration/{datatable_name}: + post: + summary: create a single datatable migration (version generated server-side) + operationId: createDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name, code_up] + properties: + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/delete_datatable_migration/{datatable_name}/{timestamp}: + delete: + summary: delete a single datatable migration definition + operationId: deleteDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: timestamp + in: path + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/upsert_datatable_migration/{datatable_name}: + post: + summary: insert or update a single datatable migration at an explicit version + operationId: upsertDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [timestamp, name, code_up] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/generate_initial_datatable_migration/{datatable_name}: + post: + summary: snapshot the current schema as an already-installed initial migration + operationId: generateInitialDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + /w/{workspace}/workspaces/create_pg_database: post: summary: create a new PostgreSQL database for a datatable @@ -29067,6 +29384,9 @@ components: type: string required: - resource_type + migrations_enabled: + type: boolean + description: Whether the SQL migrations feature is opted in for this data table forked_from: type: object description: Fork origin info with schema snapshot @@ -29075,6 +29395,40 @@ components: type: object description: Schema snapshot at fork time additionalProperties: true + DatatableMigration: + type: object + required: [datatable, timestamp, name, code_up] + properties: + datatable: + type: string + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + DatatableMigrationWithStatus: + type: object + required: [timestamp, name, code_up, status] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + status: + type: string + enum: + - ran + - not_run + - unknown DataTableSchema: type: object required: [datatable_name, schemas] @@ -29786,6 +30140,7 @@ components: - folders_changed - schedules_changed - triggers_changed + - datatable_migrations_changed - conflicts properties: total_diffs: @@ -29824,6 +30179,9 @@ components: triggers_changed: type: integer description: Number of triggers with differences (sum across all trigger kinds) + datatable_migrations_changed: + type: integer + description: Number of data table migrations with differences conflicts: type: integer description: Number of items that are both ahead and behind (conflicts) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index fa112ef4e1..839057b4b9 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1546,6 +1546,34 @@ pub(crate) async fn tarball_workspace( .await?; } + { + // Data table migrations live in the `datatable_migrations` table; surface + // them in the export as `migrations/datatable//_` + // .up.sql (and .down.sql when present) so `wmill sync` treats them like any + // other workspace item. + let migrations = sqlx::query!( + "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 ORDER BY datatable, timestamp", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + for m in migrations { + let base = format!( + "migrations/datatable/{}/{}_{}", + m.datatable, m.timestamp, m.name + ); + archive + .write_to_archive(&m.code_up, &format!("{base}.up.sql")) + .await?; + if let Some(code_down) = m.code_down { + archive + .write_to_archive(&code_down, &format!("{base}.down.sql")) + .await?; + } + } + } + archive.finish().await?; let file = tokio::fs::File::open(&file_path).await?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 3b5220d46f..a8c6e26f0e 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -164,6 +164,7 @@ pub enum ObjectType { Settings, Key, WorkspaceDependencies, + DatatableMigration, } pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28719/sync-script-to-git-repo-windmill"; @@ -771,6 +772,11 @@ pub struct DataTable { pub database: DataTableDatabase, #[serde(default, skip_serializing_if = "Option::is_none")] pub forked_from: Option, + /// Whether the SQL-migrations feature is opted in for this data table. + /// Absent on data tables created before the feature: treated as enabled only + /// when migrations already exist (see `datatable_migrations_enabled`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub migrations_enabled: Option, } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index b10f62e7a5..6c1b9bf89b 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -24,30 +24,102 @@ pub use git_sync_oss::{ #[derive(Clone, Debug)] pub enum DeployedObject { - Script { hash: ScriptHash, path: String, parent_path: Option }, - Flow { path: String, parent_path: Option, version: i64 }, - App { path: String, version: i64, parent_path: Option }, - RawApp { path: String, version: i64, parent_path: Option }, - Folder { path: String }, - Resource { path: String, parent_path: Option }, - Variable { path: String, parent_path: Option }, - Schedule { path: String }, - ResourceType { path: String }, - User { email: String }, - Group { name: String }, - HttpTrigger { path: String, parent_path: Option }, - WebsocketTrigger { path: String, parent_path: Option }, - KafkaTrigger { path: String, parent_path: Option }, - NatsTrigger { path: String, parent_path: Option }, - PostgresTrigger { path: String, parent_path: Option }, - MqttTrigger { path: String, parent_path: Option }, - SqsTrigger { path: String, parent_path: Option }, - GcpTrigger { path: String, parent_path: Option }, - AzureTrigger { path: String, parent_path: Option }, - EmailTrigger { path: String, parent_path: Option }, - Settings { setting_type: String }, - Key { key_type: String }, - WorkspaceDependencies { path: String }, + Script { + hash: ScriptHash, + path: String, + parent_path: Option, + }, + Flow { + path: String, + parent_path: Option, + version: i64, + }, + App { + path: String, + version: i64, + parent_path: Option, + }, + RawApp { + path: String, + version: i64, + parent_path: Option, + }, + Folder { + path: String, + }, + Resource { + path: String, + parent_path: Option, + }, + Variable { + path: String, + parent_path: Option, + }, + Schedule { + path: String, + }, + ResourceType { + path: String, + }, + User { + email: String, + }, + Group { + name: String, + }, + HttpTrigger { + path: String, + parent_path: Option, + }, + WebsocketTrigger { + path: String, + parent_path: Option, + }, + KafkaTrigger { + path: String, + parent_path: Option, + }, + NatsTrigger { + path: String, + parent_path: Option, + }, + PostgresTrigger { + path: String, + parent_path: Option, + }, + MqttTrigger { + path: String, + parent_path: Option, + }, + SqsTrigger { + path: String, + parent_path: Option, + }, + GcpTrigger { + path: String, + parent_path: Option, + }, + AzureTrigger { + path: String, + parent_path: Option, + }, + EmailTrigger { + path: String, + parent_path: Option, + }, + Settings { + setting_type: String, + }, + Key { + key_type: String, + }, + WorkspaceDependencies { + path: String, + }, + /// A single data table migration, identified by `/_`. + DatatableMigration { + path: String, + }, } impl DeployedObject { @@ -77,6 +149,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => "settings.yaml".to_string(), DeployedObject::Key { .. } => "encryption_key.yaml".to_string(), DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(), + DeployedObject::DatatableMigration { path } => path.to_owned(), } } @@ -118,6 +191,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => None, DeployedObject::Key { .. } => None, DeployedObject::WorkspaceDependencies { .. } => None, + DeployedObject::DatatableMigration { .. } => None, } } @@ -147,6 +221,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => "settings", DeployedObject::Key { .. } => "key", DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies", + DeployedObject::DatatableMigration { .. } => "datatable_migration", } .to_string() } diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts index 1291d4cc13..6609d08c50 100644 --- a/cli/src/commands/datatable/datatable.ts +++ b/cli/src/commands/datatable/datatable.ts @@ -9,6 +9,13 @@ import { GlobalOptions } from "../../types.ts"; import { runCatalogQuery } from "../../utils/catalog.ts"; import { psql as psqlDatatable } from "./psql.ts"; import { serve as serveDatatable } from "./serve.ts"; +import { + createMigration, + pushLocalMigrations, + rollbackMigrations, + runMigrations, + validateLocalMigrations, +} from "../datatable_migrations.ts"; const DEFAULT_DATATABLE_NAME = "main"; @@ -41,6 +48,69 @@ async function run( await runCatalogQuery(opts, "datatable", name, sql); } +function migrateNew( + opts: GlobalOptions & { datatable?: string }, + name: string, +) { + createMigration(opts.datatable ?? DEFAULT_DATATABLE_NAME, name); +} + +async function migrateUp(opts: GlobalOptions & { datatable?: string }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dt = opts.datatable ?? DEFAULT_DATATABLE_NAME; + // Reject malformed local migrations (duplicate timestamps, orphan downs) before + // pushing — the same check `wmill sync push` runs — so a duplicate timestamp + // can't silently overwrite one migration on upsert. + const errors = validateLocalMigrations(new Set([dt])); + if (errors.length > 0) { + log.error( + "Invalid datatable migrations, aborting:\n" + + errors.map((e) => ` - ${e}`).join("\n"), + ); + process.exit(1); + } + // Push any locally-created/edited migration files first (without running + // them), so `migrate up` works even before a `wmill sync push`. + await pushLocalMigrations(workspace.workspaceId, dt); + await runMigrations(workspace.workspaceId, dt); +} + +async function migrateDown(opts: GlobalOptions & { datatable?: string }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dt = opts.datatable ?? DEFAULT_DATATABLE_NAME; + await rollbackMigrations(workspace.workspaceId, dt); +} + +const migrateCommand = new Command() + .description("manage datatable migrations") + .command("new", "scaffold a new migration (.up.sql / .down.sql files)") + .arguments("") + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateNew as any) + .command( + "up", + "apply all pending migrations to the main datatable (or one via --datatable)", + ) + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateUp as any) + .command( + "down", + "roll back the most recent migration on the main datatable (or one via --datatable)", + ) + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateDown as any); + async function create( opts: GlobalOptions & { resource?: string; force?: boolean }, name?: string, @@ -124,6 +194,7 @@ const command = new Command() "Output only the final result as JSON. Useful for scripting.", ) .action(run as any) + .command("migrate", migrateCommand) .command( "create", "register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://", diff --git a/cli/src/commands/datatable_migrations.ts b/cli/src/commands/datatable_migrations.ts new file mode 100644 index 0000000000..4fcaffd4fa --- /dev/null +++ b/cli/src/commands/datatable_migrations.ts @@ -0,0 +1,340 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as log from "../core/log.ts"; +import { colors } from "@cliffy/ansi/colors"; +import * as wmill from "../../gen/services.gen.ts"; +import { readTextFile } from "../utils/utils.ts"; +import { Confirm } from "@cliffy/prompt/confirm"; + +// Migrations live under /migrations/datatable//, one folder per +// target data table, as `_.up.sql` (and optional `.down.sql`). +// They are synced as ordinary workspace files (see the workspace tarball export +// and the `datatable_migration` handling in sync.ts); this module only holds the +// `wmill datatable migrate` command helpers and the per-file push primitive. +const MIGRATIONS_DIR = path.join("migrations", "datatable"); + +// Migration names map directly onto file names and the DB `name` column. +const MIGRATION_NAME_RE = /^[a-zA-Z0-9_-]+$/; + +/** Current UTC time as a YYYYMMDDHHMMSS migration version. */ +function migrationTimestamp(): string { + const d = new Date(); + const p = (n: number) => String(n).padStart(2, "0"); + return ( + `${d.getUTCFullYear()}${p(d.getUTCMonth() + 1)}${p(d.getUTCDate())}` + + `${p(d.getUTCHours())}${p(d.getUTCMinutes())}${p(d.getUTCSeconds())}` + ); +} + +/** + * A migration version unique within a data table folder: the current UTC + * timestamp bumped past any existing version, so two migrations scaffolded in + * the same second don't collide on the `(datatable, timestamp)` identity used to + * upsert them. + */ +function nextMigrationTimestamp(dir: string): string { + const now = Number(migrationTimestamp()); + let max = 0; + if (fs.existsSync(dir)) { + for (const file of fs.readdirSync(dir)) { + const m = file.match(/^(\d+)_.*\.(up|down)\.sql$/); + if (m) max = Math.max(max, Number(m[1])); + } + } + return String(max >= now ? max + 1 : now); +} + +/** + * Scaffold a new migration under migrations/datatable// as empty + * `_.up.sql` and `.down.sql` files. Purely local — no network. + */ +export function createMigration(datatable: string, name: string): void { + if (!MIGRATION_NAME_RE.test(name)) { + throw new Error( + `Invalid migration name '${name}': use only letters, digits, '_' and '-'`, + ); + } + const dir = path.join(process.cwd(), MIGRATIONS_DIR, datatable); + fs.mkdirSync(dir, { recursive: true }); + + const timestamp = nextMigrationTimestamp(dir); + const base = `${timestamp}_${name}`; + const up = path.join(dir, `${base}.up.sql`); + const down = path.join(dir, `${base}.down.sql`); + // Frame the body in an explicit transaction so it applies atomically, matching + // the template the UI's "New migration" modal seeds. + const template = (direction: string) => + `-- ${direction} migration: ${name}\nBEGIN;\n\n-- Add your migration here\n\nEND;\n`; + fs.writeFileSync(up, template("up"), "utf-8"); + fs.writeFileSync(down, template("down"), "utf-8"); + + log.info( + colors.green(`Created migration ${base} in ${MIGRATIONS_DIR}/${datatable}/`), + ); + for (const f of [up, down]) { + log.info(colors.gray(` ${path.relative(process.cwd(), f)}`)); + } +} + +/** + * Apply the workspace's pending migrations to a data table (forwards migrations + * recorded in `_wm_migrations`). Mirrors `wmill datatable migrate up`. + */ +export async function runMigrations( + workspace: string, + datatableName: string, +): Promise { + const result = await wmill.runDatatableMigrations({ + workspace, + datatableName, + }); + const applied = result.applied ?? []; + if (applied.length === 0) { + log.info(colors.gray(`No pending migrations to run on '${datatableName}'`)); + return; + } + log.info( + colors.green(`Applied ${applied.length} migration(s) to '${datatableName}':`), + ); + for (const m of applied) { + log.info(colors.gray(` ${m.version} ${m.name}`)); + } +} + +/** + * Roll back the most recently applied migration on a data table (one step). + * Mirrors `wmill datatable migrate down`. + */ +export async function rollbackMigrations( + workspace: string, + datatableName: string, +): Promise { + const result = await wmill.rollbackDatatableMigrations({ + workspace, + datatableName, + }); + const rolledBack = result.rolled_back ?? []; + if (rolledBack.length === 0) { + log.info( + colors.gray(`No applied migrations to roll back on '${datatableName}'`), + ); + return; + } + for (const m of rolledBack) { + log.info( + colors.green(`Rolled back migration ${m.version} ${m.name} on '${datatableName}'`), + ); + } +} + +/** + * Validate the on-disk migration files for the given data tables (or all of + * them when `datatables` is omitted). Returns a list of human-readable problems; + * an empty list means the migrations are well-formed. Two states are invalid: + * - two up (or two down) files sharing the same timestamp, which collide on the + * `(datatable, timestamp)` identity used to upsert; and + * - a `.down.sql` with no matching `.up.sql` (an up file is mandatory). + */ +export function validateLocalMigrations(datatables?: Set): string[] { + const errors: string[] = []; + const root = path.join(process.cwd(), MIGRATIONS_DIR); + if (!fs.existsSync(root)) return errors; + + for (const datatable of fs.readdirSync(root)) { + if (datatables && !datatables.has(datatable)) continue; + const dtDir = path.join(root, datatable); + if (!fs.statSync(dtDir).isDirectory()) continue; + + const upNamesByTs = new Map(); + const downNamesByTs = new Map(); + const upBases = new Set(); + const downBases: { ts: number; name: string }[] = []; + + for (const file of fs.readdirSync(dtDir)) { + const m = file.match(/^(\d+)_(.*)\.(up|down)\.sql$/); + if (!m) continue; + const ts = Number(m[1]); + const name = m[2]; + if (m[3] === "up") { + (upNamesByTs.get(ts) ?? upNamesByTs.set(ts, []).get(ts)!).push(name); + upBases.add(`${ts}_${name}`); + } else { + (downNamesByTs.get(ts) ?? downNamesByTs.set(ts, []).get(ts)!).push(name); + downBases.push({ ts, name }); + } + } + + for (const [ts, names] of upNamesByTs) { + if (names.length > 1) { + errors.push( + `${datatable}: ${names.length} up migrations share timestamp ${ts} (${names.join(", ")})`, + ); + } + } + for (const [ts, names] of downNamesByTs) { + if (names.length > 1) { + errors.push( + `${datatable}: ${names.length} down migrations share timestamp ${ts} (${names.join(", ")})`, + ); + } + } + for (const d of downBases) { + if (!upBases.has(`${d.ts}_${d.name}`)) { + errors.push( + `${datatable}: ${d.ts}_${d.name}.down.sql has no matching ${d.ts}_${d.name}.up.sql`, + ); + } + } + } + + return errors; +} + +/** + * Sync a single migration to the workspace based on the current on-disk state of + * its `/_.up.sql` file: upsert it when the up file + * exists, otherwise delete it. Called by `wmill sync push` for each changed + * `datatable_migration` file. + */ +export async function pushMigrationFromDisk( + workspace: string, + m: { datatable: string; timestamp: number }, +): Promise { + const dir = path.join(process.cwd(), MIGRATIONS_DIR, m.datatable); + // Find the up file for this timestamp regardless of its name segment. A rename + // (`123_old.up.sql` -> `123_new.up.sql`) keeps the (datatable, timestamp) + // identity but changes the name; the diff sorter may process the deleted old + // path before the added new one, so keying off the passed name would delete + // the record. Scanning by timestamp upserts the surviving file instead. + const upFile = fs.existsSync(dir) + ? fs.readdirSync(dir).find((f) => { + const parsed = f.match(/^(\d+)_(.*)\.up\.sql$/); + return parsed !== null && Number(parsed[1]) === m.timestamp; + }) + : undefined; + + if (upFile === undefined) { + log.info(colors.red(`Deleting datatable_migration ${m.datatable}/${m.timestamp}`)); + await wmill.deleteDatatableMigration({ + workspace, + datatableName: m.datatable, + timestamp: m.timestamp, + }); + return; + } + + const name = upFile.match(/^(\d+)_(.*)\.up\.sql$/)![2]; + const base = `${m.timestamp}_${name}`; + const code_up = await readTextFile(path.join(dir, upFile)); + const downPath = path.join(dir, `${base}.down.sql`); + const code_down = fs.existsSync(downPath) ? await readTextFile(downPath) : undefined; + + log.info(colors.green(`Pushing datatable_migration ${m.datatable}/${base}`)); + await wmill.upsertDatatableMigration({ + workspace, + datatableName: m.datatable, + requestBody: { + timestamp: m.timestamp, + name, + code_up, + ...(code_down !== undefined ? { code_down } : {}), + }, + }); +} + +/** + * Upsert the on-disk migrations of a data table to the workspace, so a freshly + * created migration file works with `wmill datatable migrate up` even without a + * prior `wmill sync push`. Pushes only migrations that are new or edited + * (compared against the workspace's current definitions); it never deletes + * remote migrations absent on disk and never touches other item kinds. + */ +export async function pushLocalMigrations( + workspace: string, + datatableName: string, +): Promise { + const dir = path.join(process.cwd(), MIGRATIONS_DIR, datatableName); + if (!fs.existsSync(dir)) return; + + // Local migrations are identified by their `.up.sql` file (the up file is + // mandatory); this deliberately ignores files that were only deleted locally. + const local: { timestamp: number; name: string }[] = []; + for (const file of fs.readdirSync(dir)) { + const m = file.match(/^(\d+)_(.*)\.up\.sql$/); + if (m) local.push({ timestamp: Number(m[1]), name: m[2] }); + } + if (local.length === 0) return; + + const remote = await wmill.listDatatableMigrations({ workspace }); + const remoteByTs = new Map( + remote + .filter((r) => r.datatable === datatableName) + .map((r) => [r.timestamp, r] as const), + ); + + for (const { timestamp, name } of local) { + const base = `${timestamp}_${name}`; + const code_up = await readTextFile(path.join(dir, `${base}.up.sql`)); + const downPath = path.join(dir, `${base}.down.sql`); + const code_down = fs.existsSync(downPath) + ? await readTextFile(downPath) + : undefined; + + const r = remoteByTs.get(timestamp); + const unchanged = + r !== undefined && + r.name === name && + r.code_up === code_up && + (r.code_down ?? undefined) === code_down; + if (unchanged) continue; + + log.info(colors.green(`Pushing datatable_migration ${datatableName}/${base}`)); + await wmill.upsertDatatableMigration({ + workspace, + datatableName, + requestBody: { + timestamp, + name, + code_up, + ...(code_down !== undefined ? { code_down } : {}), + }, + }); + } +} + +/** + * After a push that introduced new migrations, list them and (interactively) + * offer to run them, equivalent to `wmill datatable migrate up` on each affected + * data table. + */ +export async function offerToRunNewMigrations( + workspace: string, + newMigrations: { datatable: string; timestamp: number; name: string }[], + opts?: { yes?: boolean; jsonOutput?: boolean }, +): Promise { + if (newMigrations.length === 0) return; + + log.info(colors.green("New migrations were pushed:")); + for (const m of newMigrations) { + log.info(colors.gray(` ${m.datatable}: ${m.timestamp} ${m.name}`)); + } + + // Running migrations mutates the data tables, so skip the prompt in + // non-interactive contexts (--yes, --json, no TTY). + const interactive = !opts?.jsonOutput && !opts?.yes && !!process.stdin.isTTY; + if (!interactive) { + return; + } + + const shouldRun = await Confirm.prompt({ + message: "New migrations were pushed, run them?", + default: false, + }); + if (!shouldRun) { + return; + } + + for (const datatable of new Set(newMigrations.map((m) => m.datatable))) { + await runMigrations(workspace, datatable); + } +} diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index 8517ac0912..7eab79b52d 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -2,7 +2,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { colors } from "@cliffy/ansi/colors"; import { sep as SEP } from "node:path"; -import { GlobalOptions } from "../../types.ts"; +import { GlobalOptions, isDatatableMigrationPath } from "../../types.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; @@ -54,6 +54,8 @@ async function walkLocalScripts( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || isFolderResourcePathAnyFormat(p) || + // Datatable migration `.sql` files aren't Windmill scripts. + isDatatableMigrationPath(p) || (isScriptModulePath(p) && !isModuleEntryPoint(p)), false, {}, @@ -221,6 +223,8 @@ function categorizeLocalFiles( } else if ( exts.some((ext) => p.endsWith(ext)) && !isFolderResourcePathAnyFormat(p) && + // Datatable migration `.sql` files aren't Windmill scripts. + !isDatatableMigrationPath(p) && !(isScriptModulePath(p) && !isModuleEntryPoint(p)) ) { scripts.push(p); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 8e1ecdfef6..65ce10eaad 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -23,10 +23,17 @@ import { showDiff, extractNativeTriggerInfo, redactEncryptionKey, + isDatatableMigrationPath, + parseDatatableMigrationPath, } from "../../types.ts"; import { downloadZip } from "./pull.ts"; import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts"; import { pullSharedUi, pushSharedUi } from "../shared_ui.ts"; +import { + pushMigrationFromDisk, + offerToRunNewMigrations, + validateLocalMigrations, +} from "../datatable_migrations.ts"; import { exts, @@ -2477,7 +2484,8 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { !p.startsWith("g" + SEP) && !p.startsWith("users" + SEP) && !p.startsWith("groups" + SEP) && - !p.startsWith("dependencies" + SEP) + !p.startsWith("dependencies" + SEP) && + !p.startsWith("migrations" + SEP) ); } @@ -2488,6 +2496,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { try { const typ = getTypeStrFromPath(p); + // Datatable migrations live under migrations/datatable//, outside + // the u/f/g namespaces, but are valid wmill files. + if (typ == "datatable_migration") { + return false; + } if ( typ == "resource-type" || typ == "settings" || @@ -2519,7 +2532,8 @@ export const isWhitelisted = (p: string) => { p == "ui" || p == "users" || p == "groups" || - p == "dependencies" + p == "dependencies" || + p == "migrations" ); }; @@ -2614,6 +2628,11 @@ interface ChangeTracker { } async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { + // Datatable migration .sql files are not scripts; they're synced via the + // dedicated datatable_migration handler in the push loop. + if (isDatatableMigrationPath(p)) { + return; + } const isScript = exts.some((e) => p.endsWith(e)) && !isFileResource(p) && !isFilesetResource(p); if (isScript) { if (isFlowPath(p)) { @@ -3119,7 +3138,7 @@ export async function pull( change.path.endsWith(".json") ) { log.info( - `Editing ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Editing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3137,7 +3156,7 @@ export async function pull( if (opts.stateful) { await mkdir(path.dirname(stateTarget), { recursive: true }); log.info( - `Adding ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Adding ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3146,7 +3165,7 @@ export async function pull( } await writeFile(target, change.content, "utf-8"); log.info( - `Writing ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Writing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3158,7 +3177,7 @@ export async function pull( } else if (change.name === "deleted") { try { log.info( - `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`, + `Deleting ${changeTypeLabel(change.path)}${change.path}`, ); await rm(target); if (opts.stateful) { @@ -3350,6 +3369,9 @@ export async function pull( log.warn(`Failed to pull shared UI folder: ${e}`); } + // Datatable migrations are part of the workspace export now, so they flow + // through the normal diff/apply above as `datatable_migration` items. + // Git-sync deployment-callback mode stops here: branch checkout + pull have // happened, but commit + push are the caller's job. The hub script does // them in-process with `set_gpg_signing_secret` so the agent's pre-warmed @@ -3425,6 +3447,14 @@ export async function gitDeploy( } as any); } +// Display label for a change's type, with a trailing space. Datatable migrations +// are self-describing via their `migrations/datatable/...` path, so they get no +// label prefix. +function changeTypeLabel(p: string): string { + const t = getTypeStrFromPath(p); + return t === "datatable_migration" ? "" : `${t} `; +} + function prettyChanges( changes: Change[], specificItems?: SpecificItemsConfig, @@ -3456,7 +3486,7 @@ function prettyChanges( if (change.name === "added") { log.info( colors.green( - `+ ${getTypeStrFromPath(change.path)} ` + + `+ ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote), ) + extraNote, @@ -3464,7 +3494,7 @@ function prettyChanges( } else if (change.name === "deleted") { log.info( colors.red( - `- ${getTypeStrFromPath(change.path)} ` + + `- ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote), ), @@ -3473,7 +3503,7 @@ function prettyChanges( const changeType = getTypeStrFromPath(change.path); log.info( colors.yellow( - `~ ${changeType} ` + + `~ ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote) + (change.codebase ? ` (codebase changed)` : ""), @@ -4221,6 +4251,24 @@ export async function push( )); } + // Reject malformed datatable migrations (duplicate timestamps, orphan downs) + // before touching the remote, scanning only the data tables in this push. + const migrationDatatables = new Set( + changes + .map((c) => parseDatatableMigrationPath(c.path)?.datatable) + .filter((d): d is string => !!d), + ); + if (migrationDatatables.size > 0) { + const migrationErrors = validateLocalMigrations(migrationDatatables); + if (migrationErrors.length > 0) { + log.error( + "Invalid datatable migrations, aborting push:\n" + + migrationErrors.map((e) => ` - ${e}`).join("\n"), + ); + process.exit(1); + } + } + if ( !opts.yes && !(await Confirm.prompt({ @@ -4293,6 +4341,21 @@ export async function push( // Cache git branch at the start to avoid repeated execSync calls per change const cachedWsNameForPush = wsNameForFiles || (isGitRepository() ? getCurrentGitBranch() : null); + // Datatable migrations are two files (.up.sql/.down.sql) for one record, so + // dedupe upsert/delete by (datatable, version) across the whole push. + const pushedMigrationKeys = new Set(); + // Migrations newly added by this push (an added .up.sql) — offered to run once + // the push has completed. + const newDatatableMigrations = changes + .filter((c) => c.name === "added") + .map((c) => parseDatatableMigrationPath(c.path)) + .filter((p) => !!p && p.kind === "up") + .map((p) => ({ + datatable: p!.datatable, + timestamp: p!.timestamp, + name: p!.name, + })); + while (queue.length > 0 || pool.size > 0) { // Fill the pool until we reach the effective parallelism limit. // During the folder-meta phase this is 1 (sequential) so no item change @@ -4320,6 +4383,20 @@ export async function push( } for await (const change of changes) { + // A datatable migration is one record across two files; upsert/delete + // it from disk once (deduped), regardless of which file changed. + if (isDatatableMigrationPath(change.path)) { + const parsed = parseDatatableMigrationPath(change.path); + if (parsed) { + const key = `${parsed.datatable}\0${parsed.timestamp}`; + if (!pushedMigrationKeys.has(key)) { + pushedMigrationKeys.add(key); + await pushMigrationFromDisk(workspace.workspaceId, parsed); + } + } + continue; + } + let stateTarget = undefined; if (stateful) { try { @@ -4974,6 +5051,16 @@ export async function push( } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } + try { + await offerToRunNewMigrations(workspace.workspaceId, newDatatableMigrations, { + yes: opts.yes, + jsonOutput: opts.jsonOutput, + }); + } catch (e: any) { + log.warn( + `Failed to run new datatable migrations: ${e?.body ?? e?.message ?? e}`, + ); + } const lockJobs = await checkServerLockJobs( workspace.workspaceId, pushStartedAt, @@ -5039,6 +5126,7 @@ export async function push( } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } + // No changes pushed, so no new datatable migrations to run. if (opts.jsonOutput) { console.log( JSON.stringify( diff --git a/cli/src/commands/workspace/merge.ts b/cli/src/commands/workspace/merge.ts index b8ef24f7d4..585772a9d0 100644 --- a/cli/src/commands/workspace/merge.ts +++ b/cli/src/commands/workspace/merge.ts @@ -11,10 +11,12 @@ import { deleteItemInWorkspace, getOnBehalfOf, isTriggerOrScheduleKind, + parseDatatableMigrationDeployPath, type DeployKind, type DeployProvider, type TriggerDeployKind, } from "../../../windmill-utils-internal/src/deploy.ts"; +import { offerToRunNewMigrations } from "../datatable_migrations.ts"; // --------------------------------------------------------------------------- // Provider adapter — wraps CLI's standalone API functions @@ -85,6 +87,10 @@ const provider: DeployProvider = { createSchedule: wmill.createSchedule, updateSchedule: wmill.updateSchedule, deleteSchedule: wmill.deleteSchedule, + // Datatable migrations + listDatatableMigrations: wmill.listDatatableMigrations, + upsertDatatableMigration: wmill.upsertDatatableMigration, + deleteDatatableMigration: wmill.deleteDatatableMigration, }; /** @@ -530,6 +536,14 @@ async function mergeWorkspaces( // 10. Deploy let successCount = 0; let failCount = 0; + // Datatable migrations deployed (not deleted) into the target. Deploying a + // migration only upserts its definition — the target schema is unchanged until + // the migration is run — so offer to run them afterwards (like the push path). + const deployedMigrations: { + datatable: string; + timestamp: number; + name: string; + }[] = []; for (const diff of sorted) { const label = `${diff.kind}:${diff.path}`; @@ -573,6 +587,12 @@ async function mergeWorkspaces( if (result.success) { log.info(colors.green(` ✓ ${label}`)); successCount++; + if ( + !itemDeletedInSource && + (diff.kind as DeployKind) === "datatable_migration" + ) { + deployedMigrations.push(parseDatatableMigrationDeployPath(diff.path)); + } } else { log.info(colors.red(` ✗ ${label}: ${result.error}`)); failCount++; @@ -606,6 +626,18 @@ async function mergeWorkspaces( ) ); } + + // 13. Deployed migration definitions don't touch the target schema until run; + // offer to run them on the target now (interactive only, like the push path). + if (deployedMigrations.length > 0) { + try { + await offerToRunNewMigrations(workspaceTo, deployedMigrations, { + yes: opts.yes, + }); + } catch (e) { + log.warn(colors.yellow(`Failed to run deployed migrations: ${e}`)); + } + } } export { mergeWorkspaces }; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index e2cf0be86b..03a5c09eb7 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6593,6 +6593,13 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable migrate\` - manage datatable migrations + - \`datatable migrate new \` - scaffold a new migration (.up.sql / .down.sql files) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate up\` - apply all pending migrations to the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate down\` - roll back the most recent migration on the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) - \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -6861,19 +6868,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- \`jobs pull\` -- \`jobs push\` +- \`jobs pull [workspace:string]\` - Pull completed and queued jobs from workspace + - \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) + - \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before export +- \`jobs push [workspace:string]\` - Push completed and queued jobs to workspace + - \`-c, --completed-file \` - Completed jobs input file (default: completed_jobs.json) + - \`-q, --queued-file \` - Queued jobs input file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before import ### lint diff --git a/cli/src/types.ts b/cli/src/types.ts index 5de6cea48b..c125ae2c2e 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -274,10 +274,10 @@ export function parseFromPath(p: string, content: string): any { return isWorkspaceDependencies(p) ? content : p.endsWith(".yaml") - ? yamlParseContent(p, content) - : p.endsWith(".json") - ? JSON.parse(content) - : content; + ? yamlParseContent(p, content) + : p.endsWith(".json") + ? JSON.parse(content) + : content; } export function parseFromFile(p: string): any { if (p.endsWith(".json")) { @@ -288,9 +288,38 @@ export function parseFromFile(p: string): any { throw new Error("Could not read file " + p); } } +/** + * Parse a `migrations/datatable//_.(up|down).sql` + * path into its parts. Returns undefined for any other path. + */ +export function parseDatatableMigrationPath(p: string): + | { datatable: string; timestamp: number; name: string; kind: "up" | "down" } + | undefined { + const parts = p.split("/"); + if ( + parts[0] !== "migrations" || + parts[1] !== "datatable" || + parts.length !== 4 + ) + return undefined; + const m = parts[3].match(/^(\d+)_(.*)\.(up|down)\.sql$/); + if (!m) return undefined; + return { + datatable: parts[2], + timestamp: Number(m[1]), + name: m[2], + kind: m[3] as "up" | "down", + }; +} + +export function isDatatableMigrationPath(p: string): boolean { + return parseDatatableMigrationPath(p) !== undefined; +} + export function getTypeStrFromPath( p: string ): + | "datatable_migration" | "script" | "variable" | "flow" @@ -316,6 +345,9 @@ export function getTypeStrFromPath( | "settings" | "encryption_key" | "workspace_dependencies" { + if (isDatatableMigrationPath(p)) { + return "datatable_migration"; + } if (isScriptModulePath(p)) { return "script"; } diff --git a/cli/test/datatable_migrations_unit.test.ts b/cli/test/datatable_migrations_unit.test.ts new file mode 100644 index 0000000000..09af49f890 --- /dev/null +++ b/cli/test/datatable_migrations_unit.test.ts @@ -0,0 +1,122 @@ +/** + * Unit tests for datatable-migration path parsing and local validation. + * + * These exercise pure logic with no backend: + * - `parseDatatableMigrationPath` recognizes only the + * `migrations/datatable/
/_.(up|down).sql` shape. + * - `validateLocalMigrations` rejects the two invalid on-disk states a push + * must catch: two up (or two down) files sharing a timestamp, and a + * `.down.sql` with no matching `.up.sql`. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { parseDatatableMigrationPath } from "../src/types.ts"; +import { validateLocalMigrations } from "../src/commands/datatable_migrations.ts"; + +describe("parseDatatableMigrationPath", () => { + test("parses up and down files of the new layout", () => { + expect( + parseDatatableMigrationPath( + "migrations/datatable/mydt/20260101000001_create_users.up.sql", + ), + ).toEqual({ + datatable: "mydt", + timestamp: 20260101000001, + name: "create_users", + kind: "up", + }); + expect( + parseDatatableMigrationPath( + "migrations/datatable/my-dt/42_x.down.sql", + ), + ).toEqual({ datatable: "my-dt", timestamp: 42, name: "x", kind: "down" }); + }); + + test("rejects unrelated, legacy and malformed paths", () => { + for ( + const p of [ + // legacy top-level layout + "datatable_migrations/mydt/20260101000001_x.up.sql", + // wrong sub-namespace / depth + "migrations/ducklake/mydt/1_x.up.sql", + "migrations/datatable/1_x.up.sql", + "migrations/datatable/mydt/sub/1_x.up.sql", + // not a migration file + "migrations/datatable/mydt/notes.txt", + "migrations/datatable/mydt/x.up.sql", // no numeric timestamp prefix + // unrelated workspace files + "f/foo/bar.script.yaml", + "u/admin/script.ts", + ] + ) { + expect(parseDatatableMigrationPath(p)).toBeUndefined(); + } + }); +}); + +describe("validateLocalMigrations", () => { + let prevCwd: string; + let tmp: string; + + beforeEach(() => { + prevCwd = process.cwd(); + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "dtmig-")); + process.chdir(tmp); + }); + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + function write(datatable: string, file: string) { + const dir = path.join(tmp, "migrations", "datatable", datatable); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, file), "-- sql\n"); + } + + test("accepts up+down pairs and up-only migrations", () => { + write("mydt", "20260101000001_create_users.up.sql"); + write("mydt", "20260101000001_create_users.down.sql"); + write("mydt", "20260101000002_add_email.up.sql"); // down is optional + expect(validateLocalMigrations()).toEqual([]); + }); + + test("flags two up files sharing a timestamp", () => { + write("mydt", "20260101000003_foo.up.sql"); + write("mydt", "20260101000003_bar.up.sql"); + const errors = validateLocalMigrations(); + expect(errors.length).toBe(1); + expect(errors[0]).toContain("20260101000003"); + }); + + test("flags two down files sharing a timestamp", () => { + write("mydt", "20260101000004_a.up.sql"); + write("mydt", "20260101000004_a.down.sql"); + write("mydt", "20260101000004_b.down.sql"); + const errors = validateLocalMigrations(); + // duplicate down + the b.down orphan (no b.up) + expect(errors.some((e) => e.includes("down") && e.includes("20260101000004"))).toBe(true); + }); + + test("flags a down file with no matching up", () => { + write("mydt", "20260101000005_orphan.down.sql"); + const errors = validateLocalMigrations(); + expect(errors.length).toBe(1); + expect(errors[0]).toContain("20260101000005_orphan"); + }); + + test("only validates the requested datatables", () => { + write("bad", "20260101000006_x.up.sql"); + write("bad", "20260101000006_y.up.sql"); // duplicate, but in 'bad' + write("good", "20260101000007_ok.up.sql"); + expect(validateLocalMigrations(new Set(["good"]))).toEqual([]); + expect(validateLocalMigrations(new Set(["bad"])).length).toBe(1); + }); + + test("returns no errors when the migrations folder is absent", () => { + expect(validateLocalMigrations()).toEqual([]); + }); +}); diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 37cfbfaef4..525bb0fc0b 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.7.0", + "version": "1.8.2", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 990fd47edd..4c2bd735b3 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -21,6 +21,7 @@ export type DeployKind = | "resource_type" | "folder" | "schedule" + | "datatable_migration" | "http_trigger" | "websocket_trigger" | "kafka_trigger" @@ -161,6 +162,24 @@ export interface DeployProvider { requestBody: any; }): Promise; deleteFolder(p: { workspace: string; name: string }): Promise; + // Datatable migrations. In the diff, an item's `path` is + // `/_` (see `parseDatatableMigrationDeployPath`). + listDatatableMigrations(p: { workspace: string }): Promise; + upsertDatatableMigration(p: { + workspace: string; + datatableName: string; + requestBody: { + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }; + }): Promise; + deleteDatatableMigration(p: { + workspace: string; + datatableName: string; + timestamp: number; + }): Promise; // Triggers — per-kind dispatch is delegated to the implementor so the shared // module doesn't need to know about each of the 9 trigger services. existsTriggerByKind( @@ -299,6 +318,40 @@ function toError(e: unknown): string { return err.body || err.message || String(e); } +// A datatable-migration diff item's path is `/_` +// (mirrors the backend, e.g. `mydt/20260101000001_create_users`). +export function parseDatatableMigrationDeployPath(path: string): { + datatable: string; + timestamp: number; + name: string; +} { + const slash = path.indexOf("/"); + const underscore = slash >= 0 ? path.indexOf("_", slash + 1) : -1; + if (slash < 0 || underscore < 0) { + throw new Error(`Invalid datatable migration path: ${path}`); + } + const datatable = path.slice(0, slash); + const timestamp = Number(path.slice(slash + 1, underscore)); + const name = path.slice(underscore + 1); + if (!datatable || !Number.isFinite(timestamp) || !name) { + throw new Error(`Invalid datatable migration path: ${path}`); + } + return { datatable, timestamp, name }; +} + +// The backend rejects `upsertDatatableMigration` when the target data table +// hasn't opted in to migrations. Turn that opaque 400 into an explicit, +// deploy-context message (falls back to the original error otherwise). +function asMigrationsDisabledError(e: unknown, datatable: string): unknown { + const msg = (e as { body?: string; message?: string })?.body ?? '' + if (typeof msg === "string" && /migrations are not enabled/i.test(msg)) { + return new Error( + `Data table '${datatable}' has not opted in to migrations on the target workspace; enable migrations for it there before deploying its migrations.` + ); + } + return e; +} + // --------------------------------------------------------------------------- // checkItemExists // --------------------------------------------------------------------------- @@ -325,6 +378,12 @@ export async function checkItemExists( return provider.existsFolder({ workspace, name: folderName(path) }); } else if (kind === "schedule") { return provider.existsSchedule({ workspace, path }); + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = await provider.listDatatableMigrations({ workspace }); + return (migrations as { datatable: string; timestamp: number }[]).some( + (m) => m.datatable === datatable && m.timestamp === timestamp + ); } else if (isTriggerKind(kind)) { return provider.existsTriggerByKind(kind, { workspace, path }); } @@ -639,6 +698,41 @@ export async function deployItem( requestBody, }); } + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = await provider.listDatatableMigrations({ + workspace: workspaceFrom, + }); + const migration = ( + migrations as { + datatable: string; + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }[] + ).find((m) => m.datatable === datatable && m.timestamp === timestamp); + if (!migration) { + throw new Error( + `Datatable migration ${path} not found in ${workspaceFrom}` + ); + } + try { + await provider.upsertDatatableMigration({ + workspace: workspaceTo, + datatableName: datatable, + requestBody: { + timestamp: migration.timestamp, + name: migration.name, + code_up: migration.code_up, + ...(migration.code_down != null + ? { code_down: migration.code_down } + : {}), + }, + }); + } catch (e) { + throw asMigrationsDisabledError(e, datatable); + } } else { throw new Error(`Unknown kind: ${kind}`); } @@ -684,6 +778,13 @@ export async function deleteItemInWorkspace( await provider.deleteFolder({ workspace, name: folderName(path) }); } else if (kind === "schedule") { await provider.deleteSchedule({ workspace, path }); + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + await provider.deleteDatatableMigration({ + workspace, + datatableName: datatable, + timestamp, + }); } else if (isTriggerKind(kind)) { await provider.deleteTriggerByKind(kind, { workspace, path }); } else { @@ -767,6 +868,29 @@ export async function getItemValue( } else if (isTriggerKind(kind)) { const trigger = await provider.getTriggerValue(kind, { workspace, path }); return stripTriggerOrScheduleRuntimeFields(trigger); + } else if (kind === "datatable_migration") { + // Surface the migration SQL so the diff drawer shows the up/down bodies a + // reviewer needs to inspect before deploying. + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = (await provider.listDatatableMigrations({ + workspace, + })) as { + datatable: string; + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }[]; + const migration = migrations.find( + (m) => m.datatable === datatable && m.timestamp === timestamp + ); + if (migration) { + return { + name: migration.name, + code_up: migration.code_up, + code_down: migration.code_down ?? null, + }; + } } } catch { // Item may not exist diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3023846ce6..0df7eef6f6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -94,7 +94,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.7.0", + "windmill-utils-internal": "1.8.2", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -291,7 +291,6 @@ "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", @@ -307,7 +306,6 @@ "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -865,7 +863,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -875,21 +872,21 @@ } }, "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", + "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.1", + "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", "dev": true, "license": "MIT", "optional": true, @@ -898,9 +895,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -1392,6 +1389,7 @@ "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", @@ -1548,6 +1546,7 @@ "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -1774,8 +1773,8 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", + "@emnapi/core": "1.11.2", + "@emnapi/runtime": "1.11.2", "@napi-rs/wasm-runtime": "^1.1.4" }, "engines": { @@ -1912,6 +1911,7 @@ "integrity": "sha512-iAIPEahFgDJJyvz8g0jP08KvqnM6JvdW8YfsygZ+pMeMvyM2zssWMltcsotETvjSZ82G3VlitgDtBIvpQSZrTA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -2007,6 +2007,7 @@ "integrity": "sha512-ILXmxC7HAsnkK2eslgPetrqqW1BKSL7LktsFgqzNj83MaivMGZzluWq32m25j2mDOjmSKX7GGWahePhuEs7P/g==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "deepmerge": "^4.3.1", "magic-string": "^0.30.21", @@ -2468,8 +2469,7 @@ "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/ms": { "version": "2.1.0", @@ -2482,8 +2482,7 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/semver": { "version": "7.7.1", @@ -2552,6 +2551,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -3079,6 +3079,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3132,6 +3133,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -3277,7 +3279,6 @@ "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -3305,7 +3306,6 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -3386,8 +3386,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz", "integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/base64-js": { "version": "1.5.1", @@ -3514,6 +3513,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -3711,7 +3711,6 @@ "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "camelcase": "^6.3.0", "map-obj": "^4.1.0", @@ -3731,7 +3730,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3745,7 +3743,6 @@ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3759,7 +3756,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -3868,6 +3864,7 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", + "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -4144,7 +4141,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -4209,7 +4205,6 @@ "integrity": "sha512-8HFEBPKhOpJPEPu70wJJetjKta86Gw9+CCyCnB3sui2qQfOvRyqBy4IKLKKAwdMpWb2lHXWk9Wb4Z6AmaUT1Pg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" } @@ -4397,6 +4392,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -4819,6 +4815,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -4918,6 +4915,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.21.0" }, @@ -4958,7 +4956,6 @@ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4972,7 +4969,6 @@ "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "decamelize": "^1.1.0", "map-obj": "^1.0.0" @@ -4990,7 +4986,6 @@ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5001,7 +4996,6 @@ "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5495,7 +5489,6 @@ "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -5593,6 +5586,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6149,7 +6143,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.9.1" } @@ -6509,7 +6502,6 @@ "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -6523,7 +6515,6 @@ "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -6539,7 +6530,6 @@ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -6589,8 +6579,7 @@ "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/gopd": { "version": "1.2.0", @@ -6673,7 +6662,6 @@ "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -6910,7 +6898,6 @@ "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -6924,7 +6911,6 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -6937,8 +6923,7 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/html-tags": { "version": "3.3.1", @@ -6946,7 +6931,6 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -7042,7 +7026,6 @@ "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -7073,7 +7056,6 @@ "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7143,8 +7125,7 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -7249,7 +7230,6 @@ "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7260,7 +7240,6 @@ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7365,8 +7344,7 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -7402,8 +7380,7 @@ "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-refs": { "version": "3.0.15", @@ -7600,7 +7577,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8206,8 +8182,7 @@ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.uniq": { "version": "4.5.0", @@ -8275,7 +8250,6 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -8326,7 +8300,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -8567,7 +8540,6 @@ "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/minimist": "^1.2.2", "camelcase-keys": "^7.0.0", @@ -8595,7 +8567,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -9330,7 +9301,6 @@ "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0", @@ -9396,6 +9366,7 @@ "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-25.0.0.tgz", "integrity": "sha512-uiY06RTWFo2WZdh6OybkLlDhuG+8LlkjUDpr9/wW55uucqHo4X8fx4XKEtD98cscC+6FKQkbG2yyUiOJ/npHOw==", "license": "MIT", + "peer": true, "dependencies": { "@codingame/monaco-vscode-api": "25.0.0" } @@ -9632,7 +9603,6 @@ "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -9984,7 +9954,6 @@ "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -10306,6 +10275,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -10494,6 +10464,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" @@ -10883,8 +10854,7 @@ "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -11060,6 +11030,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11363,7 +11334,6 @@ "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/normalize-package-data": "^2.4.0", "normalize-package-data": "^3.0.2", @@ -11383,7 +11353,6 @@ "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-up": "^5.0.0", "read-pkg": "^6.0.0", @@ -11402,7 +11371,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11416,7 +11384,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11459,7 +11426,6 @@ "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "indent-string": "^5.0.0", "strip-indent": "^4.0.0" @@ -12100,7 +12066,6 @@ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -12177,7 +12142,6 @@ "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" @@ -12188,8 +12152,7 @@ "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", "dev": true, - "license": "CC-BY-3.0", - "peer": true + "license": "CC-BY-3.0" }, "node_modules/spdx-expression-parse": { "version": "3.0.1", @@ -12197,7 +12160,6 @@ "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.0" @@ -12208,8 +12170,7 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", "dev": true, - "license": "CC0-1.0", - "peer": true + "license": "CC0-1.0" }, "node_modules/sprintf-js": { "version": "1.0.3", @@ -12303,7 +12264,6 @@ "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12329,8 +12289,7 @@ "resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz", "integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/style-to-object": { "version": "0.4.4", @@ -12379,7 +12338,6 @@ "integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^2.3.1", "@csstools/css-tokenizer": "^2.2.0", @@ -12462,7 +12420,6 @@ } ], "license": "MIT-0", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -12476,7 +12433,6 @@ "integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.2.0" }, @@ -12489,8 +12445,7 @@ "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz", "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/stylelint/node_modules/postcss-selector-parser": { "version": "6.1.2", @@ -12513,7 +12468,6 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -12654,7 +12608,6 @@ "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -12684,6 +12637,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.5.tgz", "integrity": "sha512-YkqERnF05g8KLdDZwZrF8/i1eSbj6Eoat8Jjr2IfruZz9StLuBqo8sfCSzjosNKd+ZrQ8DkKZDjpO5y3ht1Pow==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -12781,21 +12735,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13025,8 +12964,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", - "dev": true, - "peer": true + "dev": true }, "node_modules/svgo": { "version": "3.3.2", @@ -13077,7 +13015,6 @@ "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -13105,6 +13042,7 @@ "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -13357,6 +13295,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13419,7 +13358,6 @@ "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13537,6 +13475,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13773,7 +13712,6 @@ "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-correct": "^3.0.0", "spdx-expression-parse": "^3.0.0" @@ -13827,6 +13765,7 @@ "integrity": "sha512-MFtjBYgzmSxmgA4RAfjIyXWpGe1oALnjgUTzzV7QLx/TKxCzjtMH6Fd9/eVK+5Fg1qNoz5VAwsmMs/NofrmJvw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", @@ -14396,9 +14335,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.7.0.tgz", - "integrity": "sha512-K5kAiJKhavfGatmicbJyqT+KFspzFZK5Ou14pHj4Xg6Q2Z1a4ZgziBdrRNsaeAuBhB62yXWZA0OXDzg936Ymmw==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.8.2.tgz", + "integrity": "sha512-Otdn4iCE0QiOtMKPdHX5I89oECfZa3vmO2jdLdSETf7MBSv/gw9gvQm3oUQ29ymIHXLeuhQk/ebPD+WYolQJEA==", "license": "Apache 2.0" }, "node_modules/word-wrap": { @@ -14534,7 +14473,6 @@ "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" @@ -14730,7 +14668,6 @@ "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -14750,6 +14687,7 @@ "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", "license": "MIT", + "peer": true, "dependencies": { "lib0": "^0.2.99" }, @@ -14785,6 +14723,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index 70d24aba68..fe0366eb62 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -169,7 +169,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.7.0", + "windmill-utils-internal": "1.8.2", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 8cfa311141..a2453add27 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -646,7 +646,7 @@ deploymentStatus[statusKey] = { status: 'deployed' } } else { deploymentStatus[statusKey] = { status: 'failed', error: result.error } - sendUserToast(`Failed to deploy ${statusKey}: ${result.error}`) + sendUserToast(`Failed to deploy ${statusKey}: ${result.error}`, 'error') } } @@ -696,7 +696,10 @@ if (!aIsFolder && bIsFolder) return 1 return 0 }) + const to = mergeIntoParent ? parent : current let anyFailed = false + // Datatables whose migrations deployed cleanly — candidates for a run prompt. + const deployedMigrationDatatables = new Set() for (const itemKey of sortedItems) { const deployable = deployableItems.find((d) => d.key === itemKey) @@ -705,16 +708,19 @@ continue } - const to = mergeIntoParent ? parent : current const from = mergeIntoParent ? current : parent await deploy(deployable.kind, deployable.path, to, from, itemKey) if (deploymentStatus[itemKey]?.status === 'failed') { anyFailed = true + } else if (deployable.kind === 'datatable_migration') { + deployedMigrationDatatables.add(deployable.path.split('/')[0]) } } deploying = false deselectAll() + await maybePromptRunMigrations(deployedMigrationDatatables, to) + // If every selected item deployed cleanly and the direction was // merge-into-parent, resolve any open deployment request for this fork. if (!anyFailed && mergeIntoParent) { @@ -750,6 +756,51 @@ onChanged?.() } + /** + * After a deploy, offer to run the migrations of every cloned datatable that + * received one. `forked_from` is set on the fork's datatable config only when + * the datatable was cloned into a separate database — shared-DB datatables + * have already had the schema change applied and must not be re-run. + */ + async function maybePromptRunMigrations( + deployedMigrationDatatables: Set, + runTargetWorkspace: string + ) { + if (deployedMigrationDatatables.size === 0) return + try { + const forkSettings = await WorkspaceService.getPublicSettings({ + workspace: currentWorkspaceId + }) + const datatables = forkSettings.datatable?.datatables ?? {} + const cloned = [...deployedMigrationDatatables].filter( + (dt) => datatables[dt]?.forked_from != null + ) + if (cloned.length === 0) return + runMigrationsDatatables = cloned.sort() + runMigrationsTargetWorkspace = runTargetWorkspace + runMigrationsModalOpen = true + } catch (e) { + console.error('Failed to determine cloned datatables for migration run prompt', e) + } + } + + async function runDeployedMigrations() { + runMigrationsModalOpen = false + for (const dt of runMigrationsDatatables) { + try { + const res = await WorkspaceService.runDatatableMigrations({ + workspace: runMigrationsTargetWorkspace, + datatableName: dt + }) + sendUserToast( + `Ran ${res.applied.length} migration${res.applied.length !== 1 ? 's' : ''} on ${dt}` + ) + } catch (e: any) { + sendUserToast(`Failed to run migrations on ${dt}: ${e.body ?? e.message ?? e}`, true) + } + } + } + function toggleKey(key: string) { if (selectedItems.includes(key)) { selectedItems = selectedItems.filter((i) => i !== key) @@ -930,6 +981,17 @@ let deploymentRequestPanel: DeploymentRequestPanel | undefined = $state(undefined) let hasOpenDeploymentRequest = $state(false) + // After deploying datatable migrations to a cloned (separate-DB) datatable, we + // offer to run them in the target workspace. Shared-DB datatables are skipped: + // the schema change is already physically applied, so re-running is redundant. + let runMigrationsModalOpen = $state(false) + let runMigrationsDatatables = $state([]) + let runMigrationsTargetWorkspace = $state('') + let runMigrationsTargetWorkspaceName = $derived( + $userWorkspaces.find((w) => w.id == runMigrationsTargetWorkspace)?.name ?? + runMigrationsTargetWorkspace + ) + /** Display labels for trigger/schedule kinds in the merge UI. */ const KIND_DISPLAY_NAMES: Record = { schedule: 'Schedule', @@ -942,7 +1004,8 @@ sqs_trigger: 'SQS trigger', gcp_trigger: 'GCP trigger', azure_trigger: 'Azure trigger', - email_trigger: 'Email trigger' + email_trigger: 'Email trigger', + datatable_migration: 'Data table migration' } // Human label for a diff kind, lowercased for inline use in the hidden-items @@ -1453,9 +1516,7 @@ /> -
- -
+ {#if pinnedItems.length > 0}
@@ -1536,6 +1597,26 @@
+ (runMigrationsModalOpen = false)} + > +
+

+ Run the deployed migrations in {runMigrationsTargetWorkspaceName} now? These data tables + use a separate database, so the schema changes won't apply until the migrations are run. +

+
    + {#each runMigrationsDatatables as dt (dt)} +
  • {dt}
    • + {/each} +
+
+ + { if (dbTableEditorState.alterTableKey && dbTableEditorAlterTableData.current) { let diff = diffTableEditorValues(dbTableEditorAlterTableData.current, values) - await dbSchemaOps.onAlter({ schema: selected.schemaKey, values: diff }) + // Reverse diff (new → old) so the migration's down undoes the alter. + let reverse = diffTableEditorValues(values, dbTableEditorAlterTableData.current) + await dbSchemaOps.onAlter({ schema: selected.schemaKey, values: diff, reverse }) } else { await dbSchemaOps.onCreate({ values, schema: selected.schemaKey }) } diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index d84bdb862e..c40a2f7428 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -20,6 +20,10 @@ import type { SelectedTable } from './DBManager.svelte' import { getDbFeatures } from './apps/components/display/dbtable/dbFeatures' import { resource } from 'runed' + import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' + import { createAsyncConfirmationModal } from './common/confirmationModal/asyncConfirmationModal.svelte' + import Portal from '$lib/components/Portal.svelte' + import { outOfOrderRunMessage } from './workspaceSettings/datatableMigrationUtils' interface Props { input?: DbInput @@ -52,6 +56,8 @@ let dbSchema: DBSchema | undefined = $derived(input && $dbSchemas[getDbSchemasPath(input)]) + const outOfOrderModal = createAsyncConfirmationModal() + function getDbSchemasPath(input: DbInput): string { switch (input.type) { case 'database': @@ -163,7 +169,13 @@ })} dbSchemaOps={dbSchemaOpsWithPreviewScripts({ input: _input, - workspace: $workspaceStore + workspace: $workspaceStore, + confirmRunOutOfOrder: (pending) => + outOfOrderModal.ask({ + title: 'Run migration out of order', + confirmationText: 'Run anyway', + children: outOfOrderRunMessage(pending) + }) })} initialTableKey={input.specificTable} initialSchemaKey={input.specificSchema} @@ -192,6 +204,7 @@ onData={(data) => { replResultData = data }} + onSchemaChange={() => refresh()} placeholderTableName={sortArray( Object.keys( dbSchema?.schema[ @@ -214,3 +227,9 @@ {/if} + + + + + diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 6598103751..b303671fe8 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -16,6 +16,7 @@ Upload } from 'lucide-svelte' import DBManagerContent from './DBManagerContent.svelte' + import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' import { resource } from 'runed' import { untrack } from 'svelte' import type { DbManagerUriState } from './dbManagerDrawerModel.svelte' @@ -105,6 +106,11 @@ return toSourceIdentifier(input.resourcePath) } + function refreshManager() { + dbManagerContent?.refresh() + dbManagerContent?.dbManager()?.dbTable()?.refresh() + } + async function handleExportSchema() { const source = currentSourceIdentifier() if (!source || !$workspaceStore) return @@ -201,6 +207,13 @@ {/key} {/if} {#snippet actions()} + {#if uriState.isDatatableInput && uriState.selectedDatatable && $workspaceStore} + + {/if} {#if enableImportExport} + /> +{#if applicableCount > 0} +
+

Datatable schema changes

+ {#if loading} +
+ Loading datatable diffs... +
+ {:else if error} +
Failed to load datatable diffs: {error}
+ {:else if diffs.length > 0} +
+ {#each diffs as diff} + + - {#if expandedDatatables.has(diff.datatableName)} -
- {#if diff.aheadChanges.length > 0} -
-
Fork changes (ahead)
- {#each diff.aheadChanges as change} -
- {#if change.kind === 'added'} - - {:else if change.kind === 'removed'} - - {:else} - - {/if} - {change.schemaName}. - {change.tableName} - {operationSummary(change)} - + {#each diff.aheadChanges as change} +
+ {#if change.kind === 'added'} + + {:else if change.kind === 'removed'} + + {:else} + + {/if} + {change.schemaName}. + {change.tableName} + {operationSummary(change)} + +
+ {/each}
- {/each} + {/if} + {#if diff.behindChanges.length > 0} +
+
+ Parent changes (behind) +
+ {#each diff.behindChanges as change} +
+ {#if change.kind === 'added'} + + {:else if change.kind === 'removed'} + + {:else} + + {/if} + {change.schemaName}. + {change.tableName} + {operationSummary(change)} + +
+ {/each} +
+ {/if}
{/if} - {#if diff.behindChanges.length > 0} -
-
- Parent changes (behind) -
- {#each diff.behindChanges as change} -
- {#if change.kind === 'added'} - - {:else if change.kind === 'removed'} - - {:else} - - {/if} - {change.schemaName}. - {change.tableName} - {operationSummary(change)} - -
- {/each} -
- {/if} -
- {/if} - - {/each} + + {/each} +
+ {:else} + No changes detected + {/if}
-{:else} - No changes detected {/if} @@ -580,3 +655,7 @@ >{migrationSql} + + + + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte new file mode 100644 index 0000000000..b6833521f8 --- /dev/null +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -0,0 +1,159 @@ + + + + + +
+

+ This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? +

+ 
{promptStatement ?? ''}
+
+ + +
+
+ + + migrationsModal?.openMigration(m.timestamp)} +/> + + diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 78a75734c9..8563190a89 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -43,6 +43,7 @@ import { editorFontSize } from '$lib/editorFontSize.svelte' import { createHash as randomHash } from '$lib/editorLangUtils' import { workspaceStore } from '$lib/stores' + import DdlMigrationGuard from './DdlMigrationGuard.svelte' import { type Preview, ResourceService, @@ -222,6 +223,35 @@ let lang = $state(scriptLangToEditorLang(untrack(() => scriptLang))) + // On a postgres script targeting a datatable, DDL statements are intercepted + // on run (cmd+enter) and offered as migrations instead. + let datatableForMigrations = $derived( + scriptLang === 'postgresql' && + typeof args?.database === 'string' && + args.database.startsWith('datatable://') + ? args.database.slice('datatable://'.length).split('/')[0] + : undefined + ) + let ddlGuard = $state(undefined) + + // Run the DDL migration guard against the current code. Returns false when the + // user cancels (the run must be aborted); may rewrite the code (migrated + // statements stripped). Exported so run paths that bypass the Monaco + // Cmd+Enter binding (e.g. the Test button) can guard too. + export async function guardDdlBeforeRun(): Promise { + if (datatableForMigrations && ddlGuard) { + const res = await ddlGuard.guard(getCode()) + if (!res.proceed) return false + if (res.code !== getCode()) setCode(res.code) + } + return true + } + + async function runCmdEnterWithDdlGuard() { + if (!(await guardDdlBeforeRun())) return + cmdEnterAction?.() + } + let filePath = $state(computePath(untrack(() => path))) let initialPath: string | undefined = $state(untrack(() => path)) @@ -1639,7 +1669,8 @@ editor?.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () { updateCode() - shouldBindKey && cmdEnterAction && cmdEnterAction() + if (!shouldBindKey || !cmdEnterAction) return + void runCmdEnterWithDdlGuard() }) editor?.addCommand(KeyMod.CtrlCmd | KeyMod.Shift | KeyCode.Digit7, function () { @@ -2209,6 +2240,13 @@ +{#if datatableForMigrations && $workspaceStore} + +{/if} {#if !editor}
diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index ac941d0bf0..bf5349b126 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -793,7 +793,17 @@ args = nargs } - export async function runTest(opts?: { cascade?: boolean }) { + export async function runTest(opts?: { cascade?: boolean; skipDdlGuard?: boolean }) { + // Intercept DDL statements (offer to turn them into data table migrations) + // on every run path, not just the editor's Cmd+Enter. `skipDdlGuard` is set + // by the Cmd+Enter action, which already guarded before calling us. + if (!opts?.skipDdlGuard) { + if ((await editor?.guardDdlBeforeRun()) === false) return + // The guard may have rewritten the code (migrated statements stripped); + // `editorCode` is kept in sync by the editor binding, so mirror the + // on:change handler and pull it into `code` before we run. + if (activeModuleTab === null) code = editorCode + } // When the caller forces a cascade choice (e.g. the canvas runnable // menu's "Run + trigger N downstream"), also flip the persistent // `cascadeDownstream` state so the split button's label/icon reflect @@ -2677,7 +2687,8 @@ } else { await inferModuleSchema() } - runTest() + // The Editor already ran the DDL guard before invoking this action. + runTest({ skipDdlGuard: true }) }} formatAction={async () => { if (activeModuleTab === null) { diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 741d874f81..f1c4daa962 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -95,6 +95,7 @@ loadAsync = false, key, disabled = false, + readOnly = false, minHeight = 1000, renderLineHighlight = 'none', suggestion @@ -123,6 +124,9 @@ initialCursorPos?: IPosition key?: string disabled?: boolean + /** Read-only Monaco mode: not editable, but still scrollable/selectable + * (unlike `disabled`, which makes the editor non-interactive). */ + readOnly?: boolean minHeight?: number renderLineHighlight?: 'all' | 'line' | 'gutter' | 'none' suggestion?: string @@ -239,6 +243,9 @@ lineNumbers: $relativeLineNumbers ? 'relative' : 'on' }) }) + $effect(() => { + editor?.updateOptions({ readOnly }) + }) function onVimDisable() { vimDisposable?.dispose() @@ -342,6 +349,7 @@ ), model, ...(yPadding !== undefined ? { padding: { bottom: yPadding, top: yPadding } } : {}), + readOnly, renderLineHighlight, lineDecorationsWidth: 0, lineNumbersMinChars: 2, diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index 661b0ccd2c..6c8d90ba15 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -1,49 +1,3 @@ - - + +{#if !hideTrigger} + +{/if} + + + {#snippet headerLeft()} + + Each schema edit made in the database manager is captured here as a migration. Tracking schema + changes as migrations makes it easy to export data tables to other workspaces, and to + reproduce their schema when forking a workspace. + + {/snippet} + {#snippet headerRight()} + {#if enabled && canManage} + disableMigrations() + } + ]} + > + {#snippet buttonReplacement()} + + {/if} +
+ {:else} + {#if loadError} +
+ Could not read applied status from the data table: {loadError} +
+ {/if} +
+ {#if migrations.length === 0} +
+ No migrations yet + +
+ {:else} + {#each migrations as m (m.timestamp)} +
+
+ +
+ {/each} + {/if} +
+
+ + +
+ {/if} + + + + (newMigrationOpen = false)} + onSeeMigration={(m) => openMigration(m.timestamp)} +/> + + + {#if viewMigration} +
+ {viewMigration.timestamp} + + + + {#snippet content()} + + + + + {#if viewMigration?.code_down} + + {:else} +
No down migration
+ {/if} + + {/snippet} + +
+ {/if} + + + +
+

+ "{deleteTarget?.name}" is installed on the data table. Revert it first to undo its schema + change, or delete only the definition and leave the schema as-is — deleting without reverting + means it can no longer be reverted. +

+
+ + + +
+
+ + + + + diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index af6fe29e58..f652ed8704 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -1,6 +1,12 @@ + + + +
+ + + + + {#snippet content()} + + + + +
+ + {#if enableDown} +
+ +
+ {/if} +
+ + {/snippet} + +
+ +
+
+ + + + + diff --git a/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts new file mode 100644 index 0000000000..ed046418f2 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts @@ -0,0 +1,30 @@ +import { WorkspaceService, type DatatableMigrationWithStatus } from '$lib/gen' + +/** + * Migrations that are defined but not yet applied. A newly-created migration + * always gets the highest timestamp, so every pending migration is "earlier": + * running the new one on its own would apply it ahead of them (out of order). + */ +export function pendingMigrations( + migrations: DatatableMigrationWithStatus[] +): DatatableMigrationWithStatus[] { + return migrations.filter((m) => m.status !== 'ran') +} + +/** Fetch the data table's migration status and return the pending ones. */ +export async function fetchPendingMigrations( + workspace: string, + datatableName: string +): Promise { + const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName + }) + return pendingMigrations(migrations) +} + +/** Confirmation copy shown before running a just-created migration ahead of + * `count` still-pending earlier ones (mirrors the row-level Run warning). */ +export function outOfOrderRunMessage(count: number): string { + return `${count} earlier migration(s) have not been run yet. This migration might depend on them. Run it anyway?` +} diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 5f49eb5c49..556ac0a3c7 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -40,6 +40,8 @@ export type Kind = | 'gcp_trigger' | 'azure_trigger' | 'email_trigger' + // Data table migration, diffed per `/_` path. + | 'datatable_migration' // Legacy generic kind used by the cross-workspace `DeployWorkspace` UI, // which carries the trigger sub-kind in `additionalInformation`. | 'trigger' diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 112ed69e17..72400bb962 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -16,7 +16,8 @@ import { SqsTriggerService, UserService, VariableService, - WebsocketTriggerService + WebsocketTriggerService, + WorkspaceService } from '$lib/gen' import { fetchProtectionRulesForWorkspace, @@ -234,7 +235,11 @@ function makeProvider(): DeployProvider { getSchedule: (p) => ScheduleService.getSchedule(p), createSchedule: (p) => ScheduleService.createSchedule(p), updateSchedule: (p) => ScheduleService.updateSchedule(p), - deleteSchedule: (p) => ScheduleService.deleteSchedule(p) + deleteSchedule: (p) => ScheduleService.deleteSchedule(p), + // Datatable migrations + listDatatableMigrations: (p) => WorkspaceService.listDatatableMigrations(p), + upsertDatatableMigration: (p) => WorkspaceService.upsertDatatableMigration(p), + deleteDatatableMigration: (p) => WorkspaceService.deleteDatatableMigration(p) } } diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 01579a737a..a74fdc6a17 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -77,6 +77,13 @@ datatable related commands - `datatable run ` - run a SQL query on a datatable - `-n --name ` - Datatable name (default: main) - `-s --silent` - Output only the final result as JSON. Useful for scripting. +- `datatable migrate` - manage datatable migrations + - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) - `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -345,19 +352,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** `[workspace:string]` - -**Options:** -- `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) -- `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) -- `--skip-worker-check` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- `jobs pull` -- `jobs push` +- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace + - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) + - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before export +- `jobs push [workspace:string]` - Push completed and queued jobs to workspace + - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json) + - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before import ### lint diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 13776fe1e6..58ac1ac69d 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2755,6 +2755,13 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable migrate\` - manage datatable migrations + - \`datatable migrate new \` - scaffold a new migration (.up.sql / .down.sql files) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate up\` - apply all pending migrations to the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate down\` - roll back the most recent migration on the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) - \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -3023,19 +3030,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- \`jobs pull\` -- \`jobs push\` +- \`jobs pull [workspace:string]\` - Pull completed and queued jobs from workspace + - \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) + - \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before export +- \`jobs push [workspace:string]\` - Push completed and queued jobs to workspace + - \`-c, --completed-file \` - Completed jobs input file (default: completed_jobs.json) + - \`-q, --queued-file \` - Queued jobs input file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before import ### lint diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index cc314a5067..db04118c3f 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -82,6 +82,13 @@ datatable related commands - `datatable run ` - run a SQL query on a datatable - `-n --name ` - Datatable name (default: main) - `-s --silent` - Output only the final result as JSON. Useful for scripting. +- `datatable migrate` - manage datatable migrations + - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) - `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -350,19 +357,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** `[workspace:string]` - -**Options:** -- `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) -- `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) -- `--skip-worker-check` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- `jobs pull` -- `jobs push` +- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace + - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) + - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before export +- `jobs push [workspace:string]` - Push completed and queued jobs to workspace + - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json) + - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before import ### lint diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 98c03c4367..dac36ad97a 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -340,13 +340,54 @@ def extract_description(section: str) -> str | None: return ''.join(_unquote_js_string(p) for p in parts).strip() or None -def parse_command_block(content: str, file_path: Path | None = None) -> dict: +def extract_named_command_block(content: str, var_name: str) -> str | None: + """Return the chained-call body of `const = new Command() ...`, + from just after `new Command()` up to the next top-level statement. + + Returns None when the var isn't a *direct* `new Command()` (e.g. it's wrapped + in a helper call like `auditListOptions(new Command()...)`), so callers can + fall back to a looser match. + """ + m = re.search( + r'const\s+' + re.escape(var_name) + r'\s*=\s*new\s+Command\(\)' + r'([\s\S]*?)(?=\n(?:const|let|var|async|function|export)\b)', + content, + ) + return m.group(1) if m else None + + +def extract_exported_command_block(content: str) -> str | None: + """Return the chained-call body of the command that is `export default`ed. + + A command file may define helper `new Command()` groups (assigned to local + consts and mounted as nested subcommands via `.command("x", localCmd)`) + *before* the exported command. Anchoring on the first `new Command()` in the + file would merge those helpers into the top-level command, so resolve the + exported variable first and only then fall back to the first `new Command()` + (which covers inline/wrapped exports). + """ + export_match = re.search(r'export\s+default\s+(\w+)\s*;', content) + if export_match: + block = extract_named_command_block(content, export_match.group(1)) + if block is not None: + return block + command_match = re.search( + r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', + content, + ) + return command_match.group(1) if command_match else None + + +def parse_command_block( + content: str, file_path: Path | None = None, block: str | None = None +) -> dict: """ Parse a Cliffy Command() definition block and extract metadata. Returns a dict with: description, options, subcommands, arguments, alias If file_path is provided, imported subcommands will be resolved by parsing - the imported files. + the imported files. `block` may be passed to parse a specific pre-extracted + command body (used to recurse into locally-defined nested command groups). """ result = { 'description': '', @@ -357,15 +398,11 @@ def parse_command_block(content: str, file_path: Path | None = None) -> dict: } # Find the command block - command_match = re.search( - r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', - content - ) - if not command_match: + if block is None: + block = extract_exported_command_block(content) + if block is None: return result - block = command_match.group(1) - # Find where subcommands start first_subcommand_pos = block.find('.command(') if first_subcommand_pos == -1: @@ -451,12 +488,31 @@ def parse_command_block(content: str, file_path: Path | None = None) -> dict: 'name': cmd_name, 'description': imported_cmd.get('description', ''), 'arguments': imported_cmd.get('arguments', ''), - 'options': imported_cmd.get('options', []) + 'options': imported_cmd.get('options', []), + 'subcommands': imported_cmd.get('subcommands', []), }) continue except Exception as e: print(f" Warning: Could not parse imported command {second_arg}: {e}") cmd_desc = '' + elif second_arg and re.search( + r'const\s+' + re.escape(second_arg) + r'\s*=\s*new\s+Command\(\)', content + ): + # Locally-defined command group mounted as a subcommand + # (e.g. `.command("migrate", migrateCommand)`): recurse into its + # definition so its own subcommands/options are captured. + nested_block = extract_named_command_block(content, second_arg) + if nested_block is not None: + nested = parse_command_block(content, file_path, block=nested_block) + result['subcommands'].append({ + 'name': cmd_name, + 'description': nested.get('description', ''), + 'arguments': nested.get('arguments', ''), + 'options': nested.get('options', []), + 'subcommands': nested.get('subcommands', []), + }) + continue + cmd_desc = '' else: cmd_desc = '' @@ -628,6 +684,16 @@ def generate_cli_commands_markdown(cli_data: dict) -> str: for opt in sub['options']: md += f" - `{opt['flag']}` - {opt['description']}\n" + # Nested sub-subcommands (e.g. `datatable migrate new`) + for subsub in sub.get('subcommands', []): + ss_args = f" {subsub['arguments']}" if subsub.get('arguments') else "" + md += f" - `{cmd['name']} {sub_name} {subsub['name']}{ss_args}`" + if subsub.get('description'): + md += f" - {subsub['description']}" + md += "\n" + for opt in subsub.get('options', []): + md += f" - `{opt['flag']}` - {opt['description']}\n" + md += "\n" return md