From af019f26faf4a84a94b5cd91c76a9bd24fcf918b Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:25:08 +0200 Subject: [PATCH] feat(hub-projects): generate and apply datatable migrations on project publish/install (#9977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add datatable_migrations table * feat: add route to run datatable migrations * feat: sync datatable migrations as .up.sql/.down.sql files * feat: add datatable migrate up/down commands and post-push run prompt * feat: add datatable migrate new command to scaffold migrations * feat: add datatable migrations management UI * feat: prompt to create migration on DDL in datatable SQL editors * feat: support running a single specific datatable migration * feat: view migration content, run single migration, fix stacked modal * feat: per-row revert button with out-of-order warning * fix: avoid migrations list flicker on refresh after an action * feat: generate initial datatable migration via pg_dump * fix: surface datatable migration API error details in toasts * fix: revert created migration if create-and-run fails to run * fix: include postgres error detail in migration run/rollback failures * feat: sync datatable migrations as files via the workspace export * refactor: move datatable migrations to migrations/datatable/ path * fix: drop redundant datatable_migration label in sync output * fix: exclude datatable migration sql files from script metadata generation * feat: run datatable migrations as user-permissioned labeled jobs * feat: reject invalid datatable migrations on sync push * feat: datatable migrate up/down default to all datatables, --datatable to target one * fix: surface postgres error detail when datatable migrations fail to run * chore: regenerate CLI docs for datatable migrate commands * feat: default new datatable migration to a BEGIN/END transaction template * fix: validate datatable migration name and datatable at the API boundary * fix: ensure detected DDL ends with semicolon when wrapped in transaction * fix: re-prompt instead of stripping DDL when new-migration modal is cancelled * feat: refresh datatable schema after running a migration from the SQL REPL * feat: record db manager DDL on data tables as migrations * feat: make datatable migrations opt-in per data table * fix: make migration view editor read-only so its code can scroll * fix: don't re-prompt DDL guard when creating a migration without running * feat: generate down migrations for db manager DDL (postgres) * fix: correct down migration for db manager alters (no double-wrap, serial) * feat: explain migrations purpose with a tooltip in the migrations modal * compare paeg * feat: add datatable_migration kind to workspace diff pipeline * chore: point ee-repo-ref at datatable_migration git-sync companion * fix: harden datatable migration version allocation and initial-migration bookkeeping, add tests * feat: deploy and run datatable migrations on workspace merge Co-Authored-By: Claude Opus 4.8 (1M context) * Refactor + handle datatable setting delete/rename * refactor: move datatable migration rename/delete cascade into module Co-Authored-By: Claude Opus 4.8 (1M context) * chore(windmill-utils-internal): bump to 1.7.1 for datatable migration deploy provider methods Co-Authored-By: Claude Opus 4.8 (1M context) * feat(db-manager): add Migrations button to top bar, make Refresh icon-only Co-Authored-By: Claude Opus 4.8 (1M context) * BEGIN/END placeholder in down migration * feat: autofocus migration name input and flag it red when empty Co-Authored-By: Claude Opus 4.8 (1M context) * feat(datatable-migrations): allow non-admins to create/run/revert migrations, gate only opt in/out Co-Authored-By: Claude Opus 4.8 (1M context) * border nits * refresh db manager schema on migrations * BEGIN/END scaffold in CLI * feat(cli): push local datatable migrations before running on migrate up Co-Authored-By: Claude Opus 4.8 (1M context) * feat: flag invalid migration name with red border, not just empty Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop random slug from auto-generated migration names Co-Authored-By: Claude Opus 4.8 (1M context) * feat: offer revert-and-delete when deleting an installed migration Co-Authored-By: Claude Opus 4.8 (1M context) * feat: record fork merge as a migration when target datatable opts in * nit * clone migrations on fork * windmill-utils-internal * fix(datatable-migrations): serialize run/rollback with a per-db advisory lock Co-Authored-By: Claude Opus 4.8 (1M context) * fix(db-manager): fail closed when migrations-status check errors on DDL apply Co-Authored-By: Claude Opus 4.8 (1M context) * docs: fix generate_initial migration ordering comment to match code * chore(datatable-migrations): remove unused update_datatable_migrations endpoint Co-Authored-By: Claude Opus 4.8 (1M context) * fix: run DDL migration guard on the script editor Test button Co-Authored-By: Claude Opus 4.8 (1M context) * split * ee-repo-ref * chore(frontend): sync package-lock with package.json (@emnapi deps) Co-Authored-By: Claude Opus 4.8 (1M context) * fix(datatable-migrations): never resolve instance credentials into migration job args datatable_database_arg eagerly resolved instance data-table credentials (including the shared instance-wide Postgres password) and passed them as the migration job's plaintext `database` arg, landing in v2_job.args. Since the run route has no admin gate, a non-admin could run a migration and read args.database to recover the password, granting cross-workspace psql access to all instance data-table DBs. Pass a `datatable://` reference for both resource-backed and instance data tables instead; the pg executor already resolves it to real credentials server-side at run time, so nothing sensitive is ever stored in the job args. Co-Authored-By: Claude Opus 4.8 (1M context) * nit * fix: handle dollar-quoting and comments when splitting SQL statements * feat: deploy datatable migrations on merge with explicit opt-in error * fix(frontend): sync package-lock with npm 11 peer-dep resolution npm ci failed with 'Missing: @emnapi/core@1.11.2 / @emnapi/runtime@1.11.2 from lock file'. @napi-rs/wasm-runtime declares @emnapi/core|runtime ^1.7.1 as peerDependencies while @rolldown/binding-wasm32-wasi pins them to exactly 1.10.0. Newer npm (bundled with node 24 in CI) installs the peer deps at the highest match (1.11.2) alongside rolldown's nested 1.10.0, so the ideal tree needs both versions; the committed lock only had 1.10.0. Regenerate the lock with npm 11.18 so it carries both 1.11.2 (top-level, for the peer deps) and 1.10.0 (nested, for rolldown's pin). Verified npm ci passes. Co-Authored-By: Claude Opus 4.8 (1M context) * nit npm publish * fix: fail closed on migrations-status error in fork schema merge * nit CI emnapi/core version * prevent initial_datatable_migration if migrations already exist * fix(datatable-migrations): validate persisted data table names as path segments edit_datatable_config only validated rename segments, not the actual settings.datatables keys, so a data table could be saved directly under a name like '..' or one containing '/'. Since new tables default to migrations_enabled = true, generate_initial_datatable_migration would then insert a migration row and the sync export would build migrations/datatable//... paths from that name, producing malformed or directory-escaping export paths. Validate every persisted data table name in edit_datatable_config (alongside the existing rename checks) and add validate_datatable_path_segment to generate_initial_datatable_migration for defense in depth. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: scope datatable _wm_migrations by data table and cascade renames/deletes Co-Authored-By: Claude Opus 4.8 (1M context) * fix(system_prompts): resolve nested local command groups in CLI docs generator The CLI docs generator anchored on the first `new Command()` in a file and never resolved locally-defined command groups passed as `.command("name", localCmd)`. For datatable this flattened the nested `migrate` group: it emitted `datatable new/up/down` plus a bare `datatable migrate`, and mislabeled the datatable command with the migrate group's description. jobs was broken the same way (its description was pull's, and pull/push rendered empty). Anchor block extraction on the `export default`ed command, recurse into locally-defined `const x = new Command()` groups mounted as subcommands, and render nested sub-subcommands. Regenerated docs now show `datatable migrate new/up/down` and `jobs pull/push` with their real options. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop unreleased _wm_migrations legacy-upgrade handling Co-Authored-By: Claude Opus 4.8 (1M context) * fix: return datatable migration SQL from getItemValue for the diff drawer Co-Authored-By: Claude Opus 4.8 (1M context) * chore(frontend): use windmill-utils-internal 1.8.2 for migration diff drawer Co-Authored-By: Claude Opus 4.8 (1M context) * nit * nit * fix: handle datatable migration renames on push and dedupe timestamps * fix: reject rewriting an already-applied datatable migration on upsert Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): add missing @emnapi/core and @emnapi/runtime lockfile entries Resolves npm ci EUSAGE failure: the optional cpu:wasm32 @rolldown/binding-wasm32-wasi declares deps on @emnapi/core@1.11.2 and @emnapi/runtime@1.11.2 that had no resolved lockfile entries. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(cli): datatable migrate up/down default to main datatable, not all Co-Authored-By: Claude Opus 4.8 (1M context) * fix: fail closed when applied status unreadable on datatable migration rewrite Co-Authored-By: Claude Opus 4.8 (1M context) * fix: surface full error detail in Database Manager DDL/query errors * "See migration" button in the toast * feat: add Enter shortcut to Create-a-migration in the DDL guard * fix(frontend): warn before running a newly-created datatable migration out of order The row-level Run action warns when earlier migrations are still pending, but the create-and-run paths ran a just-created migration with `only` directly, applying it ahead of older pending migrations without that confirmation. Reuse the same "Run migration out of order" confirmation across all create-and-run paths via a shared helper (datatableMigrationUtils): - NewDataTableMigrationModal "Create and run" (and the DDL guard path) - DatatableSchemaDiff fork→parent merge - dbOps schema ops (DB manager create/alter/drop) — the pure factory throws a MigrationRunCancelled sentinel on decline, which DBTableEditor treats as a silent cancel Co-Authored-By: Claude Opus 4.8 (1M context) * fix: keep renamed datatable migrations visible in compare view * fix: record per-migration deployment on datatable migrations disable * fix(cli): run deployed datatable migrations after workspace merge The merge command upserted datatable_migration definitions into the target workspace and reported the item as successfully deployed, but never ran the migrations. For forked datatables backed by separate databases, this left the target schema unchanged until someone manually ran `wmill datatable migrate up`, while the CLI reported a successful merge. Collect the datatable migrations deployed (not deleted) into the target and, after the deploy loop, offer to run them via the existing offerToRunNewMigrations helper — the same post-deploy run prompt the push/sync path uses (interactive only; `--yes`/non-TTY skip the mutating run, matching push behavior). Export parseDatatableMigrationDeployPath so the merge path can parse the deployed items. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(backend): serialize datatable migration edits/deletes with the run lock A migration run snapshots a migration's code_up from datatable_migrations and only records its version in the data table's _wm_migrations after the job succeeds. upsert_datatable_migration checked _wm_migrations before allowing an edit but took no lock, so a concurrent edit could read "not applied yet", rewrite code_up/code_down, and then the in-flight run would record the version for the old SQL — leaving _wm_migrations pointing at SQL that was never applied (migrate up then skips it; rollback runs a down that doesn't match). Serialize definition rewrites and deletes with the same per-database advisory lock the run/rollback paths use: - Factor the connect+advisory-lock into lock_datatable_migration_runs and the applied-versions read into read_applied_versions_on_client. - run_datatable_migrations now snapshots the definitions AFTER taking the lock, so code_up can't change between snapshot and version-record. - upsert (when changing an existing def) and delete take the lock across the applied-check and the write; delete now rejects deleting an already-applied migration (would orphan its _wm_migrations record), symmetric with upsert. Both fail closed if the data table database is unreachable. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): stack the out-of-order migration confirm above the DB editor preview Creating a table on a migrations-enabled data table opened the DB table editor's "Confirm running the following" preview modal, whose confirm triggers applyDdl, which then asks for out-of-order confirmation. Both are ConfirmationModals with a hardcoded z-[9999]; the out-of-order one lives in DBManagerContent (mounted before the editor), so it rendered behind the still-open preview modal. Add an optional zIndexClass prop to ConfirmationModal (default z-[9999], backward-compatible) and give the DB-manager out-of-order confirm z-[10000] so it stacks on top. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(hub-projects): generate and apply datatable migrations for projects Detect datatable assets in a project's scripts/flows/raw apps when publishing to the Hub, generate a best-effort CREATE TABLE migration per data table from the source workspace's live schema, and let the publisher edit/toggle them in the bundle drawer. On import, offer to run the shipped migrations: recorded (datatable_migrations + _wm_migrations) when the target data table opted into migrations, otherwise as a one-off preview job. Missing target data tables are surfaced and skipped. - backend: POST /hub/migrations proxy forwarding to the Hub - frontend publish: projectMigrations.ts detection + generation, new "Data table migrations" section in DeployToHub - frontend import: run/skip modal + missing-datatable confirmation - extract pure SQL-gen from DatatableSchemaDiff.svelte into datatableSchemaSql.ts so plain .ts modules can import it Co-Authored-By: Claude Opus 4.8 (1M context) * feat(hub-projects): close datatable migration table set over foreign keys Pull a referenced table's FK targets into the generated migration transitively, so it creates every table it references (ordered by FK dependency), and drop any FK whose target still isn't in the set so the generated SQL always runs. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(hub-projects): show Data table dependencies in the publish view Detect data table usage off the predeploy bundle preview and surface it as a "Data table dependencies" summary right after "Resource dependencies", mirroring how resource types and triggers are shown. The editable migration itself stays in the bundle drawer. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(hub-projects): explain un-generated migrations with SQL comments When a table can't be found in the schema, a data table is referenced as a whole, or the schema can't be loaded, write a `--` comment describing the problem into the migration instead of leaving it blank. Partial migrations keep the CREATE TABLEs that did generate and comment the rest; comment-only migrations stay disabled. The bundle drawer now always shows the SQL box so those comments are visible and editable. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(hub-projects): review/edit migrations on import + rollback down migration Replace the plain "run migrations?" confirmation with a review drawer that previews each runnable migration, lets the user edit the SQL and toggle which to run, before the import proceeds. When recording an imported migration, also record a down migration (DROP TABLE of the created tables, in reverse order) derived from the up SQL, so it can be rolled back; the derived rollback is previewed in both the publish bundle drawer and the import review drawer. Co-Authored-By: Claude Opus 4.8 (1M context) * nit * feat(hub-projects): editable Up/Down Monaco editor for migrations Replace the plain textarea with a Monaco SQL editor split into Up/Down tabs. The down migration is now generated once as best-effort (DROP TABLE in reverse creation order) and is fully editable — no longer parsed back out of the up SQL. The down is threaded through publish → Hub → import (new project_migration.sql_down) and recorded as code_down when an imported migration is applied. - projectMigrations: GeneratedMigration.sql_down generated from the table set - MigrationSqlEditor.svelte: shared Up/Down tabbed Monaco editor (re-keyed on regeneration since Monaco ignores external code changes) - DeployToHub + install review drawer use it; sql_down pushed/applied - backend: PublishMigrationBody carries sql_down Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): generate CREATE TABLE IF NOT EXISTS for project migrations The FK closure pulls a referenced table's parents into the same transaction (e.g. `orders` drags in `customers`); those shared parents often already exist in the target, so a plain CREATE TABLE aborted the whole migration on the first collision. Emit CREATE TABLE IF NOT EXISTS for project migrations (via a new opt-in flag on generateMigrationSql, leaving the schema-diff behavior unchanged) so a pre-existing parent is skipped instead of failing. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): key FK ordering by schema-qualified table name orderByFkDependency keyed its dependency graph by bare table name (and resolved FK targets with .split('.').pop()), so two same-named tables in different schemas collapsed and one was dropped from the ordered set and never created. Key by schema.table like the rest of the pipeline, resolving FK targets through resolveTable. Also let resolveTable fall back to the bare table name when a schema-qualified ref's schema doesn't match. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): comment out generated down-migration DROP statements The generated down migration listed DROP TABLE for every table in the FK closure, including shared parent tables that may have pre-existed in the target — a rollback could drop a table the project never created (data loss). Emit all DROP statements commented out with a note, so nothing is dropped by default; the publisher uncomments the tables this migration actually owns. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): disable Import button during migration review planMigrations awaits the review / missing-datatable modals before setting installing = true, so the Import button stayed enabled during review and a second click launched a concurrent install() (second review drawer, duplicated item creation). Track a planningMigrations flag, disable the button on it, and early-return install() if already installing or planning. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): toast when migration generation fails regenerateMigrations cleared the drafts on error, showing "No data table usage detected" — indistinguishable from a genuine schema-load failure. Add a toast on the catch so the publisher can tell the two apart. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): honor cancel on the missing-data-table warning planMigrations awaited missingDatatableModal.ask() but ignored its boolean, so cancelling the "some data tables are missing" warning still proceeded with the import — the cancel affordance did nothing. Show the warning first and abort the whole import when the user cancels (planMigrations returns null; install() early-returns), so they can create the data table(s) and re-run. Confirming still imports without the missing migrations. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(hub-projects): detect data tables from low-code app DB-table config Low-code apps don't carry a persisted asset list, but the DB-table component declares its data table and table explicitly: a `oneOf` `type` config with `selected === 'datatable'` holding `datatable://` and the table. Walk the app value for those configs so an app that reads a data table is picked up by the Data table dependencies detection. Co-Authored-By: Claude Opus 4.8 (1M context) * Revert "feat(hub-projects): detect data tables from low-code app DB-table config" This reverts commit 9c43ebd5126276e9b436b5f7d2d9d94683b999e1. * fix(hub-projects): detect data tables from full-code apps' declaration Full-code (raw) apps explicitly declare the data tables/tables they use in value.data.tables (refs like main/customers or main/schema:table), which the "Data table dependencies" detection missed — it only looked at inline-script assets. Read the declaration via extractDataConfig/parseDataTableRef. The bundler previously dropped value.data (kept only files + runnables); include it so detection sees it and the imported app keeps its declaration, and pass it through on import. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(hub-projects): recompute app policy on project import Apps imported from a Hub project were created with an empty triggerables_v2 policy, so running any inline component script failed at runtime with "Path rawscript/ forbidden by policy". The policy is computed client-side on deploy and stored verbatim by the backend, and import skipped that step; retargeting also rewrites inline-script content (changing its sha), so a copied policy would not match either. Recompute the policy from the retargeted value at import, mirroring the deploy path: updatePolicy for grid apps, updateRawAppPolicy for raw apps, defaulting execution_mode to publisher. Co-Authored-By: Claude Opus 4.8 (1M context) * nit fix --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-api/src/hub_publish.rs | 35 ++ .../lib/components/DatatableSchemaDiff.svelte | 198 +--------- .../AssetGraph/AssetGraphDetailsPane.svelte | 7 +- .../src/lib/components/datatableSchemaSql.ts | 191 ++++++++++ .../workspaceSettings/DeployToHub.svelte | 158 +++++++- .../MigrationSqlEditor.svelte | 40 ++ .../projectMigrations.test.ts | 257 +++++++++++++ .../workspaceSettings/projectMigrations.ts | 349 ++++++++++++++++++ .../(logged)/projects/install/+page.svelte | 276 +++++++++++++- 9 files changed, 1301 insertions(+), 210 deletions(-) create mode 100644 frontend/src/lib/components/datatableSchemaSql.ts create mode 100644 frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts create mode 100644 frontend/src/lib/components/workspaceSettings/projectMigrations.ts diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index b653c7cca3..165dd566b5 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -31,6 +31,7 @@ pub fn workspaced_service() -> Router { .route("/resource_types", post(publish_resource_type)) .route("/resources", post(publish_resources)) .route("/triggers", post(publish_triggers)) + .route("/migrations", post(publish_migrations)) .route("/projects/{slug}/export", get(get_project_export)) .route("/projects/{slug}/submit", post(submit_project)) .route("/project", get(get_project_by_source)) @@ -425,6 +426,40 @@ async fn publish_triggers( .await } +// One best-effort data table migration attached to a project (per data table). +#[derive(Deserialize, Serialize)] +struct PublishMigrationBody { + datatable_name: String, + sql: String, + #[serde(default)] + sql_down: String, + enabled: bool, +} + +#[derive(Deserialize, Serialize)] +struct PublishMigrationsBody { + migrations: Vec, + project_slug: String, +} + +async fn publish_migrations( + authed: ApiAuthed, + tokened: Tokened, + Path(workspace): Path, + Query(scope): Query, + Json(body): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + validate_project_slug(&body.project_slug)?; + forward_to_hub( + &format!("/projects/{}/migrations", body.project_slug), + &source_key(&workspace, &scope.folder)?, + &tokened.token, + &body, + ) + .await +} + async fn get_project_export( authed: ApiAuthed, tokened: Tokened, diff --git a/frontend/src/lib/components/DatatableSchemaDiff.svelte b/frontend/src/lib/components/DatatableSchemaDiff.svelte index 9eccda8195..ac215747e0 100644 --- a/frontend/src/lib/components/DatatableSchemaDiff.svelte +++ b/frontend/src/lib/components/DatatableSchemaDiff.svelte @@ -1,195 +1,11 @@ - - + +
+
+ {#each [{ id: 'up', label: 'Up' }, { id: 'down', label: 'Down' }] as t (t.id)} + + {/each} +
+ {#key generation} +
+ {#if tab === 'up'} + + {:else} + + {/if} +
+ {/key} +
diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts new file mode 100644 index 0000000000..ce972f8e31 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts @@ -0,0 +1,257 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// inferAssets loads WASM; stub it so script detection is deterministic and no +// wasm init runs in the test. +const inferAssetsMock = vi.fn() +vi.mock('$lib/infer', () => ({ inferAssets: (...a: any[]) => inferAssetsMock(...a) })) + +// Only getDatatableFullSchema is used by the generator; stub the whole service. +const getDatatableFullSchemaMock = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableFullSchema: (...a: any[]) => getDatatableFullSchemaMock(...a) + } +})) + +import { detectDatatableTables, generateDatatableMigrations } from './projectMigrations' +import type { FetchedItem } from './projectBundle' + +describe('detectDatatableTables', () => { + beforeEach(() => inferAssetsMock.mockReset()) + + it('collects datatable/table refs from scripts (re-parsed), flows and raw apps', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [ + { kind: 'datatable', path: 'main/customers' }, + { kind: 'resource', path: 'u/admin/pg' } // ignored + ] + }) + const items: FetchedItem[] = [ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'select 1' }, + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/orders' }] + } + } + ] + } + }, + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: { + r1: { inlineScript: { assets: [{ kind: 'datatable', path: 'analytics/events' }] } } + } + }) + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])].sort()).toEqual(['customers', 'orders']) + expect([...(usage.get('analytics') ?? [])]).toEqual(['events']) + }) + + it('records a datatable used with no specific table', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [{ kind: 'datatable', path: 'main' }] + }) + const usage = await detectDatatableTables([ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'x' } + ]) + expect(usage.has('main')).toBe(true) + expect(usage.get('main')?.size).toBe(0) + }) + + it('reads a full-code app’s explicit data.tables declaration', async () => { + const items: FetchedItem[] = [ + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: {}, + data: { + datatable: 'main', + schema: 'app1', + tables: ['main/customers', 'main/app1:orders'] + } + }) + } + ] + const usage = await detectDatatableTables(items) + // public-schema ref keeps the bare name; non-public keeps schema.table. + expect([...(usage.get('main') ?? [])].sort()).toEqual(['app1.orders', 'customers']) + }) +}) + +describe('generateDatatableMigrations', () => { + beforeEach(() => getDatatableFullSchemaMock.mockReset()) + + const schema = { + public: { + customers: { + name: 'customers', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'email', datatype: 'text', nullable: true } + ], + foreign_keys: [] + }, + orders: { + name: 'orders', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'customer_id', datatype: 'integer', nullable: false } + ], + foreign_keys: [ + { + target_table: 'public.customers', + columns: [{ source_column: 'customer_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + + it('creates referenced tables in FK-dependency order in one transaction, enabled', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders', 'customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + const m = migrations[0] + expect(m.datatable_name).toBe('main') + expect(m.enabled).toBe(true) + expect(m.sql.startsWith('BEGIN;')).toBe(true) + expect(m.sql.trimEnd().endsWith('COMMIT;')).toBe(true) + // customers (FK target) must be created before orders (FK source). + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + // A single wrapping transaction, not one per table. + expect(m.sql.match(/BEGIN;/g)?.length).toBe(1) + // Idempotent: won't abort if a pulled-in parent already exists in the target. + expect(m.sql).toContain('CREATE TABLE IF NOT EXISTS "public"."customers"') + // Down migration lists drops commented out (nothing dropped by default), + // in reverse order: orders (child) before customers (parent). + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."orders";') + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."customers";') + // No uncommented DROP TABLE anywhere. + expect(/^\s*DROP TABLE/m.test(m.sql_down)).toBe(false) + expect(m.sql_down.indexOf('"public"."orders"')).toBeLessThan( + m.sql_down.indexOf('"public"."customers"') + ) + }) + + it('accepts schema-qualified table refs', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['public.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + }) + + it('keeps same-named tables from different schemas both created', async () => { + const twoSchemas = { + public: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + }, + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(twoSchemas) + const usage = new Map([['main', new Set(['public.customers', 'app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('"app"."customers"') + }) + + it('transitively pulls in FK-referenced tables not directly used', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + // Only `orders` is referenced; `customers` (its FK target) must still be + // created, and before `orders`. + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const m = migrations[0] + expect(m.enabled).toBe(true) + expect(m.sql).toContain('"public"."customers"') + expect(m.sql).toContain('"public"."orders"') + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + }) + + it('drops a foreign key whose target is not in the schema', async () => { + // `orders` references a `warehouses` table that no longer exists in the + // schema: the FK must be pruned so the migration still runs. + const schemaWithDanglingFk = { + public: { + orders: { + name: 'orders', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [ + { + target_table: 'public.warehouses', + columns: [{ source_column: 'id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(schemaWithDanglingFk) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."orders"') + expect(migrations[0].sql).not.toContain('warehouses') + }) + + it('emits a disabled comment entry when a referenced table is not found', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['nonexistent'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('-- Table "nonexistent" is referenced but was not found') + expect(migrations[0].sql).not.toContain('BEGIN;') + }) + + it('keeps found tables and comments the missing ones in one migration', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['customers', 'ghost'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('-- Table "ghost" is referenced but was not found') + // Comments precede the runnable transaction. + expect(migrations[0].sql.indexOf('-- Table "ghost"')).toBeLessThan( + migrations[0].sql.indexOf('BEGIN;') + ) + }) + + it('comments a data table used with no specific table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set()]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('no specific table was referenced') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts new file mode 100644 index 0000000000..9ff80c32eb --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts @@ -0,0 +1,349 @@ +// Best-effort data table migration generation for the "project = folder" Hub +// bundle. Detects which data tables (and tables within them) a project's +// scripts/flows/raw apps reference via `datatable` assets, then generates a +// `CREATE TABLE` bundle per data table from the source workspace's live schema, +// so importing the project into another workspace can recreate those tables. +// +// Best-effort by design: the generated SQL is shown to the publisher and is +// fully editable before publishing. Low-code (non-raw) apps have no persisted +// asset list and are not scanned. + +import { inferAssets } from '$lib/infer' +import type { SupportedLanguage } from '$lib/common' +import { getAllModules } from '$lib/components/flows/flowExplorer' +import { getFlowModuleAssets } from '$lib/components/assets/lib' +import { extractDataConfig, parseDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' +import { + apiSchemaToEditorSchema, + generateMigrationSql, + type DatabaseSchema +} from '$lib/components/datatableSchemaSql' +import { WorkspaceService } from '$lib/gen' +import type { FetchedItem } from './projectBundle' + +export interface GeneratedMigration { + datatable_name: string + /** Up migration: creates the tables. */ + sql: string + /** Down migration: drops the created tables. Best-effort, generated once and + * editable by the publisher (not re-derived from `sql`). */ + sql_down: string + enabled: boolean +} + +// A datatable asset path is `datatable`, `datatable/table`, or +// `datatable/schema.table` (see the SQL asset parser). The first segment is the +// data table name; the remainder identifies a specific table (absent = whole +// data table, no table to create). +function parseDatatableAssetPath(path: string): { datatable: string; table?: string } { + const slash = path.indexOf('/') + if (slash === -1) return { datatable: path } + const datatable = path.slice(0, slash) + const table = path.slice(slash + 1).trim() + return { datatable, table: table || undefined } +} + +function addDatatableTable( + map: Map>, + datatable: string, + table: string | undefined +): void { + if (!datatable) return + const set = map.get(datatable) ?? new Set() + if (table) set.add(table) + map.set(datatable, set) +} + +function addUsage(map: Map>, path: string): void { + const { datatable, table } = parseDatatableAssetPath(path) + addDatatableTable(map, datatable, table) +} + +/** + * Scan a project's fetched items for data table usage and return + * `datatable -> set of table refs` (a table ref is `table` or `schema.table`). + * - scripts: re-parse the code with the asset parser (`inferAssets`) + * - flows: read each module's stored `assets` + * - full-code (raw) apps: read the explicit `data.tables` declaration; fall back + * to `runnables[key].inlineScript.assets` for older apps + */ +export async function detectDatatableTables( + items: FetchedItem[] +): Promise>> { + const map = new Map>() + + for (const item of items) { + if (item.kind === 'script') { + const res = await inferAssets( + item.language as SupportedLanguage | undefined, + item.content ?? '' + ) + if (res.status === 'ok') { + for (const a of res.assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'flow') { + for (const mod of getAllModules(item.value?.modules ?? [], item.value?.failure_module)) { + const assets = getFlowModuleAssets(mod) + if (assets) for (const a of assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'raw_app') { + let parsed: any + try { + parsed = JSON.parse(item.content ?? '{}') + } catch { + continue + } + // Full-code apps explicitly declare the data tables/tables they use + // (`data.tables`, refs like `main/customers` or `main/schema:table`), so + // read that rather than parsing assets. + const config = extractDataConfig(parsed) + if (config) { + for (const ref of config.tables) { + const r = parseDataTableRef(ref) + const table = r.table + ? r.schema && r.schema !== 'public' + ? `${r.schema}.${r.table}` + : r.table + : undefined + addDatatableTable(map, r.datatable, table) + } + } + // Older raw apps instead carry datatable usage as inline-script assets. + const runnables = parsed?.runnables ?? {} + for (const key of Object.keys(runnables)) { + const assets = runnables[key]?.inlineScript?.assets + if (Array.isArray(assets)) + for (const a of assets) + if (a?.kind === 'datatable' && typeof a.path === 'string') addUsage(map, a.path) + } + } + } + return map +} + +// Resolve a table ref (`table` or `schema.table`) to a concrete +// `{ schemaName, tableName }` present in the live schema, or undefined if the +// table can't be found (dropped since, typo, …). +function resolveTable( + schema: DatabaseSchema, + tableRef: string +): { schemaName: string; tableName: string } | undefined { + const dot = tableRef.indexOf('.') + if (dot !== -1) { + const schemaName = tableRef.slice(0, dot) + const tableName = tableRef.slice(dot + 1) + if (schema[schemaName]?.[tableName]) return { schemaName, tableName } + } + // No schema qualifier (or the qualified lookup missed, e.g. a stale schema name): + // find the bare table name across every schema, first match wins. + const bareName = dot !== -1 ? tableRef.slice(dot + 1) : tableRef + for (const schemaName of Object.keys(schema)) { + if (schema[schemaName][bareName]) return { schemaName, tableName: bareName } + } + return undefined +} + +type ResolvedTable = { schemaName: string; tableName: string } + +const tableKey = (t: ResolvedTable) => `${t.schemaName}.${t.tableName}` + +// Grow the set of tables to create so it's closed under foreign keys: a used +// table's FK targets (and their FK targets, transitively) are pulled in, so the +// generated CREATE TABLEs never reference a table that isn't also created. FK +// targets that don't resolve in this schema are left out (their FK is pruned by +// pruneSchemaForTables). +function expandFkClosure(schema: DatabaseSchema, seed: ResolvedTable[]): ResolvedTable[] { + const inSet = new Map(seed.map((t) => [tableKey(t), t])) + const queue = [...seed] + while (queue.length > 0) { + const t = queue.shift()! + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && !inSet.has(tableKey(target))) { + inSet.set(tableKey(target), target) + queue.push(target) + } + } + } + return [...inSet.values()] +} + +// A copy of the schema restricted to `tables`, with each table's foreign keys +// filtered to targets that are also in `tables`. generateMigrationSql emits every +// FK it finds on a table, so pruning here keeps a stray FK (to a table outside the +// migration) from making the generated SQL fail. +function pruneSchemaForTables(schema: DatabaseSchema, tables: ResolvedTable[]): DatabaseSchema { + const inSet = new Set(tables.map(tableKey)) + const pruned: DatabaseSchema = {} + for (const t of tables) { + const orig = schema[t.schemaName]?.[t.tableName] + if (!orig) continue + ;(pruned[t.schemaName] ??= {})[t.tableName] = { + ...orig, + foreignKeys: (orig.foreignKeys ?? []).filter((fk) => { + const target = resolveTable(schema, fk.targetTable ?? '') + return target != null && inSet.has(tableKey(target)) + }) + } + } + return pruned +} + +// Order tables so a table is created after the in-set tables it references via a +// foreign key. Keyed by schema-qualified name (like the rest of the pipeline) so +// two same-named tables in different schemas aren't collapsed. Falls back to input +// order on a cycle so generation never hangs. +function orderByFkDependency(schema: DatabaseSchema, tables: ResolvedTable[]): ResolvedTable[] { + const inSet = new Set(tables.map(tableKey)) + const deps = new Map>() + for (const t of tables) { + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + const targets = new Set() + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && tableKey(target) !== tableKey(t) && inSet.has(tableKey(target))) { + targets.add(tableKey(target)) + } + } + deps.set(tableKey(t), targets) + } + const ordered: ResolvedTable[] = [] + const done = new Set() + const visiting = new Set() + const byKey = new Map(tables.map((t) => [tableKey(t), t])) + const visit = (key: string) => { + if (done.has(key) || visiting.has(key)) return + visiting.add(key) + for (const dep of deps.get(key) ?? []) visit(dep) + visiting.delete(key) + done.add(key) + const t = byKey.get(key) + if (t) ordered.push(t) + } + for (const t of tables) visit(tableKey(t)) + return ordered +} + +// Pull a readable one-line message out of an API error for embedding in a SQL +// comment (collapse whitespace so it can't break out of the `--` line). +function errorText(e: any): string { + const body = e?.body + const raw = + typeof body === 'string' && body.trim() + ? body + : body && typeof body === 'object' + ? (body.error?.message ?? body.message ?? JSON.stringify(body)) + : (e?.message ?? String(e)) + return String(raw).replace(/\s+/g, ' ').trim() +} + +// Strip the per-table `BEGIN;`/`COMMIT;` wrapper that generateMigrationSql adds, +// so several tables can share one transaction. +function unwrapTransaction(sql: string): string { + return sql + .replace(/^\s*BEGIN;\s*\n?/, '') + .replace(/\n?\s*COMMIT;\s*$/, '') + .trim() +} + +/** + * Generate one best-effort migration per used data table. Resolved tables (plus + * the tables they depend on via foreign key, in FK-dependency order) become a + * single CREATE TABLE transaction, enabled by default. Anything that couldn't be + * auto-generated — a table not found in the schema, a data table referenced as a + * whole, or a schema that couldn't be loaded — is written as a `--` SQL comment + * describing the problem, so the publisher sees what's missing instead of a blank + * entry. A migration with no runnable statements (only comments) is left disabled. + */ +export async function generateDatatableMigrations( + workspace: string, + usage: Map> +): Promise { + const out: GeneratedMigration[] = [] + for (const [datatable, tableRefs] of usage) { + let schema: DatabaseSchema + try { + const api = await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatable}` } + }) + schema = apiSchemaToEditorSchema(api) + } catch (e) { + // Couldn't reach the schema at all: leave a commented stub explaining why, + // so the publisher can fill it in rather than seeing a silent blank. + out.push({ + datatable_name: datatable, + sql: + `-- Could not load the schema of data table "${datatable}": ${errorText(e)}\n` + + `-- Add the CREATE TABLE statement(s) for the tables this project uses.`, + sql_down: '', + enabled: false + }) + continue + } + // Resolve the referenced tables; record a comment for each one we can't find + // so a partial migration still explains what's missing. + const resolved: ResolvedTable[] = [] + const comments: string[] = [] + for (const ref of tableRefs) { + const t = resolveTable(schema, ref) + if (t) resolved.push(t) + else + comments.push( + `-- Table "${ref}" is referenced but was not found in data table "${datatable}"; add its CREATE TABLE manually.` + ) + } + if (tableRefs.size === 0) { + comments.push( + `-- Data table "${datatable}" is used but no specific table was referenced; nothing to generate automatically.` + ) + } + // Pull in the tables the referenced ones depend on via FK, then generate + // against a schema whose FKs are restricted to this set, so the migration + // creates everything it references and never emits a dangling FK. + const closure = expandFkClosure(schema, resolved) + const ordered = orderByFkDependency(schema, closure) + const prunedSchema = pruneSchemaForTables(schema, ordered) + const statements = ordered + .map((t) => + unwrapTransaction( + // IF NOT EXISTS: FK closure pulls in shared parent tables (e.g. a + // referenced `orders` drags in `customers`) that often already exist in + // the target, so a plain CREATE would abort the whole transaction. The + // caveat — an existing differently-shaped table is silently left as-is — + // is acceptable for a best-effort, editable migration. + generateMigrationSql( + { schemaName: t.schemaName, tableName: t.tableName, kind: 'added' }, + prunedSchema, + { ifNotExists: true } + ) + ) + ) + .filter((s) => s.length > 0) + // Comments (the errors) go on top; the CREATE TABLE transaction, if any, + // follows. Enabled only when there's something to run. + const parts: string[] = [] + if (comments.length > 0) parts.push(comments.join('\n')) + if (statements.length > 0) parts.push(`BEGIN;\n${statements.join('\n\n')}\nCOMMIT;`) + // Best-effort down migration: the DROP TABLE statements are commented out + // because the FK closure pulls in shared parent tables that may have + // pre-existed in the target (dropping them would lose data the project never + // created). The publisher uncomments the tables this migration should drop. + const drops = [...ordered] + .reverse() + .map((t) => `-- DROP TABLE IF EXISTS "${t.schemaName}"."${t.tableName}";`) + const sqlDown = + drops.length > 0 + ? `-- Rollback: uncomment the tables this migration should drop (leave shared\n` + + `-- tables that already existed in the workspace commented out).\nBEGIN;\n${drops.join('\n')}\nCOMMIT;` + : '' + out.push({ + datatable_name: datatable, + sql: parts.join('\n\n'), + sql_down: sqlDown, + enabled: statements.length > 0 + }) + } + return out.sort((a, b) => a.datatable_name.localeCompare(b.datatable_name)) +} diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte index 676ff57414..90201fd28c 100644 --- a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte @@ -3,7 +3,8 @@ import { goto } from '$app/navigation' import { workspaceStore, enterpriseLicense } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { Button } from '$lib/components/common' + import { Button, Drawer, DrawerContent } from '$lib/components/common' + import Toggle from '$lib/components/Toggle.svelte' import { ScriptService, FlowService, @@ -11,6 +12,7 @@ ResourceService, ScheduleService, FolderService, + WorkspaceService, HttpTriggerService, WebsocketTriggerService, KafkaTriggerService, @@ -29,9 +31,23 @@ rewriteFlowValue, rewriteRawAppContent } from '$lib/components/workspaceSettings/projectBundle' + import { updatePolicy } from '$lib/components/apps/editor/appPolicy' + import { updateRawAppPolicy } from '$lib/sharedUtils' + import type { App } from '$lib/components/apps/types' + import MigrationSqlEditor from '$lib/components/workspaceSettings/MigrationSqlEditor.svelte' + import { runScriptAndPollResult } from '$lib/components/jobs/utils' + import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' + import { createAsyncConfirmationModal } from '$lib/components/common/confirmationModal/asyncConfirmationModal.svelte' + import Portal from '$lib/components/Portal.svelte' import { Cloud, Download, Loader2 } from 'lucide-svelte' type ExportItem = Record + interface ProjectMigration { + datatable_name: string + sql: string + sql_down?: string + enabled: boolean + } interface ProjectExport { project: { slug: string; name: string; summary: string; readme: string | null } scripts: ExportItem[] @@ -39,6 +55,7 @@ apps: ExportItem[] resources: ExportItem[] triggers: ExportItem[] + migrations?: ProjectMigration[] } let slug = $derived($page.url.searchParams.get('hub') ?? '') @@ -48,10 +65,45 @@ let loadError = $state(undefined) let data = $state(undefined) let installing = $state(false) + // True while the migration review/missing-datatable modals are open, before the + // import spinner starts — keeps the Import button from launching a second import. + let planningMigrations = $state(false) let results = $state<{ path: string; ok: boolean; error?: string }[]>([]) let done = $state(false) let folderName = $state('') + // When the target lacks a needed data table, "import without that migration". + const missingDatatableModal = createAsyncConfirmationModal() + + // Migration review drawer: preview + edit each runnable migration's SQL and + // choose which to run, resolved linearly via `reviewResolve`. + let reviewDrawer = $state() + let reviewList = $state< + { datatable_name: string; sql: string; sql_down: string; run: boolean }[] + >([]) + // Bumped per review session so the Monaco editors re-mount with the new SQL. + let reviewGeneration = $state(0) + let reviewResolve: ((run: boolean) => void) | undefined + function openMigrationReview(migs: ProjectMigration[]): Promise { + reviewList = migs.map((m) => ({ + datatable_name: m.datatable_name, + sql: m.sql, + sql_down: m.sql_down ?? '', + run: true + })) + reviewGeneration++ + reviewDrawer?.openDrawer() + return new Promise((resolve) => (reviewResolve = resolve)) + } + function closeMigrationReview(run: boolean) { + // Capture + clear first so the `on:close` fired by closeDrawer() (which would + // call this again with run=false) can't override an explicit Run/Skip choice. + const resolve = reviewResolve + reviewResolve = undefined + reviewDrawer?.closeDrawer() + resolve?.(run) + } + let loadSeq = 0 $effect(() => { @@ -91,7 +143,10 @@ flows: data.flows.length, apps: data.apps.length, resources: data.resources.length, - triggers: data.triggers.length + triggers: data.triggers.length, + migrations: (data.migrations ?? []).filter( + (m) => m.enabled && (m.sql ?? '').trim() !== '' + ).length } : undefined ) @@ -119,8 +174,21 @@ ) } - // Minimal non-public policy for re-created apps. - const defaultPolicy = { execution_mode: 'publisher', triggerables_v2: {} } as any + // Recompute an app's execution policy from its (retargeted) value, mirroring + // what the editor does on deploy. `triggerables_v2` is keyed by + // `:rawscript/`; retargeting rewrites that + // content, so a copied or empty policy would leave every inline runnable + // "forbidden by policy" at runtime. Default to publisher (auth required). + async function computeAppPolicy(value: any): Promise { + const policy = (await updatePolicy(value as App, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy + } + async function computeRawAppPolicy(runnables: Record): Promise { + const policy = (await updateRawAppPolicy(runnables, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy + } // EE-only kinds; the rest (http, websocket, postgres, mqtt, email) work on CE. const EE_TRIGGER_KINDS = new Set(['kafka', 'nats', 'sqs', 'gcp', 'azure']) @@ -213,14 +281,130 @@ } } + // Decide which data table migrations to run. Migrations are keyed by data table + // name and applied only to a target data table of the same name. Returns the + // migrations to run (with any edits the user made), an empty array when there's + // nothing to run, or `null` when the user backs out of the whole import at the + // missing-data-table warning. + async function planMigrations( + workspace: string, + migrations: ProjectMigration[] + ): Promise { + const enabled = migrations.filter((m) => m.enabled && (m.sql ?? '').trim() !== '') + if (enabled.length === 0) return [] + + let present: Set + try { + const dts = await WorkspaceService.listDataTables({ workspace }) + present = new Set(dts.map((d) => d.name)) + } catch { + // Can't read the target's data tables — skip migrations rather than guess. + return [] + } + const runnable = enabled.filter((m) => present.has(m.datatable_name)) + const missingNames = [ + ...new Set(enabled.filter((m) => !present.has(m.datatable_name)).map((m) => m.datatable_name)) + ] + + // Warn about missing data tables first: confirming imports without their + // migrations, cancelling backs out of the whole import so the user can create + // the data table(s) and re-run. + if (missingNames.length > 0) { + const proceed = await missingDatatableModal.ask({ + title: 'Some data tables are missing', + confirmationText: 'Import without them', + children: `This project uses data table(s) "${missingNames.join( + '", "' + )}" that don't exist in this workspace, so their migrations will be skipped. To apply them, cancel, create the data table(s) with the same name in Workspace settings → Data tables, then re-run this import.` + }) + if (!proceed) return null + } + + let toRun: ProjectMigration[] = [] + if (runnable.length > 0) { + const run = await openMigrationReview(runnable) + if (run) { + toRun = reviewList + .filter((r) => r.run && r.sql.trim() !== '') + .map((r) => ({ + datatable_name: r.datatable_name, + sql: r.sql, + sql_down: r.sql_down, + enabled: true + })) + } + } + return toRun + } + + // Apply one migration to the target data table. If the data table opted into + // migrations, record it (datatable_migrations + _wm_migrations, run only this + // version); otherwise run the SQL once as a preview job (unrecorded). + async function applyOneMigration(workspace: string, m: ProjectMigration): Promise { + let recorded = false + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: m.datatable_name + }) + recorded = !!status.enabled + } catch {} + + if (recorded) { + // Record the shipped down migration (DROP the created tables) so it can be + // rolled back. + const codeDown = (m.sql_down ?? '').trim() + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName: m.datatable_name, + requestBody: { + name: `hub_import_${data?.project.slug ?? 'project'}`, + code_up: m.sql, + code_down: codeDown || undefined + } + }) + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: m.datatable_name, + only: created.timestamp + }) + } else { + await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }) + } + } + async function install() { // Snapshot reactive state up-front: `workspace` ($derived) and `data` // ($state, replaced by load()) can both change mid-import on a workspace // switch, which would split items or mix two exports. Pin both. + // Guard against a second click while the review modal is open (the Import + // button isn't `installing` yet during planning, so it would otherwise be + // clickable and start a concurrent import). + if (installing || planningMigrations) return const workspace = $workspaceStore const exportData = data if (!exportData || !workspace) return const folder = folderName.trim() || exportData.project.slug + + // Review data table migrations first (before the import spinner), so the user + // previews/edits and decides, then the whole import runs uninterrupted. + planningMigrations = true + let migrationsToRun: ProjectMigration[] | null + try { + migrationsToRun = await planMigrations(workspace, exportData.migrations ?? []) + } finally { + planningMigrations = false + } + // User backed out at the missing-data-table warning — abort the whole import. + if (migrationsToRun === null) return + installing = true results = [] done = false @@ -294,14 +478,21 @@ const css = files['/bundle.css'] ?? '' delete files['/bundle.js'] delete files['/bundle.css'] + const runnables = parsed.runnables ?? {} return AppService.createAppRaw({ workspace, formData: { app: { path: a.path, summary: a.summary ?? '', - value: { files, runnables: parsed.runnables ?? {} }, - policy: defaultPolicy + value: { + files, + runnables, + // Keep the full-code app's explicit data table declaration. + ...(parsed.data !== undefined ? { data: parsed.data } : {}), + ...(parsed.datatables !== undefined ? { datatables: parsed.datatables } : {}) + }, + policy: await computeRawAppPolicy(runnables) }, js, css @@ -312,15 +503,16 @@ } else { await record( a.path, - AppService.createApp({ - workspace, - requestBody: { - path: a.path, - summary: a.summary ?? '', - value: a.value, - policy: defaultPolicy - } - }) + (async () => + AppService.createApp({ + workspace, + requestBody: { + path: a.path, + summary: a.summary ?? '', + value: a.value, + policy: await computeAppPolicy(a.value) + } + }))() ) } } @@ -369,6 +561,13 @@ } } } + + // Apply the reviewed data table migrations after items exist. Each is + // recorded (or run as a preview job) per applyOneMigration. + for (const m of migrationsToRun) { + await record(`data table: ${m.datatable_name}`, applyOneMigration(workspace, m)) + } + done = true const failed = results.filter((r) => !r.ok).length sendUserToast( @@ -413,6 +612,9 @@ {counts?.apps} apps {counts?.resources} resources {counts?.triggers} triggers + {#if counts && counts.migrations > 0} + {counts.migrations} data table migrations + {/if}
{#if installing} @@ -458,3 +660,45 @@ {/if} {/if}
+ + + + + + closeMigrationReview(false)}> + closeMigrationReview(false)}> +
+

+ This project ships migrations that recreate the data tables it uses. Review and edit the + SQL, then choose which to run. A migration runs against the data table of the same name in + {workspace}; if that data table has migrations enabled it is + recorded, otherwise it runs once as a preview job. +

+ {#each reviewList as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ {#if m.run} + + {/if} +
+ {/each} +
+ {#snippet actions()} + + + {/snippet} + +