diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index b653c7cca3..165dd566b5 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -31,6 +31,7 @@ pub fn workspaced_service() -> Router { .route("/resource_types", post(publish_resource_type)) .route("/resources", post(publish_resources)) .route("/triggers", post(publish_triggers)) + .route("/migrations", post(publish_migrations)) .route("/projects/{slug}/export", get(get_project_export)) .route("/projects/{slug}/submit", post(submit_project)) .route("/project", get(get_project_by_source)) @@ -425,6 +426,40 @@ async fn publish_triggers( .await } +// One best-effort data table migration attached to a project (per data table). +#[derive(Deserialize, Serialize)] +struct PublishMigrationBody { + datatable_name: String, + sql: String, + #[serde(default)] + sql_down: String, + enabled: bool, +} + +#[derive(Deserialize, Serialize)] +struct PublishMigrationsBody { + migrations: Vec, + project_slug: String, +} + +async fn publish_migrations( + authed: ApiAuthed, + tokened: Tokened, + Path(workspace): Path, + Query(scope): Query, + Json(body): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + validate_project_slug(&body.project_slug)?; + forward_to_hub( + &format!("/projects/{}/migrations", body.project_slug), + &source_key(&workspace, &scope.folder)?, + &tokened.token, + &body, + ) + .await +} + async fn get_project_export( authed: ApiAuthed, tokened: Tokened, diff --git a/frontend/src/lib/components/DatatableSchemaDiff.svelte b/frontend/src/lib/components/DatatableSchemaDiff.svelte index 9eccda8195..ac215747e0 100644 --- a/frontend/src/lib/components/DatatableSchemaDiff.svelte +++ b/frontend/src/lib/components/DatatableSchemaDiff.svelte @@ -1,195 +1,11 @@ - - + +
+
+ {#each [{ id: 'up', label: 'Up' }, { id: 'down', label: 'Down' }] as t (t.id)} + + {/each} +
+ {#key generation} +
+ {#if tab === 'up'} + + {:else} + + {/if} +
+ {/key} +
diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts new file mode 100644 index 0000000000..ce972f8e31 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts @@ -0,0 +1,257 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// inferAssets loads WASM; stub it so script detection is deterministic and no +// wasm init runs in the test. +const inferAssetsMock = vi.fn() +vi.mock('$lib/infer', () => ({ inferAssets: (...a: any[]) => inferAssetsMock(...a) })) + +// Only getDatatableFullSchema is used by the generator; stub the whole service. +const getDatatableFullSchemaMock = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableFullSchema: (...a: any[]) => getDatatableFullSchemaMock(...a) + } +})) + +import { detectDatatableTables, generateDatatableMigrations } from './projectMigrations' +import type { FetchedItem } from './projectBundle' + +describe('detectDatatableTables', () => { + beforeEach(() => inferAssetsMock.mockReset()) + + it('collects datatable/table refs from scripts (re-parsed), flows and raw apps', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [ + { kind: 'datatable', path: 'main/customers' }, + { kind: 'resource', path: 'u/admin/pg' } // ignored + ] + }) + const items: FetchedItem[] = [ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'select 1' }, + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/orders' }] + } + } + ] + } + }, + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: { + r1: { inlineScript: { assets: [{ kind: 'datatable', path: 'analytics/events' }] } } + } + }) + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])].sort()).toEqual(['customers', 'orders']) + expect([...(usage.get('analytics') ?? [])]).toEqual(['events']) + }) + + it('records a datatable used with no specific table', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [{ kind: 'datatable', path: 'main' }] + }) + const usage = await detectDatatableTables([ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'x' } + ]) + expect(usage.has('main')).toBe(true) + expect(usage.get('main')?.size).toBe(0) + }) + + it('reads a full-code app’s explicit data.tables declaration', async () => { + const items: FetchedItem[] = [ + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: {}, + data: { + datatable: 'main', + schema: 'app1', + tables: ['main/customers', 'main/app1:orders'] + } + }) + } + ] + const usage = await detectDatatableTables(items) + // public-schema ref keeps the bare name; non-public keeps schema.table. + expect([...(usage.get('main') ?? [])].sort()).toEqual(['app1.orders', 'customers']) + }) +}) + +describe('generateDatatableMigrations', () => { + beforeEach(() => getDatatableFullSchemaMock.mockReset()) + + const schema = { + public: { + customers: { + name: 'customers', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'email', datatype: 'text', nullable: true } + ], + foreign_keys: [] + }, + orders: { + name: 'orders', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'customer_id', datatype: 'integer', nullable: false } + ], + foreign_keys: [ + { + target_table: 'public.customers', + columns: [{ source_column: 'customer_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + + it('creates referenced tables in FK-dependency order in one transaction, enabled', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders', 'customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + const m = migrations[0] + expect(m.datatable_name).toBe('main') + expect(m.enabled).toBe(true) + expect(m.sql.startsWith('BEGIN;')).toBe(true) + expect(m.sql.trimEnd().endsWith('COMMIT;')).toBe(true) + // customers (FK target) must be created before orders (FK source). + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + // A single wrapping transaction, not one per table. + expect(m.sql.match(/BEGIN;/g)?.length).toBe(1) + // Idempotent: won't abort if a pulled-in parent already exists in the target. + expect(m.sql).toContain('CREATE TABLE IF NOT EXISTS "public"."customers"') + // Down migration lists drops commented out (nothing dropped by default), + // in reverse order: orders (child) before customers (parent). + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."orders";') + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."customers";') + // No uncommented DROP TABLE anywhere. + expect(/^\s*DROP TABLE/m.test(m.sql_down)).toBe(false) + expect(m.sql_down.indexOf('"public"."orders"')).toBeLessThan( + m.sql_down.indexOf('"public"."customers"') + ) + }) + + it('accepts schema-qualified table refs', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['public.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + }) + + it('keeps same-named tables from different schemas both created', async () => { + const twoSchemas = { + public: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + }, + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(twoSchemas) + const usage = new Map([['main', new Set(['public.customers', 'app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('"app"."customers"') + }) + + it('transitively pulls in FK-referenced tables not directly used', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + // Only `orders` is referenced; `customers` (its FK target) must still be + // created, and before `orders`. + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const m = migrations[0] + expect(m.enabled).toBe(true) + expect(m.sql).toContain('"public"."customers"') + expect(m.sql).toContain('"public"."orders"') + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + }) + + it('drops a foreign key whose target is not in the schema', async () => { + // `orders` references a `warehouses` table that no longer exists in the + // schema: the FK must be pruned so the migration still runs. + const schemaWithDanglingFk = { + public: { + orders: { + name: 'orders', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [ + { + target_table: 'public.warehouses', + columns: [{ source_column: 'id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(schemaWithDanglingFk) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."orders"') + expect(migrations[0].sql).not.toContain('warehouses') + }) + + it('emits a disabled comment entry when a referenced table is not found', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['nonexistent'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('-- Table "nonexistent" is referenced but was not found') + expect(migrations[0].sql).not.toContain('BEGIN;') + }) + + it('keeps found tables and comments the missing ones in one migration', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['customers', 'ghost'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('-- Table "ghost" is referenced but was not found') + // Comments precede the runnable transaction. + expect(migrations[0].sql.indexOf('-- Table "ghost"')).toBeLessThan( + migrations[0].sql.indexOf('BEGIN;') + ) + }) + + it('comments a data table used with no specific table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set()]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('no specific table was referenced') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts new file mode 100644 index 0000000000..9ff80c32eb --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts @@ -0,0 +1,349 @@ +// Best-effort data table migration generation for the "project = folder" Hub +// bundle. Detects which data tables (and tables within them) a project's +// scripts/flows/raw apps reference via `datatable` assets, then generates a +// `CREATE TABLE` bundle per data table from the source workspace's live schema, +// so importing the project into another workspace can recreate those tables. +// +// Best-effort by design: the generated SQL is shown to the publisher and is +// fully editable before publishing. Low-code (non-raw) apps have no persisted +// asset list and are not scanned. + +import { inferAssets } from '$lib/infer' +import type { SupportedLanguage } from '$lib/common' +import { getAllModules } from '$lib/components/flows/flowExplorer' +import { getFlowModuleAssets } from '$lib/components/assets/lib' +import { extractDataConfig, parseDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' +import { + apiSchemaToEditorSchema, + generateMigrationSql, + type DatabaseSchema +} from '$lib/components/datatableSchemaSql' +import { WorkspaceService } from '$lib/gen' +import type { FetchedItem } from './projectBundle' + +export interface GeneratedMigration { + datatable_name: string + /** Up migration: creates the tables. */ + sql: string + /** Down migration: drops the created tables. Best-effort, generated once and + * editable by the publisher (not re-derived from `sql`). */ + sql_down: string + enabled: boolean +} + +// A datatable asset path is `datatable`, `datatable/table`, or +// `datatable/schema.table` (see the SQL asset parser). The first segment is the +// data table name; the remainder identifies a specific table (absent = whole +// data table, no table to create). +function parseDatatableAssetPath(path: string): { datatable: string; table?: string } { + const slash = path.indexOf('/') + if (slash === -1) return { datatable: path } + const datatable = path.slice(0, slash) + const table = path.slice(slash + 1).trim() + return { datatable, table: table || undefined } +} + +function addDatatableTable( + map: Map>, + datatable: string, + table: string | undefined +): void { + if (!datatable) return + const set = map.get(datatable) ?? new Set() + if (table) set.add(table) + map.set(datatable, set) +} + +function addUsage(map: Map>, path: string): void { + const { datatable, table } = parseDatatableAssetPath(path) + addDatatableTable(map, datatable, table) +} + +/** + * Scan a project's fetched items for data table usage and return + * `datatable -> set of table refs` (a table ref is `table` or `schema.table`). + * - scripts: re-parse the code with the asset parser (`inferAssets`) + * - flows: read each module's stored `assets` + * - full-code (raw) apps: read the explicit `data.tables` declaration; fall back + * to `runnables[key].inlineScript.assets` for older apps + */ +export async function detectDatatableTables( + items: FetchedItem[] +): Promise>> { + const map = new Map>() + + for (const item of items) { + if (item.kind === 'script') { + const res = await inferAssets( + item.language as SupportedLanguage | undefined, + item.content ?? '' + ) + if (res.status === 'ok') { + for (const a of res.assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'flow') { + for (const mod of getAllModules(item.value?.modules ?? [], item.value?.failure_module)) { + const assets = getFlowModuleAssets(mod) + if (assets) for (const a of assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'raw_app') { + let parsed: any + try { + parsed = JSON.parse(item.content ?? '{}') + } catch { + continue + } + // Full-code apps explicitly declare the data tables/tables they use + // (`data.tables`, refs like `main/customers` or `main/schema:table`), so + // read that rather than parsing assets. + const config = extractDataConfig(parsed) + if (config) { + for (const ref of config.tables) { + const r = parseDataTableRef(ref) + const table = r.table + ? r.schema && r.schema !== 'public' + ? `${r.schema}.${r.table}` + : r.table + : undefined + addDatatableTable(map, r.datatable, table) + } + } + // Older raw apps instead carry datatable usage as inline-script assets. + const runnables = parsed?.runnables ?? {} + for (const key of Object.keys(runnables)) { + const assets = runnables[key]?.inlineScript?.assets + if (Array.isArray(assets)) + for (const a of assets) + if (a?.kind === 'datatable' && typeof a.path === 'string') addUsage(map, a.path) + } + } + } + return map +} + +// Resolve a table ref (`table` or `schema.table`) to a concrete +// `{ schemaName, tableName }` present in the live schema, or undefined if the +// table can't be found (dropped since, typo, …). +function resolveTable( + schema: DatabaseSchema, + tableRef: string +): { schemaName: string; tableName: string } | undefined { + const dot = tableRef.indexOf('.') + if (dot !== -1) { + const schemaName = tableRef.slice(0, dot) + const tableName = tableRef.slice(dot + 1) + if (schema[schemaName]?.[tableName]) return { schemaName, tableName } + } + // No schema qualifier (or the qualified lookup missed, e.g. a stale schema name): + // find the bare table name across every schema, first match wins. + const bareName = dot !== -1 ? tableRef.slice(dot + 1) : tableRef + for (const schemaName of Object.keys(schema)) { + if (schema[schemaName][bareName]) return { schemaName, tableName: bareName } + } + return undefined +} + +type ResolvedTable = { schemaName: string; tableName: string } + +const tableKey = (t: ResolvedTable) => `${t.schemaName}.${t.tableName}` + +// Grow the set of tables to create so it's closed under foreign keys: a used +// table's FK targets (and their FK targets, transitively) are pulled in, so the +// generated CREATE TABLEs never reference a table that isn't also created. FK +// targets that don't resolve in this schema are left out (their FK is pruned by +// pruneSchemaForTables). +function expandFkClosure(schema: DatabaseSchema, seed: ResolvedTable[]): ResolvedTable[] { + const inSet = new Map(seed.map((t) => [tableKey(t), t])) + const queue = [...seed] + while (queue.length > 0) { + const t = queue.shift()! + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && !inSet.has(tableKey(target))) { + inSet.set(tableKey(target), target) + queue.push(target) + } + } + } + return [...inSet.values()] +} + +// A copy of the schema restricted to `tables`, with each table's foreign keys +// filtered to targets that are also in `tables`. generateMigrationSql emits every +// FK it finds on a table, so pruning here keeps a stray FK (to a table outside the +// migration) from making the generated SQL fail. +function pruneSchemaForTables(schema: DatabaseSchema, tables: ResolvedTable[]): DatabaseSchema { + const inSet = new Set(tables.map(tableKey)) + const pruned: DatabaseSchema = {} + for (const t of tables) { + const orig = schema[t.schemaName]?.[t.tableName] + if (!orig) continue + ;(pruned[t.schemaName] ??= {})[t.tableName] = { + ...orig, + foreignKeys: (orig.foreignKeys ?? []).filter((fk) => { + const target = resolveTable(schema, fk.targetTable ?? '') + return target != null && inSet.has(tableKey(target)) + }) + } + } + return pruned +} + +// Order tables so a table is created after the in-set tables it references via a +// foreign key. Keyed by schema-qualified name (like the rest of the pipeline) so +// two same-named tables in different schemas aren't collapsed. Falls back to input +// order on a cycle so generation never hangs. +function orderByFkDependency(schema: DatabaseSchema, tables: ResolvedTable[]): ResolvedTable[] { + const inSet = new Set(tables.map(tableKey)) + const deps = new Map>() + for (const t of tables) { + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + const targets = new Set() + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && tableKey(target) !== tableKey(t) && inSet.has(tableKey(target))) { + targets.add(tableKey(target)) + } + } + deps.set(tableKey(t), targets) + } + const ordered: ResolvedTable[] = [] + const done = new Set() + const visiting = new Set() + const byKey = new Map(tables.map((t) => [tableKey(t), t])) + const visit = (key: string) => { + if (done.has(key) || visiting.has(key)) return + visiting.add(key) + for (const dep of deps.get(key) ?? []) visit(dep) + visiting.delete(key) + done.add(key) + const t = byKey.get(key) + if (t) ordered.push(t) + } + for (const t of tables) visit(tableKey(t)) + return ordered +} + +// Pull a readable one-line message out of an API error for embedding in a SQL +// comment (collapse whitespace so it can't break out of the `--` line). +function errorText(e: any): string { + const body = e?.body + const raw = + typeof body === 'string' && body.trim() + ? body + : body && typeof body === 'object' + ? (body.error?.message ?? body.message ?? JSON.stringify(body)) + : (e?.message ?? String(e)) + return String(raw).replace(/\s+/g, ' ').trim() +} + +// Strip the per-table `BEGIN;`/`COMMIT;` wrapper that generateMigrationSql adds, +// so several tables can share one transaction. +function unwrapTransaction(sql: string): string { + return sql + .replace(/^\s*BEGIN;\s*\n?/, '') + .replace(/\n?\s*COMMIT;\s*$/, '') + .trim() +} + +/** + * Generate one best-effort migration per used data table. Resolved tables (plus + * the tables they depend on via foreign key, in FK-dependency order) become a + * single CREATE TABLE transaction, enabled by default. Anything that couldn't be + * auto-generated — a table not found in the schema, a data table referenced as a + * whole, or a schema that couldn't be loaded — is written as a `--` SQL comment + * describing the problem, so the publisher sees what's missing instead of a blank + * entry. A migration with no runnable statements (only comments) is left disabled. + */ +export async function generateDatatableMigrations( + workspace: string, + usage: Map> +): Promise { + const out: GeneratedMigration[] = [] + for (const [datatable, tableRefs] of usage) { + let schema: DatabaseSchema + try { + const api = await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatable}` } + }) + schema = apiSchemaToEditorSchema(api) + } catch (e) { + // Couldn't reach the schema at all: leave a commented stub explaining why, + // so the publisher can fill it in rather than seeing a silent blank. + out.push({ + datatable_name: datatable, + sql: + `-- Could not load the schema of data table "${datatable}": ${errorText(e)}\n` + + `-- Add the CREATE TABLE statement(s) for the tables this project uses.`, + sql_down: '', + enabled: false + }) + continue + } + // Resolve the referenced tables; record a comment for each one we can't find + // so a partial migration still explains what's missing. + const resolved: ResolvedTable[] = [] + const comments: string[] = [] + for (const ref of tableRefs) { + const t = resolveTable(schema, ref) + if (t) resolved.push(t) + else + comments.push( + `-- Table "${ref}" is referenced but was not found in data table "${datatable}"; add its CREATE TABLE manually.` + ) + } + if (tableRefs.size === 0) { + comments.push( + `-- Data table "${datatable}" is used but no specific table was referenced; nothing to generate automatically.` + ) + } + // Pull in the tables the referenced ones depend on via FK, then generate + // against a schema whose FKs are restricted to this set, so the migration + // creates everything it references and never emits a dangling FK. + const closure = expandFkClosure(schema, resolved) + const ordered = orderByFkDependency(schema, closure) + const prunedSchema = pruneSchemaForTables(schema, ordered) + const statements = ordered + .map((t) => + unwrapTransaction( + // IF NOT EXISTS: FK closure pulls in shared parent tables (e.g. a + // referenced `orders` drags in `customers`) that often already exist in + // the target, so a plain CREATE would abort the whole transaction. The + // caveat — an existing differently-shaped table is silently left as-is — + // is acceptable for a best-effort, editable migration. + generateMigrationSql( + { schemaName: t.schemaName, tableName: t.tableName, kind: 'added' }, + prunedSchema, + { ifNotExists: true } + ) + ) + ) + .filter((s) => s.length > 0) + // Comments (the errors) go on top; the CREATE TABLE transaction, if any, + // follows. Enabled only when there's something to run. + const parts: string[] = [] + if (comments.length > 0) parts.push(comments.join('\n')) + if (statements.length > 0) parts.push(`BEGIN;\n${statements.join('\n\n')}\nCOMMIT;`) + // Best-effort down migration: the DROP TABLE statements are commented out + // because the FK closure pulls in shared parent tables that may have + // pre-existed in the target (dropping them would lose data the project never + // created). The publisher uncomments the tables this migration should drop. + const drops = [...ordered] + .reverse() + .map((t) => `-- DROP TABLE IF EXISTS "${t.schemaName}"."${t.tableName}";`) + const sqlDown = + drops.length > 0 + ? `-- Rollback: uncomment the tables this migration should drop (leave shared\n` + + `-- tables that already existed in the workspace commented out).\nBEGIN;\n${drops.join('\n')}\nCOMMIT;` + : '' + out.push({ + datatable_name: datatable, + sql: parts.join('\n\n'), + sql_down: sqlDown, + enabled: statements.length > 0 + }) + } + return out.sort((a, b) => a.datatable_name.localeCompare(b.datatable_name)) +} diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte index 676ff57414..90201fd28c 100644 --- a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte @@ -3,7 +3,8 @@ import { goto } from '$app/navigation' import { workspaceStore, enterpriseLicense } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { Button } from '$lib/components/common' + import { Button, Drawer, DrawerContent } from '$lib/components/common' + import Toggle from '$lib/components/Toggle.svelte' import { ScriptService, FlowService, @@ -11,6 +12,7 @@ ResourceService, ScheduleService, FolderService, + WorkspaceService, HttpTriggerService, WebsocketTriggerService, KafkaTriggerService, @@ -29,9 +31,23 @@ rewriteFlowValue, rewriteRawAppContent } from '$lib/components/workspaceSettings/projectBundle' + import { updatePolicy } from '$lib/components/apps/editor/appPolicy' + import { updateRawAppPolicy } from '$lib/sharedUtils' + import type { App } from '$lib/components/apps/types' + import MigrationSqlEditor from '$lib/components/workspaceSettings/MigrationSqlEditor.svelte' + import { runScriptAndPollResult } from '$lib/components/jobs/utils' + import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' + import { createAsyncConfirmationModal } from '$lib/components/common/confirmationModal/asyncConfirmationModal.svelte' + import Portal from '$lib/components/Portal.svelte' import { Cloud, Download, Loader2 } from 'lucide-svelte' type ExportItem = Record + interface ProjectMigration { + datatable_name: string + sql: string + sql_down?: string + enabled: boolean + } interface ProjectExport { project: { slug: string; name: string; summary: string; readme: string | null } scripts: ExportItem[] @@ -39,6 +55,7 @@ apps: ExportItem[] resources: ExportItem[] triggers: ExportItem[] + migrations?: ProjectMigration[] } let slug = $derived($page.url.searchParams.get('hub') ?? '') @@ -48,10 +65,45 @@ let loadError = $state(undefined) let data = $state(undefined) let installing = $state(false) + // True while the migration review/missing-datatable modals are open, before the + // import spinner starts — keeps the Import button from launching a second import. + let planningMigrations = $state(false) let results = $state<{ path: string; ok: boolean; error?: string }[]>([]) let done = $state(false) let folderName = $state('') + // When the target lacks a needed data table, "import without that migration". + const missingDatatableModal = createAsyncConfirmationModal() + + // Migration review drawer: preview + edit each runnable migration's SQL and + // choose which to run, resolved linearly via `reviewResolve`. + let reviewDrawer = $state() + let reviewList = $state< + { datatable_name: string; sql: string; sql_down: string; run: boolean }[] + >([]) + // Bumped per review session so the Monaco editors re-mount with the new SQL. + let reviewGeneration = $state(0) + let reviewResolve: ((run: boolean) => void) | undefined + function openMigrationReview(migs: ProjectMigration[]): Promise { + reviewList = migs.map((m) => ({ + datatable_name: m.datatable_name, + sql: m.sql, + sql_down: m.sql_down ?? '', + run: true + })) + reviewGeneration++ + reviewDrawer?.openDrawer() + return new Promise((resolve) => (reviewResolve = resolve)) + } + function closeMigrationReview(run: boolean) { + // Capture + clear first so the `on:close` fired by closeDrawer() (which would + // call this again with run=false) can't override an explicit Run/Skip choice. + const resolve = reviewResolve + reviewResolve = undefined + reviewDrawer?.closeDrawer() + resolve?.(run) + } + let loadSeq = 0 $effect(() => { @@ -91,7 +143,10 @@ flows: data.flows.length, apps: data.apps.length, resources: data.resources.length, - triggers: data.triggers.length + triggers: data.triggers.length, + migrations: (data.migrations ?? []).filter( + (m) => m.enabled && (m.sql ?? '').trim() !== '' + ).length } : undefined ) @@ -119,8 +174,21 @@ ) } - // Minimal non-public policy for re-created apps. - const defaultPolicy = { execution_mode: 'publisher', triggerables_v2: {} } as any + // Recompute an app's execution policy from its (retargeted) value, mirroring + // what the editor does on deploy. `triggerables_v2` is keyed by + // `:rawscript/`; retargeting rewrites that + // content, so a copied or empty policy would leave every inline runnable + // "forbidden by policy" at runtime. Default to publisher (auth required). + async function computeAppPolicy(value: any): Promise { + const policy = (await updatePolicy(value as App, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy + } + async function computeRawAppPolicy(runnables: Record): Promise { + const policy = (await updateRawAppPolicy(runnables, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy + } // EE-only kinds; the rest (http, websocket, postgres, mqtt, email) work on CE. const EE_TRIGGER_KINDS = new Set(['kafka', 'nats', 'sqs', 'gcp', 'azure']) @@ -213,14 +281,130 @@ } } + // Decide which data table migrations to run. Migrations are keyed by data table + // name and applied only to a target data table of the same name. Returns the + // migrations to run (with any edits the user made), an empty array when there's + // nothing to run, or `null` when the user backs out of the whole import at the + // missing-data-table warning. + async function planMigrations( + workspace: string, + migrations: ProjectMigration[] + ): Promise { + const enabled = migrations.filter((m) => m.enabled && (m.sql ?? '').trim() !== '') + if (enabled.length === 0) return [] + + let present: Set + try { + const dts = await WorkspaceService.listDataTables({ workspace }) + present = new Set(dts.map((d) => d.name)) + } catch { + // Can't read the target's data tables — skip migrations rather than guess. + return [] + } + const runnable = enabled.filter((m) => present.has(m.datatable_name)) + const missingNames = [ + ...new Set(enabled.filter((m) => !present.has(m.datatable_name)).map((m) => m.datatable_name)) + ] + + // Warn about missing data tables first: confirming imports without their + // migrations, cancelling backs out of the whole import so the user can create + // the data table(s) and re-run. + if (missingNames.length > 0) { + const proceed = await missingDatatableModal.ask({ + title: 'Some data tables are missing', + confirmationText: 'Import without them', + children: `This project uses data table(s) "${missingNames.join( + '", "' + )}" that don't exist in this workspace, so their migrations will be skipped. To apply them, cancel, create the data table(s) with the same name in Workspace settings → Data tables, then re-run this import.` + }) + if (!proceed) return null + } + + let toRun: ProjectMigration[] = [] + if (runnable.length > 0) { + const run = await openMigrationReview(runnable) + if (run) { + toRun = reviewList + .filter((r) => r.run && r.sql.trim() !== '') + .map((r) => ({ + datatable_name: r.datatable_name, + sql: r.sql, + sql_down: r.sql_down, + enabled: true + })) + } + } + return toRun + } + + // Apply one migration to the target data table. If the data table opted into + // migrations, record it (datatable_migrations + _wm_migrations, run only this + // version); otherwise run the SQL once as a preview job (unrecorded). + async function applyOneMigration(workspace: string, m: ProjectMigration): Promise { + let recorded = false + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: m.datatable_name + }) + recorded = !!status.enabled + } catch {} + + if (recorded) { + // Record the shipped down migration (DROP the created tables) so it can be + // rolled back. + const codeDown = (m.sql_down ?? '').trim() + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName: m.datatable_name, + requestBody: { + name: `hub_import_${data?.project.slug ?? 'project'}`, + code_up: m.sql, + code_down: codeDown || undefined + } + }) + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: m.datatable_name, + only: created.timestamp + }) + } else { + await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }) + } + } + async function install() { // Snapshot reactive state up-front: `workspace` ($derived) and `data` // ($state, replaced by load()) can both change mid-import on a workspace // switch, which would split items or mix two exports. Pin both. + // Guard against a second click while the review modal is open (the Import + // button isn't `installing` yet during planning, so it would otherwise be + // clickable and start a concurrent import). + if (installing || planningMigrations) return const workspace = $workspaceStore const exportData = data if (!exportData || !workspace) return const folder = folderName.trim() || exportData.project.slug + + // Review data table migrations first (before the import spinner), so the user + // previews/edits and decides, then the whole import runs uninterrupted. + planningMigrations = true + let migrationsToRun: ProjectMigration[] | null + try { + migrationsToRun = await planMigrations(workspace, exportData.migrations ?? []) + } finally { + planningMigrations = false + } + // User backed out at the missing-data-table warning — abort the whole import. + if (migrationsToRun === null) return + installing = true results = [] done = false @@ -294,14 +478,21 @@ const css = files['/bundle.css'] ?? '' delete files['/bundle.js'] delete files['/bundle.css'] + const runnables = parsed.runnables ?? {} return AppService.createAppRaw({ workspace, formData: { app: { path: a.path, summary: a.summary ?? '', - value: { files, runnables: parsed.runnables ?? {} }, - policy: defaultPolicy + value: { + files, + runnables, + // Keep the full-code app's explicit data table declaration. + ...(parsed.data !== undefined ? { data: parsed.data } : {}), + ...(parsed.datatables !== undefined ? { datatables: parsed.datatables } : {}) + }, + policy: await computeRawAppPolicy(runnables) }, js, css @@ -312,15 +503,16 @@ } else { await record( a.path, - AppService.createApp({ - workspace, - requestBody: { - path: a.path, - summary: a.summary ?? '', - value: a.value, - policy: defaultPolicy - } - }) + (async () => + AppService.createApp({ + workspace, + requestBody: { + path: a.path, + summary: a.summary ?? '', + value: a.value, + policy: await computeAppPolicy(a.value) + } + }))() ) } } @@ -369,6 +561,13 @@ } } } + + // Apply the reviewed data table migrations after items exist. Each is + // recorded (or run as a preview job) per applyOneMigration. + for (const m of migrationsToRun) { + await record(`data table: ${m.datatable_name}`, applyOneMigration(workspace, m)) + } + done = true const failed = results.filter((r) => !r.ok).length sendUserToast( @@ -413,6 +612,9 @@ {counts?.apps} apps {counts?.resources} resources {counts?.triggers} triggers + {#if counts && counts.migrations > 0} + {counts.migrations} data table migrations + {/if}
{#if installing} @@ -458,3 +660,45 @@ {/if} {/if}
+ + + + + + closeMigrationReview(false)}> + closeMigrationReview(false)}> +
+

+ This project ships migrations that recreate the data tables it uses. Review and edit the + SQL, then choose which to run. A migration runs against the data table of the same name in + {workspace}; if that data table has migrations enabled it is + recorded, otherwise it runs once as a preview job. +

+ {#each reviewList as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ {#if m.run} + + {/if} +
+ {/each} +
+ {#snippet actions()} + + + {/snippet} + +