diff --git a/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json new file mode 100644 index 0000000000..13536a244e --- /dev/null +++ b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e" +} diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json b/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json new file mode 100644 index 0000000000..47bfec9e5c --- /dev/null +++ b/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb" +} diff --git a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json b/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json deleted file mode 100644 index d099d97bd3..0000000000 --- a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT ws.datatable->'datatables' AS datatable_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "datatable_name", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b" -} diff --git a/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json b/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json new file mode 100644 index 0000000000..2f1ad395b6 --- /dev/null +++ b/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c" +} diff --git a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json b/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json new file mode 100644 index 0000000000..3206ba4433 --- /dev/null +++ b/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8" +} diff --git a/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json b/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json new file mode 100644 index 0000000000..1e1c5f20c3 --- /dev/null +++ b/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd" +} diff --git a/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json b/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json new file mode 100644 index 0000000000..20ac675432 --- /dev/null +++ b/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 RETURNING timestamp, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json new file mode 100644 index 0000000000..12628305cc --- /dev/null +++ b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82" +} diff --git a/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json b/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json new file mode 100644 index 0000000000..14b085db2e --- /dev/null +++ b/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables'->$2 FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f" +} diff --git a/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json b/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json new file mode 100644 index 0000000000..3a1dc9da28 --- /dev/null +++ b/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0" +} diff --git a/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json new file mode 100644 index 0000000000..384686eb35 --- /dev/null +++ b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6" +} diff --git a/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json b/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json new file mode 100644 index 0000000000..f7ff5d527a --- /dev/null +++ b/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bool", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9" +} diff --git a/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json new file mode 100644 index 0000000000..eb94fe6ab5 --- /dev/null +++ b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14" +} diff --git a/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json b/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json new file mode 100644 index 0000000000..5ac3174af6 --- /dev/null +++ b/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'false'::jsonb) WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb" +} diff --git a/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json b/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json new file mode 100644 index 0000000000..c262f3d1f6 --- /dev/null +++ b/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 ORDER BY datatable, timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025" +} diff --git a/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json b/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json new file mode 100644 index 0000000000..2a90d12ad0 --- /dev/null +++ b/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down)\n SELECT $2, datatable, timestamp, name, code_up, code_down\n FROM datatable_migrations WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020" +} diff --git a/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json b/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json new file mode 100644 index 0000000000..f3cc23c5b8 --- /dev/null +++ b/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c" +} diff --git a/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json b/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json new file mode 100644 index 0000000000..b3861c0111 --- /dev/null +++ b/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, 'initial', $4, NULL)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0" +} diff --git a/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json b/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json new file mode 100644 index 0000000000..fa1396235f --- /dev/null +++ b/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f" +} diff --git a/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json b/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json new file mode 100644 index 0000000000..b089e466cf --- /dev/null +++ b/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb" +} diff --git a/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json b/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json new file mode 100644 index 0000000000..83c306f4d8 --- /dev/null +++ b/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2) AND timestamp = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac" +} diff --git a/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json new file mode 100644 index 0000000000..d6c6a5c02f --- /dev/null +++ b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6" +} diff --git a/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json b/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json new file mode 100644 index 0000000000..fbcf0743be --- /dev/null +++ b/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'true'::jsonb) WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca" +} diff --git a/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json b/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json new file mode 100644 index 0000000000..8a6d989157 --- /dev/null +++ b/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 ORDER BY datatable, timestamp", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd" +} diff --git a/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json b/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json new file mode 100644 index 0000000000..1efaaa32e8 --- /dev/null +++ b/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET labels = (\n SELECT array_agg(DISTINCT l)\n FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) l\n ) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b" +} diff --git a/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json b/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json new file mode 100644 index 0000000000..c630f5dd05 --- /dev/null +++ b/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b2acfc2e66..177509c02e 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14473,6 +14473,7 @@ dependencies = [ "sqlx", "strum", "tokio", + "tokio-postgres", "tracing", "uuid", "windmill-api-auth", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 087459205e..1b702a3a54 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0de2412ff0734b11e12ba378c9bcc373ff9ae800 +27672e37df5d9dfde94f19963d5ffcdf8dd5448c diff --git a/backend/migrations/20260617081932_datatable_migrations.down.sql b/backend/migrations/20260617081932_datatable_migrations.down.sql new file mode 100644 index 0000000000..26efc2bdcc --- /dev/null +++ b/backend/migrations/20260617081932_datatable_migrations.down.sql @@ -0,0 +1 @@ +DROP TABLE datatable_migrations; diff --git a/backend/migrations/20260617081932_datatable_migrations.up.sql b/backend/migrations/20260617081932_datatable_migrations.up.sql new file mode 100644 index 0000000000..72c7dc5ec6 --- /dev/null +++ b/backend/migrations/20260617081932_datatable_migrations.up.sql @@ -0,0 +1,21 @@ +-- SQL migrations defined per data table within a workspace. +-- `datatable` is the target data table name, `name` is the migration name +-- (e.g. add_index_to_customers), and `timestamp` is the migration version +-- (YYYYMMDDHHMMSS), recorded as `version` in the data table's `_wm_migrations` +-- table once applied. +CREATE TABLE datatable_migrations ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + datatable VARCHAR(255) NOT NULL, + timestamp BIGINT NOT NULL, + name VARCHAR(255) NOT NULL, + code_up TEXT NOT NULL, + code_down TEXT, + PRIMARY KEY (workspace_id, datatable, timestamp) +); + +-- No standalone index: the (workspace_id, datatable, timestamp) primary-key btree +-- already serves both `WHERE workspace_id = $1` and `WHERE workspace_id = $1 AND +-- datatable = $2` lookups via its leading columns. + +GRANT ALL ON datatable_migrations TO windmill_user; +GRANT ALL ON datatable_migrations TO windmill_admin; diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index c25bb8f42b..52a58aa1a3 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -47,6 +47,7 @@ serde_json.workspace = true sha2.workspace = true sqlx.workspace = true tokio.workspace = true +tokio-postgres.workspace = true tracing.workspace = true uuid.workspace = true strum.workspace = true diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs new file mode 100644 index 0000000000..fef9963933 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -0,0 +1,1712 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Data table SQL migrations: CRUD endpoints, run/rollback execution, opt-in +//! management, and the workspace-merge diff helper. Split out of `workspaces.rs` +//! to keep that file focused on core workspace configuration. + +use crate::workspaces::{pg_dump_database, ItemComparison}; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use sqlx::{Postgres, Transaction}; +use std::collections::{HashMap, HashSet}; + +use windmill_api_auth::{require_super_admin, ApiAuthed}; +use windmill_api_jobs::run_wait_result_internal; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::db::UserDB; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, DebouncingSettings}; +use windmill_common::scripts::ScriptLang; +use windmill_common::users::username_to_permissioned_as; +use windmill_common::worker::to_raw_value; +use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; +use windmill_common::{PgDatabase, DB}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_queue::{push, PushArgs, PushIsolationLevel}; + +pub(crate) fn routes() -> Router { + Router::new() + .route( + "/run_datatable_migrations/{datatable_name}", + post(run_datatable_migrations), + ) + .route( + "/rollback_datatable_migrations/{datatable_name}", + post(rollback_datatable_migrations), + ) + .route("/list_datatable_migrations", get(list_datatable_migrations)) + .route( + "/datatable_migrations_status/{datatable_name}", + get(datatable_migrations_status), + ) + .route( + "/enable_datatable_migrations/{datatable_name}", + post(enable_datatable_migrations), + ) + .route( + "/disable_datatable_migrations/{datatable_name}", + post(disable_datatable_migrations), + ) + .route( + "/create_datatable_migration/{datatable_name}", + post(create_datatable_migration), + ) + .route( + "/delete_datatable_migration/{datatable_name}/{timestamp}", + delete(delete_datatable_migration), + ) + .route( + "/upsert_datatable_migration/{datatable_name}", + post(upsert_datatable_migration), + ) + .route( + "/generate_initial_datatable_migration/{datatable_name}", + post(generate_initial_datatable_migration), + ) +} + +#[derive(Serialize)] +struct AppliedMigration { + version: i64, + name: String, +} + +#[derive(Serialize)] +struct RunDatatableMigrationsResult { + applied: Vec, +} + +#[derive(Deserialize)] +struct RunDatatableMigrationsQuery { + /// When set, only apply pending migrations up to and including this version. + up_to: Option, + /// When set, apply only this specific migration version (if not already + /// applied), ignoring any other pending migrations. Takes precedence over + /// `up_to`. + only: Option, +} + +/// Build the `database` argument for a migration job. Both resource-backed and +/// instance data tables pass a `datatable://` reference; the pg executor +/// resolves it to real credentials server-side at run time. It must never be +/// resolved here: the resolved instance credentials include a single +/// instance-wide Postgres password, and the job's `args` are readable by the — +/// possibly non-admin — user who ran the migration. +async fn datatable_database_arg( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + // Fail fast with a clear error if the data table doesn't exist. + sqlx::query_scalar!( + "SELECT ws.datatable->'datatables'->$2 FROM workspace_settings ws WHERE ws.workspace_id = $1", + w_id, + datatable_name, + ) + .fetch_one(db) + .await? + .ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?; + + Ok(to_raw_value(&format!("datatable://{datatable_name}"))) +} + +/// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as +/// the requesting user and labelled `datatable_migration` for traceability, then +/// wait for it. Errors if the job fails. +async fn run_datatable_migration_job( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + database_arg: &Box, + sql: &str, +) -> Result<()> { + let mut args = HashMap::new(); + args.insert("database".to_string(), database_arg.clone()); + let push_args = PushArgs { extra: None, args: &args }; + + let (uuid, mut tx) = push( + db, + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), + w_id, + JobPayload::Code(RawCode { + content: sql.to_string(), + path: Some("datatable_migration".to_string()), + hash: None, + language: ScriptLang::Postgresql, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + tag: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + modules: None, + }), + push_args, + authed.display_username(), + &authed.email, + username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + None, + ) + .await?; + + // Tag the job so migration runs are easy to find in the run history. + sqlx::query!( + "UPDATE v2_job SET labels = ( + SELECT array_agg(DISTINCT l) + FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) l + ) WHERE id = $1", + uuid, + &vec!["datatable_migration".to_string()], + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + let (result, success) = + run_wait_result_internal(db, uuid, w_id, None, false, &authed.username).await?; + if !success { + // On failure the job result is `{"error": {"name", "message", ...}}`; + // surface the executor's message (the Postgres error, e.g. `relation + // "foo" does not exist`) instead of the raw JSON envelope. + let detail = serde_json::from_str::(result.get()) + .ok() + .as_ref() + .and_then(|v| v.get("error")) + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()) + .map(str::to_string) + .unwrap_or_else(|| result.get().to_string()); + return Err(Error::internal_err(detail)); + } + Ok(()) +} + +/// Ensure the `_wm_migrations` bookkeeping table exists. Migration versions are +/// only unique per data table, but several data-table configs can point at one +/// physical database, so it is keyed by `(datatable, version)` — a version-only +/// key would let one data table's migration mark another's same-version +/// migration as already applied (and rollback could touch the wrong row). +async fn ensure_wm_migrations_schema(client: &tokio_postgres::Client) -> Result<()> { + client + .batch_execute( + "CREATE TABLE IF NOT EXISTS _wm_migrations (\ + datatable TEXT NOT NULL, \ + version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), \ + PRIMARY KEY (datatable, version))", + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to ensure _wm_migrations table: {}", e)) + })?; + Ok(()) +} + +/// Open a connection to a data table's own database and hold the session-level +/// advisory lock that serializes migration runs/rollbacks. The lock is released +/// when the returned client is dropped, so callers must keep it in scope for the +/// whole critical section. +/// +/// Runs, rollbacks *and* definition rewrites/deletes all take this lock: a run +/// snapshots a migration's `code_up` from `datatable_migrations` and only records +/// its version in `_wm_migrations` after the job succeeds, so an unserialized edit +/// could rewrite the definition in that window and leave `_wm_migrations` pointing +/// at SQL that was never applied. `_wm_migrations` is per-database, so a single +/// key is sufficient. +async fn lock_datatable_migration_runs( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + client + .batch_execute("SELECT pg_advisory_lock(hashtext('windmill_datatable_migrations')::int8)") + .await + .map_err(|e| Error::internal_err(format!("Failed to acquire migration lock: {}", e)))?; + Ok(client) +} + +/// Read the versions recorded as applied in a data table's `_wm_migrations`, +/// scoped to that data table, using an existing connection. An absent table +/// (`42P01`) means nothing has been migrated yet. +async fn read_applied_versions_on_client( + client: &tokio_postgres::Client, + datatable_name: &str, +) -> Result> { + match client + .query( + "SELECT version FROM _wm_migrations WHERE datatable = $1", + &[&datatable_name], + ) + .await + { + Ok(rows) => Ok(rows.iter().map(|row| row.get::<_, i64>(0)).collect()), + Err(e) if e.as_db_error().map(|d| d.code().code()) == Some("42P01") => Ok(HashSet::new()), + Err(e) => Err(Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + e + ))), + } +} + +/// Apply the workspace's pending data table migrations to a given data table. +/// Each migration runs as a normal Windmill `postgresql` job (permissioned as +/// the requester, labelled `datatable_migration`); applied versions are then +/// recorded in the data table's own `_wm_migrations` table, so only migrations +/// not recorded there are run, in ascending `timestamp` order. +async fn run_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + audit_log( + &db, + &authed, + "workspaces.run_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; + + // Take the run-serialization lock before snapshotting the migration + // definitions: a concurrent definition rewrite/delete takes the same lock, so + // the `code_up` we read here can't change between now and when we record its + // version below. The lock is held until `client` drops at return. + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name).await?; + + let migrations = sqlx::query!( + "SELECT timestamp, name, code_up FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + ensure_wm_migrations_schema(&client).await?; + + let applied_versions = read_applied_versions_on_client(&client, &datatable_name).await?; + + let mut applied = Vec::new(); + for m in migrations { + if let Some(only) = query.only { + // Run a single specific migration, skipping every other one. + if m.timestamp != only { + continue; + } + } else if query.up_to.is_some_and(|up_to| m.timestamp > up_to) { + // Migrations are ordered ascending, so once we pass `up_to` we're done. + break; + } + if applied_versions.contains(&m.timestamp) { + continue; + } + run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to apply migration {} ({}): {}", + m.timestamp, m.name, e + )) + })?; + // Record the migration as installed once its job has succeeded. + client + .execute( + "INSERT INTO _wm_migrations (datatable, version) VALUES ($1, $2) \ + ON CONFLICT (datatable, version) DO NOTHING", + &[&datatable_name, &m.timestamp], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to record migration: {}", e)))?; + applied.push(AppliedMigration { version: m.timestamp, name: m.name }); + } + + Ok(Json(RunDatatableMigrationsResult { applied })) +} + +#[derive(Serialize)] +struct RolledBackMigration { + version: i64, + name: String, +} + +#[derive(Serialize)] +struct RollbackDatatableMigrationsResult { + rolled_back: Vec, +} + +#[derive(Deserialize)] +struct RollbackDatatableMigrationsQuery { + /// When set, roll back this specific applied migration version instead of + /// the most recently applied one. + only: Option, +} + +/// Roll back a migration on a given data table: run its `code_down` as a normal +/// Windmill `postgresql` job (permissioned as the requester, labelled +/// `datatable_migration`) then drop its `_wm_migrations` row. Without `only` this +/// targets the most recently applied migration (one step); with `only` it +/// targets that specific applied version. +async fn rollback_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + audit_log( + &db, + &authed, + "workspaces.rollback_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + // The data table's `_wm_migrations` bookkeeping is read here and the version + // dropped after the job succeeds; the down SQL itself runs in the job. The + // lock is held (until `client` drops at return) so a concurrent run or + // definition rewrite can't interleave with this rollback. + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name).await?; + + ensure_wm_migrations_schema(&client).await?; + + // Resolve which applied version to roll back: a specific one when `only` is + // given (and actually applied), otherwise the most recently applied. Scoped + // to this data table so a shared physical database can't surface another + // data table's version. + let target = match query.only { + Some(only) => client + .query_opt( + "SELECT version FROM _wm_migrations WHERE datatable = $1 AND version = $2", + &[&datatable_name, &only], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + None => client + .query_opt( + "SELECT version FROM _wm_migrations WHERE datatable = $1 \ + ORDER BY version DESC LIMIT 1", + &[&datatable_name], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + }; + + let version: i64 = match target { + Some(row) => row.get::<_, i64>(0), + None => { + return Ok(Json(RollbackDatatableMigrationsResult { + rolled_back: vec![], + })) + } + }; + + let definition = sqlx::query!( + "SELECT name, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + version + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Cannot roll back migration {version}: its definition no longer exists" + )) + })?; + + let code_down = definition.code_down.ok_or_else(|| { + Error::BadRequest(format!( + "Cannot roll back migration {} ({}): it has no down migration", + version, definition.name + )) + })?; + + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; + run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to roll back migration {} ({}): {}", + version, definition.name, e + )) + })?; + + // Forget the version once its down job has succeeded. + client + .execute( + "DELETE FROM _wm_migrations WHERE datatable = $1 AND version = $2", + &[&datatable_name, &version], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to drop migration record: {}", e)))?; + + Ok(Json(RollbackDatatableMigrationsResult { + rolled_back: vec![RolledBackMigration { version, name: definition.name }], + })) +} + +#[derive(Serialize, Deserialize)] +pub struct DatatableMigration { + pub datatable: String, + pub timestamp: i64, + pub name: String, + pub code_up: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_down: Option, +} + +async fn list_datatable_migrations( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let migrations = sqlx::query_as!( + DatatableMigration, + "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 ORDER BY datatable, timestamp ASC", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(migrations)) +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum DatatableMigrationRunStatus { + /// Recorded in the data table's `_wm_migrations` table. + Ran, + /// Defined but not yet applied. + NotRun, + /// Applied status could not be determined (connection failure). + Unknown, +} + +#[derive(Serialize)] +struct DatatableMigrationWithStatus { + timestamp: i64, + name: String, + code_up: String, + #[serde(skip_serializing_if = "Option::is_none")] + code_down: Option, + status: DatatableMigrationRunStatus, +} + +#[derive(Serialize)] +struct DatatableMigrationsStatusResult { + /// Whether the migrations feature is opted in for this data table. + enabled: bool, + migrations: Vec, + /// Set when the applied status couldn't be read from the data table. + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +/// Whether the SQL-migrations feature is enabled for a data table. Honors the +/// explicit `migrations_enabled` flag; when unset (data tables predating the +/// feature) it is considered enabled only if migrations already exist. +async fn datatable_migrations_enabled(db: &DB, w_id: &str, datatable_name: &str) -> Result { + let flag: Option = sqlx::query_scalar!( + "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean \ + FROM workspace_settings ws WHERE ws.workspace_id = $1", + w_id, + datatable_name, + ) + .fetch_optional(db) + .await? + .flatten(); + + match flag { + Some(v) => Ok(v), + None => Ok(sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2)", + w_id, + datatable_name, + ) + .fetch_one(db) + .await? + .unwrap_or(false)), + } +} + +/// Reject the request when migrations are not enabled for the data table. +async fn ensure_datatable_migrations_enabled( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result<()> { + if !datatable_migrations_enabled(db, w_id, datatable_name).await? { + return Err(Error::BadRequest(format!( + "Migrations are not enabled for data table '{}'. Enable them first.", + datatable_name + ))); + } + Ok(()) +} + +/// Read the versions recorded in a data table's `_wm_migrations` table. A +/// missing table means nothing has been applied yet (empty set, not an error). +async fn read_applied_datatable_versions( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + + // Read-only status path: don't create the table here, and don't take the run + // lock — a stale-by-a-moment applied set is fine for display. + read_applied_versions_on_client(&client, datatable_name).await +} + +/// List a data table's migrations annotated with whether each has been applied. +async fn datatable_migrations_status( + _authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + if !enabled { + return Ok(Json(DatatableMigrationsStatusResult { + enabled: false, + migrations: vec![], + error: None, + })); + } + + let defs = sqlx::query!( + "SELECT timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + let (applied, error) = match read_applied_datatable_versions(&db, &w_id, &datatable_name).await + { + Ok(set) => (Some(set), None), + Err(e) => (None, Some(e.to_string())), + }; + + let migrations = defs + .into_iter() + .map(|m| { + let status = match &applied { + Some(set) if set.contains(&m.timestamp) => DatatableMigrationRunStatus::Ran, + Some(_) => DatatableMigrationRunStatus::NotRun, + None => DatatableMigrationRunStatus::Unknown, + }; + DatatableMigrationWithStatus { + timestamp: m.timestamp, + name: m.name, + code_up: m.code_up, + code_down: m.code_down, + status, + } + }) + .collect(); + + Ok(Json(DatatableMigrationsStatusResult { + enabled: true, + migrations, + error, + })) +} + +/// Only workspace admins and super admins may opt a data table in or out of +/// migrations. +async fn require_datatable_migrations_manager(db: &DB, authed: &ApiAuthed) -> Result<()> { + if authed.is_admin || require_super_admin(db, &authed.email).await.is_ok() { + Ok(()) + } else { + Err(Error::BadRequest( + "Only workspace admins and super admins can enable or disable data table migrations" + .to_string(), + )) + } +} + +async fn enable_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> Result { + require_datatable_migrations_manager(&db, &authed).await?; + + let updated = sqlx::query_scalar!( + "UPDATE workspace_settings \ + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'true'::jsonb) \ + WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) \ + RETURNING 1", + &w_id, + &datatable_name, + ) + .fetch_optional(&db) + .await?; + if updated.is_none() { + return Err(Error::NotFound(format!( + "data table {datatable_name} not found" + ))); + } + + audit_log( + &db, + &authed, + "workspaces.enable_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + Ok(format!( + "Enabled migrations for data table {datatable_name}" + )) +} + +/// Opt a data table out of migrations. Deletes ALL of its migration definitions. +async fn disable_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> Result { + require_datatable_migrations_manager(&db, &authed).await?; + + let mut tx = db.begin().await?; + + let updated = sqlx::query_scalar!( + "UPDATE workspace_settings \ + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'false'::jsonb) \ + WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) \ + RETURNING 1", + &w_id, + &datatable_name, + ) + .fetch_optional(&mut *tx) + .await?; + if updated.is_none() { + return Err(Error::NotFound(format!( + "data table {datatable_name} not found" + ))); + } + + // Capture the deleted definitions so each removal is tallied as a deployed + // object (like single-migration deletion), keeping workspace comparison and + // git-sync callbacks in sync when a fork opts back out of migrations. + let deleted = sqlx::query!( + "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 \ + RETURNING timestamp, name", + &w_id, + &datatable_name, + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.disable_datatable_migrations", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + for m in deleted { + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + m.timestamp, + &m.name, + ) + .await?; + } + + Ok(format!( + "Disabled migrations for data table {datatable_name} and deleted its migrations" + )) +} + +#[derive(Deserialize)] +pub struct CreateDatatableMigration { + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Migration names map onto on-disk file names and the `_wm_migrations` record, +/// so keep them to a safe path-segment charset (matches the CLI scaffold). +fn validate_migration_name(name: &str) -> Result<()> { + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(Error::BadRequest(format!( + "Invalid migration name '{name}': use only letters, digits, '_' and '-'" + ))); + } + Ok(()) +} + +/// The data table name becomes a directory segment in the sync export +/// (`migrations/datatable//...`); reject anything that could escape it. +pub(crate) fn validate_datatable_path_segment(datatable: &str) -> Result<()> { + if datatable.is_empty() + || datatable.contains('/') + || datatable.contains('\\') + || datatable.contains("..") + { + return Err(Error::BadRequest(format!( + "Invalid data table name '{datatable}': must not contain '/', '\\' or '..'" + ))); + } + Ok(()) +} + +/// Record a data table migration change as a deployed object so it is tallied +/// into `workspace_diff` and shows up as a `datatable_migration` item in the +/// workspace-merge diff. The diff path is `/_`, +/// matching `parse_datatable_migration_diff_path`. +async fn record_datatable_migration_deployment( + authed: &ApiAuthed, + db: &DB, + w_id: &str, + datatable: &str, + timestamp: i64, + name: &str, +) -> Result<()> { + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + w_id, + DeployedObject::DatatableMigration { path: format!("{datatable}/{timestamp}_{name}") }, + Some(format!( + "Data table migration {name} ({timestamp}) on {datatable}" + )), + false, + None, + ) + .await +} + +/// Allocate the next version for a data table and insert the migration +/// definition, in one transaction. A per-(workspace, data table) advisory lock +/// serializes concurrent version allocation so two creates can't read the same +/// `MAX(timestamp)` and collide on the `(workspace_id, datatable, timestamp)` +/// primary key. The version is the current UTC `YYYYMMDDHHMMSS`, bumped past any +/// existing version to stay unique and monotonically increasing. +async fn insert_datatable_migration_def( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + datatable: &str, + name: &str, + code_up: &str, + code_down: Option<&str>, +) -> Result { + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", + w_id, + datatable, + ) + .execute(&mut **tx) + .await?; + + let now_ts: i64 = Utc::now() + .format("%Y%m%d%H%M%S") + .to_string() + .parse() + .map_err(|e| Error::internal_err(format!("Failed to build migration version: {}", e)))?; + let max_existing: Option = sqlx::query_scalar!( + "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + w_id, + datatable, + ) + .fetch_one(&mut **tx) + .await?; + let timestamp = match max_existing { + Some(m) if m >= now_ts => m + 1, + _ => now_ts, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6)", + w_id, + datatable, + timestamp, + name, + code_up, + code_down, + ) + .execute(&mut **tx) + .await?; + + Ok(timestamp) +} + +/// Mark a version as already installed in a data table's `_wm_migrations` table +/// (ensuring the table exists first). +async fn mark_datatable_version_installed( + db: &DB, + pg_db: &PgDatabase, + datatable: &str, + version: i64, +) -> Result<()> { + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + ensure_wm_migrations_schema(&client).await?; + client + .execute( + "INSERT INTO _wm_migrations (datatable, version) VALUES ($1, $2) \ + ON CONFLICT (datatable, version) DO NOTHING", + &[&datatable, &version], + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to mark initial migration installed: {}", e)) + })?; + Ok(()) +} + +/// Create a single migration for a data table. The version is generated +/// server-side (current UTC `YYYYMMDDHHMMSS`), bumped past any existing version +/// so it stays unique and monotonically increasing. +async fn create_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> JsonResult { + validate_datatable_path_segment(&datatable_name)?; + validate_migration_name(&payload.name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + let mut tx = db.begin().await?; + let timestamp = insert_datatable_migration_def( + &mut tx, + &w_id, + &datatable_name, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; + tx.commit().await?; + + audit_log( + &db, + &authed, + "workspaces.create_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + &payload.name, + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: payload.name, + code_up: payload.code_up, + code_down: payload.code_down, + })) +} + +/// Delete a single migration definition from a data table. +async fn delete_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, +) -> Result { + // Hold the run-serialization lock across the applied-check and the delete: a + // run snapshots a migration's SQL before recording its version, so an + // unserialized delete could race it and leave `_wm_migrations` pointing at a + // definition that no longer exists (breaking rollback and hiding the applied + // version). Held until the handler returns. Fail closed if we can't verify. + let unreachable = |e| { + Error::internal_err(format!( + "Cannot verify whether migration {} on data table '{}' has already been applied \ + (its database is unreachable: {}). Refusing to delete it; retry once the database \ + is reachable.", + timestamp, datatable_name, e + )) + }; + let lock_client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) + .await + .map_err(unreachable)?; + let applied = read_applied_versions_on_client(&lock_client, &datatable_name) + .await + .map_err(unreachable)?; + if applied.contains(×tamp) { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' has already been applied and cannot be deleted. \ + Revert it first.", + timestamp, datatable_name + ))); + } + + let deleted_name = sqlx::query_scalar!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 \ + RETURNING name", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_optional(&db) + .await?; + // The definition is removed; runs may resume (audit/deploy metadata below + // don't need the lock). + drop(lock_client); + + audit_log( + &db, + &authed, + "workspaces.delete_datatable_migration", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + // Only tally a change if a migration was actually deleted. + if let Some(name) = deleted_name { + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + &name, + ) + .await?; + } + + Ok(format!( + "Deleted migration {} from {}", + timestamp, datatable_name + )) +} + +#[derive(Deserialize)] +pub struct UpsertDatatableMigration { + pub timestamp: i64, + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Insert or update a single migration at an explicit version. Used by +/// `wmill sync` to push a `migrations/datatable/
/_.up.sql` +/// (and `.down.sql`) file as the source of truth. +async fn upsert_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + validate_datatable_path_segment(&datatable_name)?; + validate_migration_name(&payload.name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + // Guard against silently rewriting a migration that has already run in the + // data table's database: its `_wm_migrations` record would no longer match + // its SQL, so a later `migrate up` would skip it and a rollback would run a + // `down` that doesn't correspond to what was applied. Only an actual change + // to an existing migration is guarded; unchanged re-pushes (e.g. + // `wmill sync push`) always proceed. + let existing = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + payload.timestamp, + ) + .fetch_optional(&db) + .await?; + // When modifying an existing definition, hold the run-serialization lock + // across the applied-check and the write below so an in-flight run can't + // record a version for the SQL we're about to overwrite. Held until the end + // of the handler (well past the write); a new/unchanged upsert needs no lock. + let _run_lock = match existing { + Some(existing) + if !(existing.name == payload.name + && existing.code_up == payload.code_up + && existing.code_down == payload.code_down) => + { + // Fail closed: if we can't lock/read the applied set (e.g. the + // data-table database is temporarily unreachable), refuse the change + // rather than risk overwriting a migration that has already run. + let unreachable = |e| { + Error::internal_err(format!( + "Cannot verify whether migration {} on data table '{}' has already been \ + applied (its database is unreachable: {}). Refusing to modify it; retry \ + once the database is reachable.", + payload.timestamp, datatable_name, e + )) + }; + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) + .await + .map_err(unreachable)?; + let applied = read_applied_versions_on_client(&client, &datatable_name) + .await + .map_err(unreachable)?; + if applied.contains(&payload.timestamp) { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' has already been applied and cannot be modified. \ + Revert it first, or add a new migration instead.", + payload.timestamp, datatable_name + ))); + } + Some(client) + } + _ => None, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ + SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + &w_id, + &datatable_name, + payload.timestamp, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .execute(&db) + .await?; + // The definition is written; runs may resume (audit/deploy metadata below + // don't need the lock). + drop(_run_lock); + + audit_log( + &db, + &authed, + "workspaces.upsert_datatable_migration", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + payload.timestamp, + &payload.name, + ) + .await?; + + Ok(format!( + "Upserted migration {} in {}", + payload.timestamp, datatable_name + )) +} + +/// Generate the first migration for a data table by snapshotting its current +/// schema with `pg_dump`. The migration is recorded as already installed (the +/// definition is written first, then its version is marked in the data table's +/// `_wm_migrations`, so it ends up considered applied and is never re-run) and +/// has no down migration. +async fn generate_initial_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + validate_datatable_path_segment(&datatable_name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + // The initial snapshot only makes sense on a data table with no migrations + // yet; reject otherwise so repeated calls don't pile up duplicate "initial" + // definitions (each at a distinct timestamp, each marked installed). + let has_migrations: bool = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2)", + &w_id, + &datatable_name, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if has_migrations { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' already has migrations; the initial migration can only be generated when there are none." + ))); + } + + let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + + // Snapshot the schema, excluding Windmill's own migration bookkeeping table. + let dump_file = pg_dump_database(&pg_db, true, &["_wm_migrations"]).await?; + let raw_dump = tokio::fs::read_to_string(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to read schema dump: {}", e)))?; + // pg_dump emits psql meta-commands (\restrict / \unrestrict) that aren't + // valid SQL; drop them so the migration body can run via a plain query. + let code_up: String = raw_dump + .lines() + .filter(|line| !line.trim_start().starts_with('\\')) + .collect::>() + .join("\n"); + + // Record the definition first, then mark it installed. If marking fails we + // delete the definition, so a failure leaves no phantom "initial" (rather + // than a `_wm_migrations` version with no definition that the UI can't + // clear). The narrow window where it briefly shows "not run" is benign: + // running it would just no-op/fail harmlessly against the existing schema. + let mut tx = db.begin().await?; + let timestamp = + insert_datatable_migration_def(&mut tx, &w_id, &datatable_name, "initial", &code_up, None) + .await?; + tx.commit().await?; + + if let Err(e) = mark_datatable_version_installed(&db, &pg_db, &datatable_name, timestamp).await + { + let _ = sqlx::query!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .execute(&db) + .await; + return Err(e); + } + + audit_log( + &db, + &authed, + "workspaces.generate_initial_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + "initial", + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: "initial".to_string(), + code_up, + code_down: None, + })) +} + +/// A datatable migration diff item has path `/_`. +/// Parse out the (datatable, timestamp) needed to look it up. +fn parse_datatable_migration_diff_path(path: &str) -> Option<(String, i64)> { + let (datatable, file) = path.split_once('/')?; + let ts_str: String = file.chars().take_while(|c| c.is_ascii_digit()).collect(); + let timestamp = ts_str.parse::().ok()?; + Some((datatable.to_string(), timestamp)) +} + +pub(crate) async fn compare_two_datatable_migration( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + let (datatable, timestamp) = match parse_datatable_migration_diff_path(path) { + Some(v) => v, + None => { + return Ok(ItemComparison { + has_changes: false, + exists_in_source: false, + exists_in_fork: false, + }) + } + }; + + let source = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + source_workspace_id, + datatable, + timestamp, + ) + .fetch_optional(db) + .await?; + let target = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + fork_workspace_id, + datatable, + timestamp, + ) + .fetch_optional(db) + .await?; + + let has_changes = match (&source, &target) { + (Some(s), Some(t)) => { + s.name != t.name || s.code_up != t.code_up || s.code_down != t.code_down + } + _ => source.is_some() || target.is_some(), + }; + + Ok(ItemComparison { + has_changes, + exists_in_source: source.is_some(), + exists_in_fork: target.is_some(), + }) +} + +#[derive(Deserialize, Debug)] +pub(crate) struct DatatableRename { + pub(crate) from: String, + pub(crate) to: String, +} + +async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?; + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} + +/// Tolerate a data table whose database has no `_wm_migrations` yet (42P01 = +/// undefined_table): it has never run a migration, so nothing to rename or forget. +fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> { + match e.as_db_error().map(|d| d.code().code()) { + Some("42P01") => Ok(()), + _ => Err(Error::internal_err(format!( + "Failed to update _wm_migrations: {}", + e + ))), + } +} + +/// Drop a data table's rows from its own database's `_wm_migrations`. +async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> { + let pg_db = resolve_datatable_pg(db, w_id, datatable).await?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + let _ = connection.await; + }); + client + .execute( + "DELETE FROM _wm_migrations WHERE datatable = $1", + &[&datatable], + ) + .await + .map(|_| ()) + .or_else(ignore_missing_wm_migrations) +} + +/// Relabel a data table's rows in its own database's `_wm_migrations`. `resolve_by` +/// names the config entry used to find the database (the old name, still present +/// pre-commit); `from`/`to` are the `datatable` column values to move between. +async fn remote_rename_datatable_migrations( + db: &DB, + w_id: &str, + resolve_by: &str, + from: &str, + to: &str, +) -> Result<()> { + let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + let _ = connection.await; + }); + client + .execute( + "UPDATE _wm_migrations SET datatable = $2 WHERE datatable = $1", + &[&from, &to], + ) + .await + .map(|_| ()) + .or_else(ignore_missing_wm_migrations) +} + +/// Keep migration bookkeeping in sync when data tables are renamed or deleted in +/// the workspace config: the control table `datatable_migrations` (this database, +/// in `tx`) and each data table's own `_wm_migrations` (its own database, keyed +/// by data table name — see [`ensure_wm_migrations_schema`]). +/// +/// Renames are applied in two phases through a temporary key so that a rename +/// chain or a swap (A->B, B->A) can't transiently collide on the +/// (datatable, ...) uniqueness mid-update. +/// +/// The `_wm_migrations` updates are best-effort: they run just before `tx` +/// commits (resolved via the pool, which still exposes the old names), and a +/// temporarily unreachable data-table database is logged rather than failing the +/// whole config edit. If one is missed, the next run re-applies its migrations +/// against the existing schema. +pub(crate) async fn cascade_datatable_migration_renames_and_deletes( + db: &DB, + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + renames: &[DatatableRename], + deleted_datatables: &[String], +) -> Result<()> { + if !deleted_datatables.is_empty() { + sqlx::query!( + "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", + w_id, + deleted_datatables + ) + .execute(&mut **tx) + .await?; + } + + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + sqlx::query!( + "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + w_id, + &r.from, + &tmp + ) + .execute(&mut **tx) + .await?; + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + sqlx::query!( + "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + w_id, + &tmp, + &r.to + ) + .execute(&mut **tx) + .await?; + } + + for name in deleted_datatables { + if let Err(e) = remote_forget_datatable_migrations(db, w_id, name).await { + tracing::warn!("Failed to clear _wm_migrations for deleted data table {name}: {e}"); + } + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + if let Err(e) = remote_rename_datatable_migrations(db, w_id, &r.from, &r.from, &tmp).await { + tracing::warn!( + "Failed to stage _wm_migrations rename {} -> {}: {e}", + r.from, + r.to + ); + } + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + if let Err(e) = remote_rename_datatable_migrations(db, w_id, &r.from, &tmp, &r.to).await { + tracing::warn!( + "Failed to finish _wm_migrations rename {} -> {}: {e}", + r.from, + r.to + ); + } + } + + Ok(()) +} + +/// Copy a workspace's migration definitions to another workspace, so a fork +/// inherits the same per-data-table migration history as its parent. +pub(crate) async fn clone_datatable_migrations( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) + SELECT $2, datatable, timestamp, name, code_up, code_down + FROM datatable_migrations WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_migration_name_accepts_safe_names() { + for name in ["initial", "add_index_to_customers", "fix-bug_2", "ABC123"] { + assert!( + validate_migration_name(name).is_ok(), + "{name} should be valid" + ); + } + } + + #[test] + fn validate_migration_name_rejects_unsafe_names() { + for name in [ + "", + "add index", + "a/b", + "a\\b", + "a..b", + "a.b", + "naïve", + "a/../b", + ] { + assert!( + validate_migration_name(name).is_err(), + "{name} should be rejected" + ); + } + } + + #[test] + fn validate_datatable_path_segment_accepts_and_rejects() { + for ok in ["mydt", "my-dt", "main", "a b"] { + assert!( + validate_datatable_path_segment(ok).is_ok(), + "{ok} should be ok" + ); + } + for bad in ["", "a/b", "a\\b", "..", "a..b", "../etc", "x/.."] { + assert!( + validate_datatable_path_segment(bad).is_err(), + "{bad} should be rejected" + ); + } + } + + #[test] + fn parse_datatable_migration_diff_path_roundtrips() { + assert_eq!( + parse_datatable_migration_diff_path("mydt/20260101000001_create_users.up.sql"), + Some(("mydt".to_string(), 20260101000001)) + ); + // up and down map to the same (datatable, timestamp) record. + assert_eq!( + parse_datatable_migration_diff_path("mydt/20260101000001_create_users.down.sql"), + Some(("mydt".to_string(), 20260101000001)) + ); + // datatable names may themselves be hyphenated. + assert_eq!( + parse_datatable_migration_diff_path("my-dt/42_x.up.sql"), + Some(("my-dt".to_string(), 42)) + ); + } + + #[test] + fn parse_datatable_migration_diff_path_rejects_malformed() { + // no slash → not a migration path + assert_eq!(parse_datatable_migration_diff_path("nofile"), None); + // filename not starting with digits → no timestamp + assert_eq!( + parse_datatable_migration_diff_path("mydt/create_users.up.sql"), + None + ); + // empty filename + assert_eq!(parse_datatable_migration_diff_path("mydt/"), None); + } + + async fn seed_migration(pool: &DB, w_id: &str, datatable: &str, timestamp: i64, name: &str) { + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) \ + VALUES ($1, $2, $3, $4, 'select 1;')", + ) + .bind(w_id) + .bind(datatable) + .bind(timestamp) + .bind(name) + .execute(pool) + .await + .unwrap(); + } + + async fn migration_keys(pool: &DB, w_id: &str) -> Vec<(String, String)> { + sqlx::query_as::<_, (String, String)>( + "SELECT datatable, name FROM datatable_migrations WHERE workspace_id = $1 \ + ORDER BY datatable, timestamp", + ) + .bind(w_id) + .fetch_all(pool) + .await + .unwrap() + } + + // The cascade keeps each data table's migrations attached to its name when a + // data table is renamed, drops them when it is deleted, and survives a swap + // (A->B, B->A) at a shared timestamp without a primary-key collision. + #[sqlx::test(migrations = "../migrations")] + async fn cascade_renames_and_deletes_datatable_migrations(pool: DB) { + let w_id = format!("dtmig{}", uuid::Uuid::new_v4().simple()); + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')") + .bind(&w_id) + .execute(&pool) + .await + .unwrap(); + + // rename a -> a2, delete d, and swap sa <-> sb (both at timestamp 5000) + seed_migration(&pool, &w_id, "a", 1, "a_mig").await; + seed_migration(&pool, &w_id, "d", 1, "d_mig").await; + seed_migration(&pool, &w_id, "sa", 5000, "sa_mig").await; + seed_migration(&pool, &w_id, "sb", 5000, "sb_mig").await; + + let mut tx = pool.begin().await.unwrap(); + cascade_datatable_migration_renames_and_deletes( + &pool, + &mut tx, + &w_id, + &[ + DatatableRename { from: "a".to_string(), to: "a2".to_string() }, + DatatableRename { from: "sa".to_string(), to: "sb".to_string() }, + DatatableRename { from: "sb".to_string(), to: "sa".to_string() }, + ], + &["d".to_string()], + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!( + migration_keys(&pool, &w_id).await, + vec![ + ("a2".to_string(), "a_mig".to_string()), + ("sa".to_string(), "sb_mig".to_string()), + ("sb".to_string(), "sa_mig".to_string()), + ] + ); + } + + // A fork inherits its parent's migration definitions unchanged. + #[sqlx::test(migrations = "../migrations")] + async fn clone_copies_datatable_migrations_to_target(pool: DB) { + let src = format!("src{}", uuid::Uuid::new_v4().simple()); + let dst = format!("dst{}", uuid::Uuid::new_v4().simple()); + for w in [&src, &dst] { + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')", + ) + .bind(w) + .execute(&pool) + .await + .unwrap(); + } + seed_migration(&pool, &src, "customers", 1, "create_customers").await; + seed_migration(&pool, &src, "customers", 2, "add_index").await; + seed_migration(&pool, &src, "orders", 3, "create_orders").await; + + let mut tx = pool.begin().await.unwrap(); + clone_datatable_migrations(&mut tx, &src, &dst) + .await + .unwrap(); + tx.commit().await.unwrap(); + + // the target ends up with an identical set, and the source is untouched. + let expected = vec![ + ("customers".to_string(), "create_customers".to_string()), + ("customers".to_string(), "add_index".to_string()), + ("orders".to_string(), "create_orders".to_string()), + ]; + assert_eq!(migration_keys(&pool, &dst).await, expected); + assert_eq!(migration_keys(&pool, &src).await, expected); + } +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index b3e9853de9..c2b62d450b 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,3 +1,4 @@ +pub mod datatable_migrations; pub mod deployment_requests; pub mod workspaces; pub mod workspaces_extra; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8d53562635..0b5b600bf1 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -132,6 +132,7 @@ pub fn workspaced_service() -> Router { get(get_datatable_table_schema), ) .route("/edit_datatable_config", post(edit_datatable_config)) + .merge(crate::datatable_migrations::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) @@ -430,6 +431,12 @@ pub struct DucklakeSettings { #[derive(Deserialize, Debug)] struct EditDataTableConfig { settings: DataTableSettings, + // Data table renames (old -> new) and deletions, tracked client-side by a + // stable id, so we can cascade or drop each data table's migrations. + #[serde(default)] + renames: Vec, + #[serde(default)] + deleted_datatables: Vec, } #[derive(Deserialize, Serialize, Debug)] @@ -1973,8 +1980,8 @@ pub(crate) async fn resolve_pg_source_checked( } /// A temporary file for pg_dump output that is automatically deleted when dropped. -struct DumpFile { - path: std::path::PathBuf, +pub(crate) struct DumpFile { + pub(crate) path: std::path::PathBuf, } impl DumpFile { @@ -2021,7 +2028,11 @@ impl Drop for DumpFile { /// Run pg_dump against a PgDatabase, writing output to a temp file on disk. /// Returns a DumpFile handle; the file is deleted when the handle is dropped. -async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result { +pub(crate) async fn pg_dump_database( + pg_db: &PgDatabase, + schema_only: bool, + exclude_tables: &[&str], +) -> Result { let dump_file = DumpFile::new()?; let host = &pg_db.host; @@ -2034,6 +2045,9 @@ async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result, ) -> Result { let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let dump_file = pg_dump_database(&pg, true).await?; + let dump_file = pg_dump_database(&pg, true, &[]).await?; tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) @@ -2416,13 +2430,57 @@ async fn edit_datatable_config( Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, email, .. }: ApiAuthed, - Json(new_config): Json, + Json(mut new_config): Json, ) -> Result { require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &email).await.is_ok(); let mut tx = db.begin().await?; + let old_datatables: HashMap = serde_json::from_value( + sqlx::query_scalar!( + "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null), + ) + .unwrap_or_default(); + + // Validate every persisted data table name and rename segment before + // touching anything, since they become directory segments in migration + // storage/export keys (`migrations/datatable//...`). + for name in new_config.settings.datatables.keys() { + crate::datatable_migrations::validate_datatable_path_segment(name)?; + } + for r in &new_config.renames { + crate::datatable_migrations::validate_datatable_path_segment(&r.from)?; + crate::datatable_migrations::validate_datatable_path_segment(&r.to)?; + } + + // Map new name -> old name so a renamed data table inherits the previous + // flag instead of being treated as brand new. + let rename_src: HashMap<&str, &str> = new_config + .renames + .iter() + .map(|r| (r.to.as_str(), r.from.as_str())) + .collect(); + + // Migrations opt-in is owned by the enable/disable endpoints, not this config + // form: preserve each existing data table's flag, and default brand-new data + // tables to enabled. + for (name, dt) in new_config.settings.datatables.iter_mut() { + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + dt.migrations_enabled = match old_datatables.get(lookup) { + Some(old) => old.migrations_enabled, + None => Some(true), + }; + } + let args_for_audit = format!("{:?}", new_config.settings); audit_log( &mut *tx, @@ -2437,19 +2495,6 @@ async fn edit_datatable_config( // Check that non-superadmins are not abusing Instance databases if !is_superadmin { - let old_datatables = sqlx::query_scalar!( - r#" - SELECT ws.datatable->'datatables' AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(serde_json::Value::Null); - let old_datatables: HashMap = - serde_json::from_value(old_datatables).unwrap_or_default(); for (name, dt) in new_config.settings.datatables.iter() { if dt.database.resource_type == DataTableCatalogResourceType::Instance { let old_dt = old_datatables.get(name); @@ -2478,6 +2523,15 @@ async fn edit_datatable_config( .execute(&mut *tx) .await?; + crate::datatable_migrations::cascade_datatable_migration_renames_and_deletes( + &db, + &mut tx, + &w_id, + &new_config.renames, + &new_config.deleted_datatables, + ) + .await?; + tx.commit().await?; Ok(format!("Edit datatable config for workspace {}", &w_id)) @@ -4068,6 +4122,15 @@ async fn clone_workspace_data( // Clone workspace settings (merge with existing basic settings) update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; + // Clone data table migration definitions (the settings above carry the data + // table config; this carries their migration history). + crate::datatable_migrations::clone_datatable_migrations( + tx, + source_workspace_id, + target_workspace_id, + ) + .await?; + // Clone workspace environment variables clone_workspace_env(tx, source_workspace_id, target_workspace_id).await?; @@ -7418,6 +7481,7 @@ pub struct CompareSummary { pub folders_changed: usize, pub schedules_changed: usize, pub triggers_changed: usize, + pub datatable_migrations_changed: usize, pub conflicts: usize, // Items that are both ahead and behind } @@ -7609,6 +7673,15 @@ async fn compare_workspaces( compare_two_folders(&db, &source_workspace_id, &fork_workspace_id, &item.path) .await?, ), + "datatable_migration" => Some( + crate::datatable_migrations::compare_two_datatable_migration( + &db, + &source_workspace_id, + &fork_workspace_id, + &item.path, + ) + .await?, + ), // Triggers and schedules are diffed against a hardcoded ignore list // (mode/enabled/server_id/last_server_ping/edited_at/by/error/extra_perms/permissioned_as/email) // so that fork-clones — which differ from the parent only in the runtime @@ -7731,6 +7804,10 @@ async fn compare_workspaces( .iter() .filter(|s| s.kind.ends_with("_trigger")) .count(), + datatable_migrations_changed: visible_diffs + .iter() + .filter(|s| s.kind == "datatable_migration") + .count(), conflicts: visible_diffs .iter() .filter(|s| s.ahead > 0 && s.behind > 0) @@ -8036,6 +8113,45 @@ async fn query_visible_items<'c>( .fetch_all(&mut **tx) .await? } + "datatable_migration" => { + // Match by (datatable, timestamp), not the full path: a migration + // keeps its identity across a rename, so the candidate path's + // `name` segment can differ from the stored one. Parse each + // `/_` candidate, probe existence by + // (datatable, timestamp), and return the *original* candidate path + // so the visibility set stays keyed by the diff's path. + let parsed: Vec<(String, i64, String)> = paths_vec + .iter() + .filter_map(|p| { + let (dt, rest) = p.split_once('/')?; + let ts = rest.split_once('_')?.0.parse::().ok()?; + Some((dt.to_string(), ts, p.clone())) + }) + .collect(); + if parsed.is_empty() { + vec![] + } else { + let dts: Vec = parsed.iter().map(|(d, _, _)| d.clone()).collect(); + let tss: Vec = parsed.iter().map(|(_, t, _)| *t).collect(); + let existing: HashSet<(String, i64)> = sqlx::query!( + "SELECT datatable, timestamp FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = ANY($2) AND timestamp = ANY($3)", + workspace_id, + &dts, + &tss, + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|r| (r.datatable, r.timestamp)) + .collect(); + parsed + .into_iter() + .filter(|(d, t, _)| existing.contains(&(d.clone(), *t))) + .map(|(_, _, p)| p) + .collect() + } + } k if TRIGGER_OR_SCHEDULE_TABLES.contains(&k) => { // SAFETY: `kind` comes from a hardcoded allowlist // TRIGGER_OR_SCHEDULE_TABLES, not user input. @@ -8103,10 +8219,10 @@ async fn existing_runnables( } #[derive(Debug)] -struct ItemComparison { - has_changes: bool, - exists_in_source: bool, - exists_in_fork: bool, +pub(crate) struct ItemComparison { + pub(crate) has_changes: bool, + pub(crate) exists_in_source: bool, + pub(crate) exists_in_fork: bool, } async fn compare_two_scripts( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 323de509bd..177b22662d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4631,6 +4631,22 @@ paths: properties: settings: $ref: "#/components/schemas/DataTableSettings" + renames: + description: data tables renamed in this save, so their migrations cascade + type: array + items: + type: object + required: [from, to] + properties: + from: + type: string + to: + type: string + deleted_datatables: + description: data tables removed in this save, so their migrations are deleted + type: array + items: + type: string responses: "200": description: status @@ -4638,6 +4654,307 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}: + post: + summary: run pending datatable migrations against a datatable + operationId: runDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: up_to + in: query + required: false + description: only apply pending migrations up to and including this version + schema: + type: integer + format: int64 + - name: only + in: query + required: false + description: apply only this specific migration version, ignoring others + schema: + type: integer + format: int64 + responses: + "200": + description: applied migrations + content: + application/json: + schema: + type: object + required: [applied] + properties: + applied: + type: array + items: + type: object + required: [version, name] + properties: + version: + type: integer + format: int64 + name: + type: string + + /w/{workspace}/workspaces/rollback_datatable_migrations/{datatable_name}: + post: + summary: roll back the most recently applied migration on a datatable + operationId: rollbackDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: only + in: query + required: false + description: roll back this specific applied migration version instead of the latest + schema: + type: integer + format: int64 + responses: + "200": + description: rolled back migrations + content: + application/json: + schema: + type: object + required: [rolled_back] + properties: + rolled_back: + type: array + items: + type: object + required: [version, name] + properties: + version: + type: integer + format: int64 + name: + type: string + + /w/{workspace}/workspaces/list_datatable_migrations: + get: + summary: list datatable migrations for a workspace + operationId: listDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: datatable migrations + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/datatable_migrations_status/{datatable_name}: + get: + summary: list a datatable's migrations with their applied status + operationId: getDatatableMigrationsStatus + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: migrations with status + content: + application/json: + schema: + type: object + required: [enabled, migrations] + properties: + enabled: + type: boolean + migrations: + type: array + items: + $ref: "#/components/schemas/DatatableMigrationWithStatus" + error: + type: string + + /w/{workspace}/workspaces/enable_datatable_migrations/{datatable_name}: + post: + summary: opt a datatable in to migrations (admins / super admins only) + operationId: enableDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/disable_datatable_migrations/{datatable_name}: + post: + summary: opt a datatable out of migrations, deleting all of them (admins / super admins only) + operationId: disableDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/create_datatable_migration/{datatable_name}: + post: + summary: create a single datatable migration (version generated server-side) + operationId: createDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name, code_up] + properties: + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/delete_datatable_migration/{datatable_name}/{timestamp}: + delete: + summary: delete a single datatable migration definition + operationId: deleteDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: timestamp + in: path + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/upsert_datatable_migration/{datatable_name}: + post: + summary: insert or update a single datatable migration at an explicit version + operationId: upsertDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [timestamp, name, code_up] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/generate_initial_datatable_migration/{datatable_name}: + post: + summary: snapshot the current schema as an already-installed initial migration + operationId: generateInitialDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + /w/{workspace}/workspaces/create_pg_database: post: summary: create a new PostgreSQL database for a datatable @@ -29067,6 +29384,9 @@ components: type: string required: - resource_type + migrations_enabled: + type: boolean + description: Whether the SQL migrations feature is opted in for this data table forked_from: type: object description: Fork origin info with schema snapshot @@ -29075,6 +29395,40 @@ components: type: object description: Schema snapshot at fork time additionalProperties: true + DatatableMigration: + type: object + required: [datatable, timestamp, name, code_up] + properties: + datatable: + type: string + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + DatatableMigrationWithStatus: + type: object + required: [timestamp, name, code_up, status] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + status: + type: string + enum: + - ran + - not_run + - unknown DataTableSchema: type: object required: [datatable_name, schemas] @@ -29786,6 +30140,7 @@ components: - folders_changed - schedules_changed - triggers_changed + - datatable_migrations_changed - conflicts properties: total_diffs: @@ -29824,6 +30179,9 @@ components: triggers_changed: type: integer description: Number of triggers with differences (sum across all trigger kinds) + datatable_migrations_changed: + type: integer + description: Number of data table migrations with differences conflicts: type: integer description: Number of items that are both ahead and behind (conflicts) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index fa112ef4e1..839057b4b9 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1546,6 +1546,34 @@ pub(crate) async fn tarball_workspace( .await?; } + { + // Data table migrations live in the `datatable_migrations` table; surface + // them in the export as `migrations/datatable//_` + // .up.sql (and .down.sql when present) so `wmill sync` treats them like any + // other workspace item. + let migrations = sqlx::query!( + "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 ORDER BY datatable, timestamp", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + for m in migrations { + let base = format!( + "migrations/datatable/{}/{}_{}", + m.datatable, m.timestamp, m.name + ); + archive + .write_to_archive(&m.code_up, &format!("{base}.up.sql")) + .await?; + if let Some(code_down) = m.code_down { + archive + .write_to_archive(&code_down, &format!("{base}.down.sql")) + .await?; + } + } + } + archive.finish().await?; let file = tokio::fs::File::open(&file_path).await?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 3b5220d46f..a8c6e26f0e 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -164,6 +164,7 @@ pub enum ObjectType { Settings, Key, WorkspaceDependencies, + DatatableMigration, } pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28719/sync-script-to-git-repo-windmill"; @@ -771,6 +772,11 @@ pub struct DataTable { pub database: DataTableDatabase, #[serde(default, skip_serializing_if = "Option::is_none")] pub forked_from: Option, + /// Whether the SQL-migrations feature is opted in for this data table. + /// Absent on data tables created before the feature: treated as enabled only + /// when migrations already exist (see `datatable_migrations_enabled`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub migrations_enabled: Option, } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index b10f62e7a5..6c1b9bf89b 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -24,30 +24,102 @@ pub use git_sync_oss::{ #[derive(Clone, Debug)] pub enum DeployedObject { - Script { hash: ScriptHash, path: String, parent_path: Option }, - Flow { path: String, parent_path: Option, version: i64 }, - App { path: String, version: i64, parent_path: Option }, - RawApp { path: String, version: i64, parent_path: Option }, - Folder { path: String }, - Resource { path: String, parent_path: Option }, - Variable { path: String, parent_path: Option }, - Schedule { path: String }, - ResourceType { path: String }, - User { email: String }, - Group { name: String }, - HttpTrigger { path: String, parent_path: Option }, - WebsocketTrigger { path: String, parent_path: Option }, - KafkaTrigger { path: String, parent_path: Option }, - NatsTrigger { path: String, parent_path: Option }, - PostgresTrigger { path: String, parent_path: Option }, - MqttTrigger { path: String, parent_path: Option }, - SqsTrigger { path: String, parent_path: Option }, - GcpTrigger { path: String, parent_path: Option }, - AzureTrigger { path: String, parent_path: Option }, - EmailTrigger { path: String, parent_path: Option }, - Settings { setting_type: String }, - Key { key_type: String }, - WorkspaceDependencies { path: String }, + Script { + hash: ScriptHash, + path: String, + parent_path: Option, + }, + Flow { + path: String, + parent_path: Option, + version: i64, + }, + App { + path: String, + version: i64, + parent_path: Option, + }, + RawApp { + path: String, + version: i64, + parent_path: Option, + }, + Folder { + path: String, + }, + Resource { + path: String, + parent_path: Option, + }, + Variable { + path: String, + parent_path: Option, + }, + Schedule { + path: String, + }, + ResourceType { + path: String, + }, + User { + email: String, + }, + Group { + name: String, + }, + HttpTrigger { + path: String, + parent_path: Option, + }, + WebsocketTrigger { + path: String, + parent_path: Option, + }, + KafkaTrigger { + path: String, + parent_path: Option, + }, + NatsTrigger { + path: String, + parent_path: Option, + }, + PostgresTrigger { + path: String, + parent_path: Option, + }, + MqttTrigger { + path: String, + parent_path: Option, + }, + SqsTrigger { + path: String, + parent_path: Option, + }, + GcpTrigger { + path: String, + parent_path: Option, + }, + AzureTrigger { + path: String, + parent_path: Option, + }, + EmailTrigger { + path: String, + parent_path: Option, + }, + Settings { + setting_type: String, + }, + Key { + key_type: String, + }, + WorkspaceDependencies { + path: String, + }, + /// A single data table migration, identified by `/_`. + DatatableMigration { + path: String, + }, } impl DeployedObject { @@ -77,6 +149,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => "settings.yaml".to_string(), DeployedObject::Key { .. } => "encryption_key.yaml".to_string(), DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(), + DeployedObject::DatatableMigration { path } => path.to_owned(), } } @@ -118,6 +191,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => None, DeployedObject::Key { .. } => None, DeployedObject::WorkspaceDependencies { .. } => None, + DeployedObject::DatatableMigration { .. } => None, } } @@ -147,6 +221,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => "settings", DeployedObject::Key { .. } => "key", DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies", + DeployedObject::DatatableMigration { .. } => "datatable_migration", } .to_string() } diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts index 1291d4cc13..6609d08c50 100644 --- a/cli/src/commands/datatable/datatable.ts +++ b/cli/src/commands/datatable/datatable.ts @@ -9,6 +9,13 @@ import { GlobalOptions } from "../../types.ts"; import { runCatalogQuery } from "../../utils/catalog.ts"; import { psql as psqlDatatable } from "./psql.ts"; import { serve as serveDatatable } from "./serve.ts"; +import { + createMigration, + pushLocalMigrations, + rollbackMigrations, + runMigrations, + validateLocalMigrations, +} from "../datatable_migrations.ts"; const DEFAULT_DATATABLE_NAME = "main"; @@ -41,6 +48,69 @@ async function run( await runCatalogQuery(opts, "datatable", name, sql); } +function migrateNew( + opts: GlobalOptions & { datatable?: string }, + name: string, +) { + createMigration(opts.datatable ?? DEFAULT_DATATABLE_NAME, name); +} + +async function migrateUp(opts: GlobalOptions & { datatable?: string }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dt = opts.datatable ?? DEFAULT_DATATABLE_NAME; + // Reject malformed local migrations (duplicate timestamps, orphan downs) before + // pushing — the same check `wmill sync push` runs — so a duplicate timestamp + // can't silently overwrite one migration on upsert. + const errors = validateLocalMigrations(new Set([dt])); + if (errors.length > 0) { + log.error( + "Invalid datatable migrations, aborting:\n" + + errors.map((e) => ` - ${e}`).join("\n"), + ); + process.exit(1); + } + // Push any locally-created/edited migration files first (without running + // them), so `migrate up` works even before a `wmill sync push`. + await pushLocalMigrations(workspace.workspaceId, dt); + await runMigrations(workspace.workspaceId, dt); +} + +async function migrateDown(opts: GlobalOptions & { datatable?: string }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dt = opts.datatable ?? DEFAULT_DATATABLE_NAME; + await rollbackMigrations(workspace.workspaceId, dt); +} + +const migrateCommand = new Command() + .description("manage datatable migrations") + .command("new", "scaffold a new migration (.up.sql / .down.sql files)") + .arguments("") + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateNew as any) + .command( + "up", + "apply all pending migrations to the main datatable (or one via --datatable)", + ) + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateUp as any) + .command( + "down", + "roll back the most recent migration on the main datatable (or one via --datatable)", + ) + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateDown as any); + async function create( opts: GlobalOptions & { resource?: string; force?: boolean }, name?: string, @@ -124,6 +194,7 @@ const command = new Command() "Output only the final result as JSON. Useful for scripting.", ) .action(run as any) + .command("migrate", migrateCommand) .command( "create", "register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://", diff --git a/cli/src/commands/datatable_migrations.ts b/cli/src/commands/datatable_migrations.ts new file mode 100644 index 0000000000..4fcaffd4fa --- /dev/null +++ b/cli/src/commands/datatable_migrations.ts @@ -0,0 +1,340 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as log from "../core/log.ts"; +import { colors } from "@cliffy/ansi/colors"; +import * as wmill from "../../gen/services.gen.ts"; +import { readTextFile } from "../utils/utils.ts"; +import { Confirm } from "@cliffy/prompt/confirm"; + +// Migrations live under /migrations/datatable//, one folder per +// target data table, as `_.up.sql` (and optional `.down.sql`). +// They are synced as ordinary workspace files (see the workspace tarball export +// and the `datatable_migration` handling in sync.ts); this module only holds the +// `wmill datatable migrate` command helpers and the per-file push primitive. +const MIGRATIONS_DIR = path.join("migrations", "datatable"); + +// Migration names map directly onto file names and the DB `name` column. +const MIGRATION_NAME_RE = /^[a-zA-Z0-9_-]+$/; + +/** Current UTC time as a YYYYMMDDHHMMSS migration version. */ +function migrationTimestamp(): string { + const d = new Date(); + const p = (n: number) => String(n).padStart(2, "0"); + return ( + `${d.getUTCFullYear()}${p(d.getUTCMonth() + 1)}${p(d.getUTCDate())}` + + `${p(d.getUTCHours())}${p(d.getUTCMinutes())}${p(d.getUTCSeconds())}` + ); +} + +/** + * A migration version unique within a data table folder: the current UTC + * timestamp bumped past any existing version, so two migrations scaffolded in + * the same second don't collide on the `(datatable, timestamp)` identity used to + * upsert them. + */ +function nextMigrationTimestamp(dir: string): string { + const now = Number(migrationTimestamp()); + let max = 0; + if (fs.existsSync(dir)) { + for (const file of fs.readdirSync(dir)) { + const m = file.match(/^(\d+)_.*\.(up|down)\.sql$/); + if (m) max = Math.max(max, Number(m[1])); + } + } + return String(max >= now ? max + 1 : now); +} + +/** + * Scaffold a new migration under migrations/datatable// as empty + * `_.up.sql` and `.down.sql` files. Purely local — no network. + */ +export function createMigration(datatable: string, name: string): void { + if (!MIGRATION_NAME_RE.test(name)) { + throw new Error( + `Invalid migration name '${name}': use only letters, digits, '_' and '-'`, + ); + } + const dir = path.join(process.cwd(), MIGRATIONS_DIR, datatable); + fs.mkdirSync(dir, { recursive: true }); + + const timestamp = nextMigrationTimestamp(dir); + const base = `${timestamp}_${name}`; + const up = path.join(dir, `${base}.up.sql`); + const down = path.join(dir, `${base}.down.sql`); + // Frame the body in an explicit transaction so it applies atomically, matching + // the template the UI's "New migration" modal seeds. + const template = (direction: string) => + `-- ${direction} migration: ${name}\nBEGIN;\n\n-- Add your migration here\n\nEND;\n`; + fs.writeFileSync(up, template("up"), "utf-8"); + fs.writeFileSync(down, template("down"), "utf-8"); + + log.info( + colors.green(`Created migration ${base} in ${MIGRATIONS_DIR}/${datatable}/`), + ); + for (const f of [up, down]) { + log.info(colors.gray(` ${path.relative(process.cwd(), f)}`)); + } +} + +/** + * Apply the workspace's pending migrations to a data table (forwards migrations + * recorded in `_wm_migrations`). Mirrors `wmill datatable migrate up`. + */ +export async function runMigrations( + workspace: string, + datatableName: string, +): Promise { + const result = await wmill.runDatatableMigrations({ + workspace, + datatableName, + }); + const applied = result.applied ?? []; + if (applied.length === 0) { + log.info(colors.gray(`No pending migrations to run on '${datatableName}'`)); + return; + } + log.info( + colors.green(`Applied ${applied.length} migration(s) to '${datatableName}':`), + ); + for (const m of applied) { + log.info(colors.gray(` ${m.version} ${m.name}`)); + } +} + +/** + * Roll back the most recently applied migration on a data table (one step). + * Mirrors `wmill datatable migrate down`. + */ +export async function rollbackMigrations( + workspace: string, + datatableName: string, +): Promise { + const result = await wmill.rollbackDatatableMigrations({ + workspace, + datatableName, + }); + const rolledBack = result.rolled_back ?? []; + if (rolledBack.length === 0) { + log.info( + colors.gray(`No applied migrations to roll back on '${datatableName}'`), + ); + return; + } + for (const m of rolledBack) { + log.info( + colors.green(`Rolled back migration ${m.version} ${m.name} on '${datatableName}'`), + ); + } +} + +/** + * Validate the on-disk migration files for the given data tables (or all of + * them when `datatables` is omitted). Returns a list of human-readable problems; + * an empty list means the migrations are well-formed. Two states are invalid: + * - two up (or two down) files sharing the same timestamp, which collide on the + * `(datatable, timestamp)` identity used to upsert; and + * - a `.down.sql` with no matching `.up.sql` (an up file is mandatory). + */ +export function validateLocalMigrations(datatables?: Set): string[] { + const errors: string[] = []; + const root = path.join(process.cwd(), MIGRATIONS_DIR); + if (!fs.existsSync(root)) return errors; + + for (const datatable of fs.readdirSync(root)) { + if (datatables && !datatables.has(datatable)) continue; + const dtDir = path.join(root, datatable); + if (!fs.statSync(dtDir).isDirectory()) continue; + + const upNamesByTs = new Map(); + const downNamesByTs = new Map(); + const upBases = new Set(); + const downBases: { ts: number; name: string }[] = []; + + for (const file of fs.readdirSync(dtDir)) { + const m = file.match(/^(\d+)_(.*)\.(up|down)\.sql$/); + if (!m) continue; + const ts = Number(m[1]); + const name = m[2]; + if (m[3] === "up") { + (upNamesByTs.get(ts) ?? upNamesByTs.set(ts, []).get(ts)!).push(name); + upBases.add(`${ts}_${name}`); + } else { + (downNamesByTs.get(ts) ?? downNamesByTs.set(ts, []).get(ts)!).push(name); + downBases.push({ ts, name }); + } + } + + for (const [ts, names] of upNamesByTs) { + if (names.length > 1) { + errors.push( + `${datatable}: ${names.length} up migrations share timestamp ${ts} (${names.join(", ")})`, + ); + } + } + for (const [ts, names] of downNamesByTs) { + if (names.length > 1) { + errors.push( + `${datatable}: ${names.length} down migrations share timestamp ${ts} (${names.join(", ")})`, + ); + } + } + for (const d of downBases) { + if (!upBases.has(`${d.ts}_${d.name}`)) { + errors.push( + `${datatable}: ${d.ts}_${d.name}.down.sql has no matching ${d.ts}_${d.name}.up.sql`, + ); + } + } + } + + return errors; +} + +/** + * Sync a single migration to the workspace based on the current on-disk state of + * its `/_.up.sql` file: upsert it when the up file + * exists, otherwise delete it. Called by `wmill sync push` for each changed + * `datatable_migration` file. + */ +export async function pushMigrationFromDisk( + workspace: string, + m: { datatable: string; timestamp: number }, +): Promise { + const dir = path.join(process.cwd(), MIGRATIONS_DIR, m.datatable); + // Find the up file for this timestamp regardless of its name segment. A rename + // (`123_old.up.sql` -> `123_new.up.sql`) keeps the (datatable, timestamp) + // identity but changes the name; the diff sorter may process the deleted old + // path before the added new one, so keying off the passed name would delete + // the record. Scanning by timestamp upserts the surviving file instead. + const upFile = fs.existsSync(dir) + ? fs.readdirSync(dir).find((f) => { + const parsed = f.match(/^(\d+)_(.*)\.up\.sql$/); + return parsed !== null && Number(parsed[1]) === m.timestamp; + }) + : undefined; + + if (upFile === undefined) { + log.info(colors.red(`Deleting datatable_migration ${m.datatable}/${m.timestamp}`)); + await wmill.deleteDatatableMigration({ + workspace, + datatableName: m.datatable, + timestamp: m.timestamp, + }); + return; + } + + const name = upFile.match(/^(\d+)_(.*)\.up\.sql$/)![2]; + const base = `${m.timestamp}_${name}`; + const code_up = await readTextFile(path.join(dir, upFile)); + const downPath = path.join(dir, `${base}.down.sql`); + const code_down = fs.existsSync(downPath) ? await readTextFile(downPath) : undefined; + + log.info(colors.green(`Pushing datatable_migration ${m.datatable}/${base}`)); + await wmill.upsertDatatableMigration({ + workspace, + datatableName: m.datatable, + requestBody: { + timestamp: m.timestamp, + name, + code_up, + ...(code_down !== undefined ? { code_down } : {}), + }, + }); +} + +/** + * Upsert the on-disk migrations of a data table to the workspace, so a freshly + * created migration file works with `wmill datatable migrate up` even without a + * prior `wmill sync push`. Pushes only migrations that are new or edited + * (compared against the workspace's current definitions); it never deletes + * remote migrations absent on disk and never touches other item kinds. + */ +export async function pushLocalMigrations( + workspace: string, + datatableName: string, +): Promise { + const dir = path.join(process.cwd(), MIGRATIONS_DIR, datatableName); + if (!fs.existsSync(dir)) return; + + // Local migrations are identified by their `.up.sql` file (the up file is + // mandatory); this deliberately ignores files that were only deleted locally. + const local: { timestamp: number; name: string }[] = []; + for (const file of fs.readdirSync(dir)) { + const m = file.match(/^(\d+)_(.*)\.up\.sql$/); + if (m) local.push({ timestamp: Number(m[1]), name: m[2] }); + } + if (local.length === 0) return; + + const remote = await wmill.listDatatableMigrations({ workspace }); + const remoteByTs = new Map( + remote + .filter((r) => r.datatable === datatableName) + .map((r) => [r.timestamp, r] as const), + ); + + for (const { timestamp, name } of local) { + const base = `${timestamp}_${name}`; + const code_up = await readTextFile(path.join(dir, `${base}.up.sql`)); + const downPath = path.join(dir, `${base}.down.sql`); + const code_down = fs.existsSync(downPath) + ? await readTextFile(downPath) + : undefined; + + const r = remoteByTs.get(timestamp); + const unchanged = + r !== undefined && + r.name === name && + r.code_up === code_up && + (r.code_down ?? undefined) === code_down; + if (unchanged) continue; + + log.info(colors.green(`Pushing datatable_migration ${datatableName}/${base}`)); + await wmill.upsertDatatableMigration({ + workspace, + datatableName, + requestBody: { + timestamp, + name, + code_up, + ...(code_down !== undefined ? { code_down } : {}), + }, + }); + } +} + +/** + * After a push that introduced new migrations, list them and (interactively) + * offer to run them, equivalent to `wmill datatable migrate up` on each affected + * data table. + */ +export async function offerToRunNewMigrations( + workspace: string, + newMigrations: { datatable: string; timestamp: number; name: string }[], + opts?: { yes?: boolean; jsonOutput?: boolean }, +): Promise { + if (newMigrations.length === 0) return; + + log.info(colors.green("New migrations were pushed:")); + for (const m of newMigrations) { + log.info(colors.gray(` ${m.datatable}: ${m.timestamp} ${m.name}`)); + } + + // Running migrations mutates the data tables, so skip the prompt in + // non-interactive contexts (--yes, --json, no TTY). + const interactive = !opts?.jsonOutput && !opts?.yes && !!process.stdin.isTTY; + if (!interactive) { + return; + } + + const shouldRun = await Confirm.prompt({ + message: "New migrations were pushed, run them?", + default: false, + }); + if (!shouldRun) { + return; + } + + for (const datatable of new Set(newMigrations.map((m) => m.datatable))) { + await runMigrations(workspace, datatable); + } +} diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index 8517ac0912..7eab79b52d 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -2,7 +2,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { colors } from "@cliffy/ansi/colors"; import { sep as SEP } from "node:path"; -import { GlobalOptions } from "../../types.ts"; +import { GlobalOptions, isDatatableMigrationPath } from "../../types.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; @@ -54,6 +54,8 @@ async function walkLocalScripts( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || isFolderResourcePathAnyFormat(p) || + // Datatable migration `.sql` files aren't Windmill scripts. + isDatatableMigrationPath(p) || (isScriptModulePath(p) && !isModuleEntryPoint(p)), false, {}, @@ -221,6 +223,8 @@ function categorizeLocalFiles( } else if ( exts.some((ext) => p.endsWith(ext)) && !isFolderResourcePathAnyFormat(p) && + // Datatable migration `.sql` files aren't Windmill scripts. + !isDatatableMigrationPath(p) && !(isScriptModulePath(p) && !isModuleEntryPoint(p)) ) { scripts.push(p); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 8e1ecdfef6..65ce10eaad 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -23,10 +23,17 @@ import { showDiff, extractNativeTriggerInfo, redactEncryptionKey, + isDatatableMigrationPath, + parseDatatableMigrationPath, } from "../../types.ts"; import { downloadZip } from "./pull.ts"; import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts"; import { pullSharedUi, pushSharedUi } from "../shared_ui.ts"; +import { + pushMigrationFromDisk, + offerToRunNewMigrations, + validateLocalMigrations, +} from "../datatable_migrations.ts"; import { exts, @@ -2477,7 +2484,8 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { !p.startsWith("g" + SEP) && !p.startsWith("users" + SEP) && !p.startsWith("groups" + SEP) && - !p.startsWith("dependencies" + SEP) + !p.startsWith("dependencies" + SEP) && + !p.startsWith("migrations" + SEP) ); } @@ -2488,6 +2496,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { try { const typ = getTypeStrFromPath(p); + // Datatable migrations live under migrations/datatable//, outside + // the u/f/g namespaces, but are valid wmill files. + if (typ == "datatable_migration") { + return false; + } if ( typ == "resource-type" || typ == "settings" || @@ -2519,7 +2532,8 @@ export const isWhitelisted = (p: string) => { p == "ui" || p == "users" || p == "groups" || - p == "dependencies" + p == "dependencies" || + p == "migrations" ); }; @@ -2614,6 +2628,11 @@ interface ChangeTracker { } async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { + // Datatable migration .sql files are not scripts; they're synced via the + // dedicated datatable_migration handler in the push loop. + if (isDatatableMigrationPath(p)) { + return; + } const isScript = exts.some((e) => p.endsWith(e)) && !isFileResource(p) && !isFilesetResource(p); if (isScript) { if (isFlowPath(p)) { @@ -3119,7 +3138,7 @@ export async function pull( change.path.endsWith(".json") ) { log.info( - `Editing ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Editing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3137,7 +3156,7 @@ export async function pull( if (opts.stateful) { await mkdir(path.dirname(stateTarget), { recursive: true }); log.info( - `Adding ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Adding ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3146,7 +3165,7 @@ export async function pull( } await writeFile(target, change.content, "utf-8"); log.info( - `Writing ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Writing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3158,7 +3177,7 @@ export async function pull( } else if (change.name === "deleted") { try { log.info( - `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`, + `Deleting ${changeTypeLabel(change.path)}${change.path}`, ); await rm(target); if (opts.stateful) { @@ -3350,6 +3369,9 @@ export async function pull( log.warn(`Failed to pull shared UI folder: ${e}`); } + // Datatable migrations are part of the workspace export now, so they flow + // through the normal diff/apply above as `datatable_migration` items. + // Git-sync deployment-callback mode stops here: branch checkout + pull have // happened, but commit + push are the caller's job. The hub script does // them in-process with `set_gpg_signing_secret` so the agent's pre-warmed @@ -3425,6 +3447,14 @@ export async function gitDeploy( } as any); } +// Display label for a change's type, with a trailing space. Datatable migrations +// are self-describing via their `migrations/datatable/...` path, so they get no +// label prefix. +function changeTypeLabel(p: string): string { + const t = getTypeStrFromPath(p); + return t === "datatable_migration" ? "" : `${t} `; +} + function prettyChanges( changes: Change[], specificItems?: SpecificItemsConfig, @@ -3456,7 +3486,7 @@ function prettyChanges( if (change.name === "added") { log.info( colors.green( - `+ ${getTypeStrFromPath(change.path)} ` + + `+ ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote), ) + extraNote, @@ -3464,7 +3494,7 @@ function prettyChanges( } else if (change.name === "deleted") { log.info( colors.red( - `- ${getTypeStrFromPath(change.path)} ` + + `- ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote), ), @@ -3473,7 +3503,7 @@ function prettyChanges( const changeType = getTypeStrFromPath(change.path); log.info( colors.yellow( - `~ ${changeType} ` + + `~ ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote) + (change.codebase ? ` (codebase changed)` : ""), @@ -4221,6 +4251,24 @@ export async function push( )); } + // Reject malformed datatable migrations (duplicate timestamps, orphan downs) + // before touching the remote, scanning only the data tables in this push. + const migrationDatatables = new Set( + changes + .map((c) => parseDatatableMigrationPath(c.path)?.datatable) + .filter((d): d is string => !!d), + ); + if (migrationDatatables.size > 0) { + const migrationErrors = validateLocalMigrations(migrationDatatables); + if (migrationErrors.length > 0) { + log.error( + "Invalid datatable migrations, aborting push:\n" + + migrationErrors.map((e) => ` - ${e}`).join("\n"), + ); + process.exit(1); + } + } + if ( !opts.yes && !(await Confirm.prompt({ @@ -4293,6 +4341,21 @@ export async function push( // Cache git branch at the start to avoid repeated execSync calls per change const cachedWsNameForPush = wsNameForFiles || (isGitRepository() ? getCurrentGitBranch() : null); + // Datatable migrations are two files (.up.sql/.down.sql) for one record, so + // dedupe upsert/delete by (datatable, version) across the whole push. + const pushedMigrationKeys = new Set(); + // Migrations newly added by this push (an added .up.sql) — offered to run once + // the push has completed. + const newDatatableMigrations = changes + .filter((c) => c.name === "added") + .map((c) => parseDatatableMigrationPath(c.path)) + .filter((p) => !!p && p.kind === "up") + .map((p) => ({ + datatable: p!.datatable, + timestamp: p!.timestamp, + name: p!.name, + })); + while (queue.length > 0 || pool.size > 0) { // Fill the pool until we reach the effective parallelism limit. // During the folder-meta phase this is 1 (sequential) so no item change @@ -4320,6 +4383,20 @@ export async function push( } for await (const change of changes) { + // A datatable migration is one record across two files; upsert/delete + // it from disk once (deduped), regardless of which file changed. + if (isDatatableMigrationPath(change.path)) { + const parsed = parseDatatableMigrationPath(change.path); + if (parsed) { + const key = `${parsed.datatable}\0${parsed.timestamp}`; + if (!pushedMigrationKeys.has(key)) { + pushedMigrationKeys.add(key); + await pushMigrationFromDisk(workspace.workspaceId, parsed); + } + } + continue; + } + let stateTarget = undefined; if (stateful) { try { @@ -4974,6 +5051,16 @@ export async function push( } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } + try { + await offerToRunNewMigrations(workspace.workspaceId, newDatatableMigrations, { + yes: opts.yes, + jsonOutput: opts.jsonOutput, + }); + } catch (e: any) { + log.warn( + `Failed to run new datatable migrations: ${e?.body ?? e?.message ?? e}`, + ); + } const lockJobs = await checkServerLockJobs( workspace.workspaceId, pushStartedAt, @@ -5039,6 +5126,7 @@ export async function push( } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } + // No changes pushed, so no new datatable migrations to run. if (opts.jsonOutput) { console.log( JSON.stringify( diff --git a/cli/src/commands/workspace/merge.ts b/cli/src/commands/workspace/merge.ts index b8ef24f7d4..585772a9d0 100644 --- a/cli/src/commands/workspace/merge.ts +++ b/cli/src/commands/workspace/merge.ts @@ -11,10 +11,12 @@ import { deleteItemInWorkspace, getOnBehalfOf, isTriggerOrScheduleKind, + parseDatatableMigrationDeployPath, type DeployKind, type DeployProvider, type TriggerDeployKind, } from "../../../windmill-utils-internal/src/deploy.ts"; +import { offerToRunNewMigrations } from "../datatable_migrations.ts"; // --------------------------------------------------------------------------- // Provider adapter — wraps CLI's standalone API functions @@ -85,6 +87,10 @@ const provider: DeployProvider = { createSchedule: wmill.createSchedule, updateSchedule: wmill.updateSchedule, deleteSchedule: wmill.deleteSchedule, + // Datatable migrations + listDatatableMigrations: wmill.listDatatableMigrations, + upsertDatatableMigration: wmill.upsertDatatableMigration, + deleteDatatableMigration: wmill.deleteDatatableMigration, }; /** @@ -530,6 +536,14 @@ async function mergeWorkspaces( // 10. Deploy let successCount = 0; let failCount = 0; + // Datatable migrations deployed (not deleted) into the target. Deploying a + // migration only upserts its definition — the target schema is unchanged until + // the migration is run — so offer to run them afterwards (like the push path). + const deployedMigrations: { + datatable: string; + timestamp: number; + name: string; + }[] = []; for (const diff of sorted) { const label = `${diff.kind}:${diff.path}`; @@ -573,6 +587,12 @@ async function mergeWorkspaces( if (result.success) { log.info(colors.green(` ✓ ${label}`)); successCount++; + if ( + !itemDeletedInSource && + (diff.kind as DeployKind) === "datatable_migration" + ) { + deployedMigrations.push(parseDatatableMigrationDeployPath(diff.path)); + } } else { log.info(colors.red(` ✗ ${label}: ${result.error}`)); failCount++; @@ -606,6 +626,18 @@ async function mergeWorkspaces( ) ); } + + // 13. Deployed migration definitions don't touch the target schema until run; + // offer to run them on the target now (interactive only, like the push path). + if (deployedMigrations.length > 0) { + try { + await offerToRunNewMigrations(workspaceTo, deployedMigrations, { + yes: opts.yes, + }); + } catch (e) { + log.warn(colors.yellow(`Failed to run deployed migrations: ${e}`)); + } + } } export { mergeWorkspaces }; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index e2cf0be86b..03a5c09eb7 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6593,6 +6593,13 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable migrate\` - manage datatable migrations + - \`datatable migrate new \` - scaffold a new migration (.up.sql / .down.sql files) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate up\` - apply all pending migrations to the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate down\` - roll back the most recent migration on the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) - \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -6861,19 +6868,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- \`jobs pull\` -- \`jobs push\` +- \`jobs pull [workspace:string]\` - Pull completed and queued jobs from workspace + - \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) + - \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before export +- \`jobs push [workspace:string]\` - Push completed and queued jobs to workspace + - \`-c, --completed-file \` - Completed jobs input file (default: completed_jobs.json) + - \`-q, --queued-file \` - Queued jobs input file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before import ### lint diff --git a/cli/src/types.ts b/cli/src/types.ts index 5de6cea48b..c125ae2c2e 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -274,10 +274,10 @@ export function parseFromPath(p: string, content: string): any { return isWorkspaceDependencies(p) ? content : p.endsWith(".yaml") - ? yamlParseContent(p, content) - : p.endsWith(".json") - ? JSON.parse(content) - : content; + ? yamlParseContent(p, content) + : p.endsWith(".json") + ? JSON.parse(content) + : content; } export function parseFromFile(p: string): any { if (p.endsWith(".json")) { @@ -288,9 +288,38 @@ export function parseFromFile(p: string): any { throw new Error("Could not read file " + p); } } +/** + * Parse a `migrations/datatable//_.(up|down).sql` + * path into its parts. Returns undefined for any other path. + */ +export function parseDatatableMigrationPath(p: string): + | { datatable: string; timestamp: number; name: string; kind: "up" | "down" } + | undefined { + const parts = p.split("/"); + if ( + parts[0] !== "migrations" || + parts[1] !== "datatable" || + parts.length !== 4 + ) + return undefined; + const m = parts[3].match(/^(\d+)_(.*)\.(up|down)\.sql$/); + if (!m) return undefined; + return { + datatable: parts[2], + timestamp: Number(m[1]), + name: m[2], + kind: m[3] as "up" | "down", + }; +} + +export function isDatatableMigrationPath(p: string): boolean { + return parseDatatableMigrationPath(p) !== undefined; +} + export function getTypeStrFromPath( p: string ): + | "datatable_migration" | "script" | "variable" | "flow" @@ -316,6 +345,9 @@ export function getTypeStrFromPath( | "settings" | "encryption_key" | "workspace_dependencies" { + if (isDatatableMigrationPath(p)) { + return "datatable_migration"; + } if (isScriptModulePath(p)) { return "script"; } diff --git a/cli/test/datatable_migrations_unit.test.ts b/cli/test/datatable_migrations_unit.test.ts new file mode 100644 index 0000000000..09af49f890 --- /dev/null +++ b/cli/test/datatable_migrations_unit.test.ts @@ -0,0 +1,122 @@ +/** + * Unit tests for datatable-migration path parsing and local validation. + * + * These exercise pure logic with no backend: + * - `parseDatatableMigrationPath` recognizes only the + * `migrations/datatable/
/_.(up|down).sql` shape. + * - `validateLocalMigrations` rejects the two invalid on-disk states a push + * must catch: two up (or two down) files sharing a timestamp, and a + * `.down.sql` with no matching `.up.sql`. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { parseDatatableMigrationPath } from "../src/types.ts"; +import { validateLocalMigrations } from "../src/commands/datatable_migrations.ts"; + +describe("parseDatatableMigrationPath", () => { + test("parses up and down files of the new layout", () => { + expect( + parseDatatableMigrationPath( + "migrations/datatable/mydt/20260101000001_create_users.up.sql", + ), + ).toEqual({ + datatable: "mydt", + timestamp: 20260101000001, + name: "create_users", + kind: "up", + }); + expect( + parseDatatableMigrationPath( + "migrations/datatable/my-dt/42_x.down.sql", + ), + ).toEqual({ datatable: "my-dt", timestamp: 42, name: "x", kind: "down" }); + }); + + test("rejects unrelated, legacy and malformed paths", () => { + for ( + const p of [ + // legacy top-level layout + "datatable_migrations/mydt/20260101000001_x.up.sql", + // wrong sub-namespace / depth + "migrations/ducklake/mydt/1_x.up.sql", + "migrations/datatable/1_x.up.sql", + "migrations/datatable/mydt/sub/1_x.up.sql", + // not a migration file + "migrations/datatable/mydt/notes.txt", + "migrations/datatable/mydt/x.up.sql", // no numeric timestamp prefix + // unrelated workspace files + "f/foo/bar.script.yaml", + "u/admin/script.ts", + ] + ) { + expect(parseDatatableMigrationPath(p)).toBeUndefined(); + } + }); +}); + +describe("validateLocalMigrations", () => { + let prevCwd: string; + let tmp: string; + + beforeEach(() => { + prevCwd = process.cwd(); + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "dtmig-")); + process.chdir(tmp); + }); + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + function write(datatable: string, file: string) { + const dir = path.join(tmp, "migrations", "datatable", datatable); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, file), "-- sql\n"); + } + + test("accepts up+down pairs and up-only migrations", () => { + write("mydt", "20260101000001_create_users.up.sql"); + write("mydt", "20260101000001_create_users.down.sql"); + write("mydt", "20260101000002_add_email.up.sql"); // down is optional + expect(validateLocalMigrations()).toEqual([]); + }); + + test("flags two up files sharing a timestamp", () => { + write("mydt", "20260101000003_foo.up.sql"); + write("mydt", "20260101000003_bar.up.sql"); + const errors = validateLocalMigrations(); + expect(errors.length).toBe(1); + expect(errors[0]).toContain("20260101000003"); + }); + + test("flags two down files sharing a timestamp", () => { + write("mydt", "20260101000004_a.up.sql"); + write("mydt", "20260101000004_a.down.sql"); + write("mydt", "20260101000004_b.down.sql"); + const errors = validateLocalMigrations(); + // duplicate down + the b.down orphan (no b.up) + expect(errors.some((e) => e.includes("down") && e.includes("20260101000004"))).toBe(true); + }); + + test("flags a down file with no matching up", () => { + write("mydt", "20260101000005_orphan.down.sql"); + const errors = validateLocalMigrations(); + expect(errors.length).toBe(1); + expect(errors[0]).toContain("20260101000005_orphan"); + }); + + test("only validates the requested datatables", () => { + write("bad", "20260101000006_x.up.sql"); + write("bad", "20260101000006_y.up.sql"); // duplicate, but in 'bad' + write("good", "20260101000007_ok.up.sql"); + expect(validateLocalMigrations(new Set(["good"]))).toEqual([]); + expect(validateLocalMigrations(new Set(["bad"])).length).toBe(1); + }); + + test("returns no errors when the migrations folder is absent", () => { + expect(validateLocalMigrations()).toEqual([]); + }); +}); diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 37cfbfaef4..525bb0fc0b 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.7.0", + "version": "1.8.2", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 990fd47edd..4c2bd735b3 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -21,6 +21,7 @@ export type DeployKind = | "resource_type" | "folder" | "schedule" + | "datatable_migration" | "http_trigger" | "websocket_trigger" | "kafka_trigger" @@ -161,6 +162,24 @@ export interface DeployProvider { requestBody: any; }): Promise; deleteFolder(p: { workspace: string; name: string }): Promise; + // Datatable migrations. In the diff, an item's `path` is + // `/_` (see `parseDatatableMigrationDeployPath`). + listDatatableMigrations(p: { workspace: string }): Promise; + upsertDatatableMigration(p: { + workspace: string; + datatableName: string; + requestBody: { + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }; + }): Promise; + deleteDatatableMigration(p: { + workspace: string; + datatableName: string; + timestamp: number; + }): Promise; // Triggers — per-kind dispatch is delegated to the implementor so the shared // module doesn't need to know about each of the 9 trigger services. existsTriggerByKind( @@ -299,6 +318,40 @@ function toError(e: unknown): string { return err.body || err.message || String(e); } +// A datatable-migration diff item's path is `/_` +// (mirrors the backend, e.g. `mydt/20260101000001_create_users`). +export function parseDatatableMigrationDeployPath(path: string): { + datatable: string; + timestamp: number; + name: string; +} { + const slash = path.indexOf("/"); + const underscore = slash >= 0 ? path.indexOf("_", slash + 1) : -1; + if (slash < 0 || underscore < 0) { + throw new Error(`Invalid datatable migration path: ${path}`); + } + const datatable = path.slice(0, slash); + const timestamp = Number(path.slice(slash + 1, underscore)); + const name = path.slice(underscore + 1); + if (!datatable || !Number.isFinite(timestamp) || !name) { + throw new Error(`Invalid datatable migration path: ${path}`); + } + return { datatable, timestamp, name }; +} + +// The backend rejects `upsertDatatableMigration` when the target data table +// hasn't opted in to migrations. Turn that opaque 400 into an explicit, +// deploy-context message (falls back to the original error otherwise). +function asMigrationsDisabledError(e: unknown, datatable: string): unknown { + const msg = (e as { body?: string; message?: string })?.body ?? '' + if (typeof msg === "string" && /migrations are not enabled/i.test(msg)) { + return new Error( + `Data table '${datatable}' has not opted in to migrations on the target workspace; enable migrations for it there before deploying its migrations.` + ); + } + return e; +} + // --------------------------------------------------------------------------- // checkItemExists // --------------------------------------------------------------------------- @@ -325,6 +378,12 @@ export async function checkItemExists( return provider.existsFolder({ workspace, name: folderName(path) }); } else if (kind === "schedule") { return provider.existsSchedule({ workspace, path }); + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = await provider.listDatatableMigrations({ workspace }); + return (migrations as { datatable: string; timestamp: number }[]).some( + (m) => m.datatable === datatable && m.timestamp === timestamp + ); } else if (isTriggerKind(kind)) { return provider.existsTriggerByKind(kind, { workspace, path }); } @@ -639,6 +698,41 @@ export async function deployItem( requestBody, }); } + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = await provider.listDatatableMigrations({ + workspace: workspaceFrom, + }); + const migration = ( + migrations as { + datatable: string; + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }[] + ).find((m) => m.datatable === datatable && m.timestamp === timestamp); + if (!migration) { + throw new Error( + `Datatable migration ${path} not found in ${workspaceFrom}` + ); + } + try { + await provider.upsertDatatableMigration({ + workspace: workspaceTo, + datatableName: datatable, + requestBody: { + timestamp: migration.timestamp, + name: migration.name, + code_up: migration.code_up, + ...(migration.code_down != null + ? { code_down: migration.code_down } + : {}), + }, + }); + } catch (e) { + throw asMigrationsDisabledError(e, datatable); + } } else { throw new Error(`Unknown kind: ${kind}`); } @@ -684,6 +778,13 @@ export async function deleteItemInWorkspace( await provider.deleteFolder({ workspace, name: folderName(path) }); } else if (kind === "schedule") { await provider.deleteSchedule({ workspace, path }); + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + await provider.deleteDatatableMigration({ + workspace, + datatableName: datatable, + timestamp, + }); } else if (isTriggerKind(kind)) { await provider.deleteTriggerByKind(kind, { workspace, path }); } else { @@ -767,6 +868,29 @@ export async function getItemValue( } else if (isTriggerKind(kind)) { const trigger = await provider.getTriggerValue(kind, { workspace, path }); return stripTriggerOrScheduleRuntimeFields(trigger); + } else if (kind === "datatable_migration") { + // Surface the migration SQL so the diff drawer shows the up/down bodies a + // reviewer needs to inspect before deploying. + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = (await provider.listDatatableMigrations({ + workspace, + })) as { + datatable: string; + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }[]; + const migration = migrations.find( + (m) => m.datatable === datatable && m.timestamp === timestamp + ); + if (migration) { + return { + name: migration.name, + code_up: migration.code_up, + code_down: migration.code_down ?? null, + }; + } } } catch { // Item may not exist diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3023846ce6..0df7eef6f6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -94,7 +94,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.7.0", + "windmill-utils-internal": "1.8.2", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -291,7 +291,6 @@ "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", @@ -307,7 +306,6 @@ "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -865,7 +863,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -875,21 +872,21 @@ } }, "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", + "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.1", + "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", "dev": true, "license": "MIT", "optional": true, @@ -898,9 +895,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -1392,6 +1389,7 @@ "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", @@ -1548,6 +1546,7 @@ "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -1774,8 +1773,8 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", + "@emnapi/core": "1.11.2", + "@emnapi/runtime": "1.11.2", "@napi-rs/wasm-runtime": "^1.1.4" }, "engines": { @@ -1912,6 +1911,7 @@ "integrity": "sha512-iAIPEahFgDJJyvz8g0jP08KvqnM6JvdW8YfsygZ+pMeMvyM2zssWMltcsotETvjSZ82G3VlitgDtBIvpQSZrTA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -2007,6 +2007,7 @@ "integrity": "sha512-ILXmxC7HAsnkK2eslgPetrqqW1BKSL7LktsFgqzNj83MaivMGZzluWq32m25j2mDOjmSKX7GGWahePhuEs7P/g==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "deepmerge": "^4.3.1", "magic-string": "^0.30.21", @@ -2468,8 +2469,7 @@ "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/ms": { "version": "2.1.0", @@ -2482,8 +2482,7 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/semver": { "version": "7.7.1", @@ -2552,6 +2551,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -3079,6 +3079,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3132,6 +3133,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -3277,7 +3279,6 @@ "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -3305,7 +3306,6 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -3386,8 +3386,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz", "integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/base64-js": { "version": "1.5.1", @@ -3514,6 +3513,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -3711,7 +3711,6 @@ "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "camelcase": "^6.3.0", "map-obj": "^4.1.0", @@ -3731,7 +3730,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3745,7 +3743,6 @@ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3759,7 +3756,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -3868,6 +3864,7 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", + "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -4144,7 +4141,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -4209,7 +4205,6 @@ "integrity": "sha512-8HFEBPKhOpJPEPu70wJJetjKta86Gw9+CCyCnB3sui2qQfOvRyqBy4IKLKKAwdMpWb2lHXWk9Wb4Z6AmaUT1Pg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" } @@ -4397,6 +4392,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -4819,6 +4815,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -4918,6 +4915,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.21.0" }, @@ -4958,7 +4956,6 @@ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4972,7 +4969,6 @@ "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "decamelize": "^1.1.0", "map-obj": "^1.0.0" @@ -4990,7 +4986,6 @@ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5001,7 +4996,6 @@ "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5495,7 +5489,6 @@ "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -5593,6 +5586,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6149,7 +6143,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.9.1" } @@ -6509,7 +6502,6 @@ "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -6523,7 +6515,6 @@ "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -6539,7 +6530,6 @@ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -6589,8 +6579,7 @@ "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/gopd": { "version": "1.2.0", @@ -6673,7 +6662,6 @@ "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -6910,7 +6898,6 @@ "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -6924,7 +6911,6 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -6937,8 +6923,7 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/html-tags": { "version": "3.3.1", @@ -6946,7 +6931,6 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -7042,7 +7026,6 @@ "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -7073,7 +7056,6 @@ "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7143,8 +7125,7 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -7249,7 +7230,6 @@ "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7260,7 +7240,6 @@ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7365,8 +7344,7 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -7402,8 +7380,7 @@ "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-refs": { "version": "3.0.15", @@ -7600,7 +7577,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8206,8 +8182,7 @@ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.uniq": { "version": "4.5.0", @@ -8275,7 +8250,6 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -8326,7 +8300,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -8567,7 +8540,6 @@ "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/minimist": "^1.2.2", "camelcase-keys": "^7.0.0", @@ -8595,7 +8567,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -9330,7 +9301,6 @@ "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0", @@ -9396,6 +9366,7 @@ "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-25.0.0.tgz", "integrity": "sha512-uiY06RTWFo2WZdh6OybkLlDhuG+8LlkjUDpr9/wW55uucqHo4X8fx4XKEtD98cscC+6FKQkbG2yyUiOJ/npHOw==", "license": "MIT", + "peer": true, "dependencies": { "@codingame/monaco-vscode-api": "25.0.0" } @@ -9632,7 +9603,6 @@ "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -9984,7 +9954,6 @@ "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -10306,6 +10275,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -10494,6 +10464,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" @@ -10883,8 +10854,7 @@ "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -11060,6 +11030,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11363,7 +11334,6 @@ "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/normalize-package-data": "^2.4.0", "normalize-package-data": "^3.0.2", @@ -11383,7 +11353,6 @@ "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-up": "^5.0.0", "read-pkg": "^6.0.0", @@ -11402,7 +11371,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11416,7 +11384,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11459,7 +11426,6 @@ "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "indent-string": "^5.0.0", "strip-indent": "^4.0.0" @@ -12100,7 +12066,6 @@ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -12177,7 +12142,6 @@ "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" @@ -12188,8 +12152,7 @@ "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", "dev": true, - "license": "CC-BY-3.0", - "peer": true + "license": "CC-BY-3.0" }, "node_modules/spdx-expression-parse": { "version": "3.0.1", @@ -12197,7 +12160,6 @@ "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.0" @@ -12208,8 +12170,7 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", "dev": true, - "license": "CC0-1.0", - "peer": true + "license": "CC0-1.0" }, "node_modules/sprintf-js": { "version": "1.0.3", @@ -12303,7 +12264,6 @@ "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12329,8 +12289,7 @@ "resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz", "integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/style-to-object": { "version": "0.4.4", @@ -12379,7 +12338,6 @@ "integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^2.3.1", "@csstools/css-tokenizer": "^2.2.0", @@ -12462,7 +12420,6 @@ } ], "license": "MIT-0", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -12476,7 +12433,6 @@ "integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.2.0" }, @@ -12489,8 +12445,7 @@ "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz", "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/stylelint/node_modules/postcss-selector-parser": { "version": "6.1.2", @@ -12513,7 +12468,6 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -12654,7 +12608,6 @@ "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -12684,6 +12637,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.5.tgz", "integrity": "sha512-YkqERnF05g8KLdDZwZrF8/i1eSbj6Eoat8Jjr2IfruZz9StLuBqo8sfCSzjosNKd+ZrQ8DkKZDjpO5y3ht1Pow==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -12781,21 +12735,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13025,8 +12964,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", - "dev": true, - "peer": true + "dev": true }, "node_modules/svgo": { "version": "3.3.2", @@ -13077,7 +13015,6 @@ "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -13105,6 +13042,7 @@ "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -13357,6 +13295,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13419,7 +13358,6 @@ "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13537,6 +13475,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13773,7 +13712,6 @@ "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-correct": "^3.0.0", "spdx-expression-parse": "^3.0.0" @@ -13827,6 +13765,7 @@ "integrity": "sha512-MFtjBYgzmSxmgA4RAfjIyXWpGe1oALnjgUTzzV7QLx/TKxCzjtMH6Fd9/eVK+5Fg1qNoz5VAwsmMs/NofrmJvw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", @@ -14396,9 +14335,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.7.0.tgz", - "integrity": "sha512-K5kAiJKhavfGatmicbJyqT+KFspzFZK5Ou14pHj4Xg6Q2Z1a4ZgziBdrRNsaeAuBhB62yXWZA0OXDzg936Ymmw==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.8.2.tgz", + "integrity": "sha512-Otdn4iCE0QiOtMKPdHX5I89oECfZa3vmO2jdLdSETf7MBSv/gw9gvQm3oUQ29ymIHXLeuhQk/ebPD+WYolQJEA==", "license": "Apache 2.0" }, "node_modules/word-wrap": { @@ -14534,7 +14473,6 @@ "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" @@ -14730,7 +14668,6 @@ "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -14750,6 +14687,7 @@ "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", "license": "MIT", + "peer": true, "dependencies": { "lib0": "^0.2.99" }, @@ -14785,6 +14723,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index 70d24aba68..fe0366eb62 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -169,7 +169,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.7.0", + "windmill-utils-internal": "1.8.2", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 8cfa311141..a2453add27 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -646,7 +646,7 @@ deploymentStatus[statusKey] = { status: 'deployed' } } else { deploymentStatus[statusKey] = { status: 'failed', error: result.error } - sendUserToast(`Failed to deploy ${statusKey}: ${result.error}`) + sendUserToast(`Failed to deploy ${statusKey}: ${result.error}`, 'error') } } @@ -696,7 +696,10 @@ if (!aIsFolder && bIsFolder) return 1 return 0 }) + const to = mergeIntoParent ? parent : current let anyFailed = false + // Datatables whose migrations deployed cleanly — candidates for a run prompt. + const deployedMigrationDatatables = new Set() for (const itemKey of sortedItems) { const deployable = deployableItems.find((d) => d.key === itemKey) @@ -705,16 +708,19 @@ continue } - const to = mergeIntoParent ? parent : current const from = mergeIntoParent ? current : parent await deploy(deployable.kind, deployable.path, to, from, itemKey) if (deploymentStatus[itemKey]?.status === 'failed') { anyFailed = true + } else if (deployable.kind === 'datatable_migration') { + deployedMigrationDatatables.add(deployable.path.split('/')[0]) } } deploying = false deselectAll() + await maybePromptRunMigrations(deployedMigrationDatatables, to) + // If every selected item deployed cleanly and the direction was // merge-into-parent, resolve any open deployment request for this fork. if (!anyFailed && mergeIntoParent) { @@ -750,6 +756,51 @@ onChanged?.() } + /** + * After a deploy, offer to run the migrations of every cloned datatable that + * received one. `forked_from` is set on the fork's datatable config only when + * the datatable was cloned into a separate database — shared-DB datatables + * have already had the schema change applied and must not be re-run. + */ + async function maybePromptRunMigrations( + deployedMigrationDatatables: Set, + runTargetWorkspace: string + ) { + if (deployedMigrationDatatables.size === 0) return + try { + const forkSettings = await WorkspaceService.getPublicSettings({ + workspace: currentWorkspaceId + }) + const datatables = forkSettings.datatable?.datatables ?? {} + const cloned = [...deployedMigrationDatatables].filter( + (dt) => datatables[dt]?.forked_from != null + ) + if (cloned.length === 0) return + runMigrationsDatatables = cloned.sort() + runMigrationsTargetWorkspace = runTargetWorkspace + runMigrationsModalOpen = true + } catch (e) { + console.error('Failed to determine cloned datatables for migration run prompt', e) + } + } + + async function runDeployedMigrations() { + runMigrationsModalOpen = false + for (const dt of runMigrationsDatatables) { + try { + const res = await WorkspaceService.runDatatableMigrations({ + workspace: runMigrationsTargetWorkspace, + datatableName: dt + }) + sendUserToast( + `Ran ${res.applied.length} migration${res.applied.length !== 1 ? 's' : ''} on ${dt}` + ) + } catch (e: any) { + sendUserToast(`Failed to run migrations on ${dt}: ${e.body ?? e.message ?? e}`, true) + } + } + } + function toggleKey(key: string) { if (selectedItems.includes(key)) { selectedItems = selectedItems.filter((i) => i !== key) @@ -930,6 +981,17 @@ let deploymentRequestPanel: DeploymentRequestPanel | undefined = $state(undefined) let hasOpenDeploymentRequest = $state(false) + // After deploying datatable migrations to a cloned (separate-DB) datatable, we + // offer to run them in the target workspace. Shared-DB datatables are skipped: + // the schema change is already physically applied, so re-running is redundant. + let runMigrationsModalOpen = $state(false) + let runMigrationsDatatables = $state([]) + let runMigrationsTargetWorkspace = $state('') + let runMigrationsTargetWorkspaceName = $derived( + $userWorkspaces.find((w) => w.id == runMigrationsTargetWorkspace)?.name ?? + runMigrationsTargetWorkspace + ) + /** Display labels for trigger/schedule kinds in the merge UI. */ const KIND_DISPLAY_NAMES: Record = { schedule: 'Schedule', @@ -942,7 +1004,8 @@ sqs_trigger: 'SQS trigger', gcp_trigger: 'GCP trigger', azure_trigger: 'Azure trigger', - email_trigger: 'Email trigger' + email_trigger: 'Email trigger', + datatable_migration: 'Data table migration' } // Human label for a diff kind, lowercased for inline use in the hidden-items @@ -1453,9 +1516,7 @@ /> -
- -
+ {#if pinnedItems.length > 0}
@@ -1536,6 +1597,26 @@
+ (runMigrationsModalOpen = false)} + > +
+

+ Run the deployed migrations in {runMigrationsTargetWorkspaceName} now? These data tables + use a separate database, so the schema changes won't apply until the migrations are run. +

+
    + {#each runMigrationsDatatables as dt (dt)} +
  • {dt}
    • + {/each} +
+
+ + { if (dbTableEditorState.alterTableKey && dbTableEditorAlterTableData.current) { let diff = diffTableEditorValues(dbTableEditorAlterTableData.current, values) - await dbSchemaOps.onAlter({ schema: selected.schemaKey, values: diff }) + // Reverse diff (new → old) so the migration's down undoes the alter. + let reverse = diffTableEditorValues(values, dbTableEditorAlterTableData.current) + await dbSchemaOps.onAlter({ schema: selected.schemaKey, values: diff, reverse }) } else { await dbSchemaOps.onCreate({ values, schema: selected.schemaKey }) } diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index d84bdb862e..c40a2f7428 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -20,6 +20,10 @@ import type { SelectedTable } from './DBManager.svelte' import { getDbFeatures } from './apps/components/display/dbtable/dbFeatures' import { resource } from 'runed' + import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' + import { createAsyncConfirmationModal } from './common/confirmationModal/asyncConfirmationModal.svelte' + import Portal from '$lib/components/Portal.svelte' + import { outOfOrderRunMessage } from './workspaceSettings/datatableMigrationUtils' interface Props { input?: DbInput @@ -52,6 +56,8 @@ let dbSchema: DBSchema | undefined = $derived(input && $dbSchemas[getDbSchemasPath(input)]) + const outOfOrderModal = createAsyncConfirmationModal() + function getDbSchemasPath(input: DbInput): string { switch (input.type) { case 'database': @@ -163,7 +169,13 @@ })} dbSchemaOps={dbSchemaOpsWithPreviewScripts({ input: _input, - workspace: $workspaceStore + workspace: $workspaceStore, + confirmRunOutOfOrder: (pending) => + outOfOrderModal.ask({ + title: 'Run migration out of order', + confirmationText: 'Run anyway', + children: outOfOrderRunMessage(pending) + }) })} initialTableKey={input.specificTable} initialSchemaKey={input.specificSchema} @@ -192,6 +204,7 @@ onData={(data) => { replResultData = data }} + onSchemaChange={() => refresh()} placeholderTableName={sortArray( Object.keys( dbSchema?.schema[ @@ -214,3 +227,9 @@ {/if} + + + + + diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 6598103751..b303671fe8 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -16,6 +16,7 @@ Upload } from 'lucide-svelte' import DBManagerContent from './DBManagerContent.svelte' + import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' import { resource } from 'runed' import { untrack } from 'svelte' import type { DbManagerUriState } from './dbManagerDrawerModel.svelte' @@ -105,6 +106,11 @@ return toSourceIdentifier(input.resourcePath) } + function refreshManager() { + dbManagerContent?.refresh() + dbManagerContent?.dbManager()?.dbTable()?.refresh() + } + async function handleExportSchema() { const source = currentSourceIdentifier() if (!source || !$workspaceStore) return @@ -201,6 +207,13 @@ {/key} {/if} {#snippet actions()} + {#if uriState.isDatatableInput && uriState.selectedDatatable && $workspaceStore} + + {/if} {#if enableImportExport} + /> +{#if applicableCount > 0} +
+

Datatable schema changes

+ {#if loading} +
+ Loading datatable diffs... +
+ {:else if error} +
Failed to load datatable diffs: {error}
+ {:else if diffs.length > 0} +
+ {#each diffs as diff} + + - {#if expandedDatatables.has(diff.datatableName)} -
- {#if diff.aheadChanges.length > 0} -
-
Fork changes (ahead)
- {#each diff.aheadChanges as change} -
- {#if change.kind === 'added'} - - {:else if change.kind === 'removed'} - - {:else} - - {/if} - {change.schemaName}. - {change.tableName} - {operationSummary(change)} - + {#each diff.aheadChanges as change} +
+ {#if change.kind === 'added'} + + {:else if change.kind === 'removed'} + + {:else} + + {/if} + {change.schemaName}. + {change.tableName} + {operationSummary(change)} + +
+ {/each}
- {/each} + {/if} + {#if diff.behindChanges.length > 0} +
+
+ Parent changes (behind) +
+ {#each diff.behindChanges as change} +
+ {#if change.kind === 'added'} + + {:else if change.kind === 'removed'} + + {:else} + + {/if} + {change.schemaName}. + {change.tableName} + {operationSummary(change)} + +
+ {/each} +
+ {/if}
{/if} - {#if diff.behindChanges.length > 0} -
-
- Parent changes (behind) -
- {#each diff.behindChanges as change} -
- {#if change.kind === 'added'} - - {:else if change.kind === 'removed'} - - {:else} - - {/if} - {change.schemaName}. - {change.tableName} - {operationSummary(change)} - -
- {/each} -
- {/if} -
- {/if} - - {/each} + + {/each} +
+ {:else} + No changes detected + {/if}
-{:else} - No changes detected {/if} @@ -580,3 +655,7 @@ >{migrationSql} + + + + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte new file mode 100644 index 0000000000..b6833521f8 --- /dev/null +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -0,0 +1,159 @@ + + + + + +
+

+ This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? +

+ 
{promptStatement ?? ''}
+
+ + +
+
+ + + migrationsModal?.openMigration(m.timestamp)} +/> + + diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 78a75734c9..8563190a89 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -43,6 +43,7 @@ import { editorFontSize } from '$lib/editorFontSize.svelte' import { createHash as randomHash } from '$lib/editorLangUtils' import { workspaceStore } from '$lib/stores' + import DdlMigrationGuard from './DdlMigrationGuard.svelte' import { type Preview, ResourceService, @@ -222,6 +223,35 @@ let lang = $state(scriptLangToEditorLang(untrack(() => scriptLang))) + // On a postgres script targeting a datatable, DDL statements are intercepted + // on run (cmd+enter) and offered as migrations instead. + let datatableForMigrations = $derived( + scriptLang === 'postgresql' && + typeof args?.database === 'string' && + args.database.startsWith('datatable://') + ? args.database.slice('datatable://'.length).split('/')[0] + : undefined + ) + let ddlGuard = $state(undefined) + + // Run the DDL migration guard against the current code. Returns false when the + // user cancels (the run must be aborted); may rewrite the code (migrated + // statements stripped). Exported so run paths that bypass the Monaco + // Cmd+Enter binding (e.g. the Test button) can guard too. + export async function guardDdlBeforeRun(): Promise { + if (datatableForMigrations && ddlGuard) { + const res = await ddlGuard.guard(getCode()) + if (!res.proceed) return false + if (res.code !== getCode()) setCode(res.code) + } + return true + } + + async function runCmdEnterWithDdlGuard() { + if (!(await guardDdlBeforeRun())) return + cmdEnterAction?.() + } + let filePath = $state(computePath(untrack(() => path))) let initialPath: string | undefined = $state(untrack(() => path)) @@ -1639,7 +1669,8 @@ editor?.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () { updateCode() - shouldBindKey && cmdEnterAction && cmdEnterAction() + if (!shouldBindKey || !cmdEnterAction) return + void runCmdEnterWithDdlGuard() }) editor?.addCommand(KeyMod.CtrlCmd | KeyMod.Shift | KeyCode.Digit7, function () { @@ -2209,6 +2240,13 @@ +{#if datatableForMigrations && $workspaceStore} + +{/if} {#if !editor}
diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index ac941d0bf0..bf5349b126 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -793,7 +793,17 @@ args = nargs } - export async function runTest(opts?: { cascade?: boolean }) { + export async function runTest(opts?: { cascade?: boolean; skipDdlGuard?: boolean }) { + // Intercept DDL statements (offer to turn them into data table migrations) + // on every run path, not just the editor's Cmd+Enter. `skipDdlGuard` is set + // by the Cmd+Enter action, which already guarded before calling us. + if (!opts?.skipDdlGuard) { + if ((await editor?.guardDdlBeforeRun()) === false) return + // The guard may have rewritten the code (migrated statements stripped); + // `editorCode` is kept in sync by the editor binding, so mirror the + // on:change handler and pull it into `code` before we run. + if (activeModuleTab === null) code = editorCode + } // When the caller forces a cascade choice (e.g. the canvas runnable // menu's "Run + trigger N downstream"), also flip the persistent // `cascadeDownstream` state so the split button's label/icon reflect @@ -2677,7 +2687,8 @@ } else { await inferModuleSchema() } - runTest() + // The Editor already ran the DDL guard before invoking this action. + runTest({ skipDdlGuard: true }) }} formatAction={async () => { if (activeModuleTab === null) { diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 741d874f81..f1c4daa962 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -95,6 +95,7 @@ loadAsync = false, key, disabled = false, + readOnly = false, minHeight = 1000, renderLineHighlight = 'none', suggestion @@ -123,6 +124,9 @@ initialCursorPos?: IPosition key?: string disabled?: boolean + /** Read-only Monaco mode: not editable, but still scrollable/selectable + * (unlike `disabled`, which makes the editor non-interactive). */ + readOnly?: boolean minHeight?: number renderLineHighlight?: 'all' | 'line' | 'gutter' | 'none' suggestion?: string @@ -239,6 +243,9 @@ lineNumbers: $relativeLineNumbers ? 'relative' : 'on' }) }) + $effect(() => { + editor?.updateOptions({ readOnly }) + }) function onVimDisable() { vimDisposable?.dispose() @@ -342,6 +349,7 @@ ), model, ...(yPadding !== undefined ? { padding: { bottom: yPadding, top: yPadding } } : {}), + readOnly, renderLineHighlight, lineDecorationsWidth: 0, lineNumbersMinChars: 2, diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index 661b0ccd2c..6c8d90ba15 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -1,49 +1,3 @@ - - + +{#if !hideTrigger} + +{/if} + + + {#snippet headerLeft()} + + Each schema edit made in the database manager is captured here as a migration. Tracking schema + changes as migrations makes it easy to export data tables to other workspaces, and to + reproduce their schema when forking a workspace. + + {/snippet} + {#snippet headerRight()} + {#if enabled && canManage} + disableMigrations() + } + ]} + > + {#snippet buttonReplacement()} + + {/if} +
+ {:else} + {#if loadError} +
+ Could not read applied status from the data table: {loadError} +
+ {/if} +
+ {#if migrations.length === 0} +
+ No migrations yet + +
+ {:else} + {#each migrations as m (m.timestamp)} +
+
+ +
+ {/each} + {/if} +
+
+ + +
+ {/if} + + + + (newMigrationOpen = false)} + onSeeMigration={(m) => openMigration(m.timestamp)} +/> + + + {#if viewMigration} +
+ {viewMigration.timestamp} + + + + {#snippet content()} + + + + + {#if viewMigration?.code_down} + + {:else} +
No down migration
+ {/if} + + {/snippet} + +
+ {/if} + + + +
+

+ "{deleteTarget?.name}" is installed on the data table. Revert it first to undo its schema + change, or delete only the definition and leave the schema as-is — deleting without reverting + means it can no longer be reverted. +

+
+ + + +
+
+ + + + + diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index af6fe29e58..f652ed8704 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -1,6 +1,12 @@ + + + +
+ + + + + {#snippet content()} + + + + +
+ + {#if enableDown} +
+ +
+ {/if} +
+ + {/snippet} + +
+ +
+
+ + + + + diff --git a/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts new file mode 100644 index 0000000000..ed046418f2 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts @@ -0,0 +1,30 @@ +import { WorkspaceService, type DatatableMigrationWithStatus } from '$lib/gen' + +/** + * Migrations that are defined but not yet applied. A newly-created migration + * always gets the highest timestamp, so every pending migration is "earlier": + * running the new one on its own would apply it ahead of them (out of order). + */ +export function pendingMigrations( + migrations: DatatableMigrationWithStatus[] +): DatatableMigrationWithStatus[] { + return migrations.filter((m) => m.status !== 'ran') +} + +/** Fetch the data table's migration status and return the pending ones. */ +export async function fetchPendingMigrations( + workspace: string, + datatableName: string +): Promise { + const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName + }) + return pendingMigrations(migrations) +} + +/** Confirmation copy shown before running a just-created migration ahead of + * `count` still-pending earlier ones (mirrors the row-level Run warning). */ +export function outOfOrderRunMessage(count: number): string { + return `${count} earlier migration(s) have not been run yet. This migration might depend on them. Run it anyway?` +} diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 5f49eb5c49..556ac0a3c7 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -40,6 +40,8 @@ export type Kind = | 'gcp_trigger' | 'azure_trigger' | 'email_trigger' + // Data table migration, diffed per `/_` path. + | 'datatable_migration' // Legacy generic kind used by the cross-workspace `DeployWorkspace` UI, // which carries the trigger sub-kind in `additionalInformation`. | 'trigger' diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 112ed69e17..72400bb962 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -16,7 +16,8 @@ import { SqsTriggerService, UserService, VariableService, - WebsocketTriggerService + WebsocketTriggerService, + WorkspaceService } from '$lib/gen' import { fetchProtectionRulesForWorkspace, @@ -234,7 +235,11 @@ function makeProvider(): DeployProvider { getSchedule: (p) => ScheduleService.getSchedule(p), createSchedule: (p) => ScheduleService.createSchedule(p), updateSchedule: (p) => ScheduleService.updateSchedule(p), - deleteSchedule: (p) => ScheduleService.deleteSchedule(p) + deleteSchedule: (p) => ScheduleService.deleteSchedule(p), + // Datatable migrations + listDatatableMigrations: (p) => WorkspaceService.listDatatableMigrations(p), + upsertDatatableMigration: (p) => WorkspaceService.upsertDatatableMigration(p), + deleteDatatableMigration: (p) => WorkspaceService.deleteDatatableMigration(p) } } diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 01579a737a..a74fdc6a17 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -77,6 +77,13 @@ datatable related commands - `datatable run ` - run a SQL query on a datatable - `-n --name ` - Datatable name (default: main) - `-s --silent` - Output only the final result as JSON. Useful for scripting. +- `datatable migrate` - manage datatable migrations + - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) - `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -345,19 +352,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** `[workspace:string]` - -**Options:** -- `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) -- `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) -- `--skip-worker-check` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- `jobs pull` -- `jobs push` +- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace + - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) + - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before export +- `jobs push [workspace:string]` - Push completed and queued jobs to workspace + - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json) + - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before import ### lint diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 13776fe1e6..58ac1ac69d 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2755,6 +2755,13 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable migrate\` - manage datatable migrations + - \`datatable migrate new \` - scaffold a new migration (.up.sql / .down.sql files) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate up\` - apply all pending migrations to the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate down\` - roll back the most recent migration on the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) - \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -3023,19 +3030,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- \`jobs pull\` -- \`jobs push\` +- \`jobs pull [workspace:string]\` - Pull completed and queued jobs from workspace + - \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) + - \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before export +- \`jobs push [workspace:string]\` - Push completed and queued jobs to workspace + - \`-c, --completed-file \` - Completed jobs input file (default: completed_jobs.json) + - \`-q, --queued-file \` - Queued jobs input file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before import ### lint diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index cc314a5067..db04118c3f 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -82,6 +82,13 @@ datatable related commands - `datatable run ` - run a SQL query on a datatable - `-n --name ` - Datatable name (default: main) - `-s --silent` - Output only the final result as JSON. Useful for scripting. +- `datatable migrate` - manage datatable migrations + - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) - `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -350,19 +357,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** `[workspace:string]` - -**Options:** -- `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) -- `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) -- `--skip-worker-check` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- `jobs pull` -- `jobs push` +- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace + - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) + - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before export +- `jobs push [workspace:string]` - Push completed and queued jobs to workspace + - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json) + - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before import ### lint diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 98c03c4367..dac36ad97a 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -340,13 +340,54 @@ def extract_description(section: str) -> str | None: return ''.join(_unquote_js_string(p) for p in parts).strip() or None -def parse_command_block(content: str, file_path: Path | None = None) -> dict: +def extract_named_command_block(content: str, var_name: str) -> str | None: + """Return the chained-call body of `const = new Command() ...`, + from just after `new Command()` up to the next top-level statement. + + Returns None when the var isn't a *direct* `new Command()` (e.g. it's wrapped + in a helper call like `auditListOptions(new Command()...)`), so callers can + fall back to a looser match. + """ + m = re.search( + r'const\s+' + re.escape(var_name) + r'\s*=\s*new\s+Command\(\)' + r'([\s\S]*?)(?=\n(?:const|let|var|async|function|export)\b)', + content, + ) + return m.group(1) if m else None + + +def extract_exported_command_block(content: str) -> str | None: + """Return the chained-call body of the command that is `export default`ed. + + A command file may define helper `new Command()` groups (assigned to local + consts and mounted as nested subcommands via `.command("x", localCmd)`) + *before* the exported command. Anchoring on the first `new Command()` in the + file would merge those helpers into the top-level command, so resolve the + exported variable first and only then fall back to the first `new Command()` + (which covers inline/wrapped exports). + """ + export_match = re.search(r'export\s+default\s+(\w+)\s*;', content) + if export_match: + block = extract_named_command_block(content, export_match.group(1)) + if block is not None: + return block + command_match = re.search( + r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', + content, + ) + return command_match.group(1) if command_match else None + + +def parse_command_block( + content: str, file_path: Path | None = None, block: str | None = None +) -> dict: """ Parse a Cliffy Command() definition block and extract metadata. Returns a dict with: description, options, subcommands, arguments, alias If file_path is provided, imported subcommands will be resolved by parsing - the imported files. + the imported files. `block` may be passed to parse a specific pre-extracted + command body (used to recurse into locally-defined nested command groups). """ result = { 'description': '', @@ -357,15 +398,11 @@ def parse_command_block(content: str, file_path: Path | None = None) -> dict: } # Find the command block - command_match = re.search( - r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', - content - ) - if not command_match: + if block is None: + block = extract_exported_command_block(content) + if block is None: return result - block = command_match.group(1) - # Find where subcommands start first_subcommand_pos = block.find('.command(') if first_subcommand_pos == -1: @@ -451,12 +488,31 @@ def parse_command_block(content: str, file_path: Path | None = None) -> dict: 'name': cmd_name, 'description': imported_cmd.get('description', ''), 'arguments': imported_cmd.get('arguments', ''), - 'options': imported_cmd.get('options', []) + 'options': imported_cmd.get('options', []), + 'subcommands': imported_cmd.get('subcommands', []), }) continue except Exception as e: print(f" Warning: Could not parse imported command {second_arg}: {e}") cmd_desc = '' + elif second_arg and re.search( + r'const\s+' + re.escape(second_arg) + r'\s*=\s*new\s+Command\(\)', content + ): + # Locally-defined command group mounted as a subcommand + # (e.g. `.command("migrate", migrateCommand)`): recurse into its + # definition so its own subcommands/options are captured. + nested_block = extract_named_command_block(content, second_arg) + if nested_block is not None: + nested = parse_command_block(content, file_path, block=nested_block) + result['subcommands'].append({ + 'name': cmd_name, + 'description': nested.get('description', ''), + 'arguments': nested.get('arguments', ''), + 'options': nested.get('options', []), + 'subcommands': nested.get('subcommands', []), + }) + continue + cmd_desc = '' else: cmd_desc = '' @@ -628,6 +684,16 @@ def generate_cli_commands_markdown(cli_data: dict) -> str: for opt in sub['options']: md += f" - `{opt['flag']}` - {opt['description']}\n" + # Nested sub-subcommands (e.g. `datatable migrate new`) + for subsub in sub.get('subcommands', []): + ss_args = f" {subsub['arguments']}" if subsub.get('arguments') else "" + md += f" - `{cmd['name']} {sub_name} {subsub['name']}{ss_args}`" + if subsub.get('description'): + md += f" - {subsub['description']}" + md += "\n" + for opt in subsub.get('options', []): + md += f" - `{opt['flag']}` - {opt['description']}\n" + md += "\n" return md