diff --git a/backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json b/backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json index 39374fabd8..31ec682852 100644 --- a/backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json +++ b/backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json @@ -37,6 +37,11 @@ "ordinal": 6, "name": "format_extension", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_fileset", + "type_info": "Bool" } ], "parameters": { @@ -52,7 +57,8 @@ true, true, true, - true + true, + false ] }, "hash": "03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3" diff --git a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json b/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json new file mode 100644 index 0000000000..fc354fb9a2 --- /dev/null +++ b/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset\n FROM resource_type\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986" +} diff --git a/backend/.sqlx/query-7bc9fc05dbd162866bef1fdd3e7faeb50429881ed1bc962903f06e4b3d5f8d44.json b/backend/.sqlx/query-2768622b76ad92c05f4f44d997aff285707e1a43ce85e5bb8e87849d78a0637f.json similarity index 59% rename from backend/.sqlx/query-7bc9fc05dbd162866bef1fdd3e7faeb50429881ed1bc962903f06e4b3d5f8d44.json rename to backend/.sqlx/query-2768622b76ad92c05f4f44d997aff285707e1a43ce85e5bb8e87849d78a0637f.json index 6cd6f3432c..e69c06f887 100644 --- a/backend/.sqlx/query-7bc9fc05dbd162866bef1fdd3e7faeb50429881ed1bc962903f06e4b3d5f8d44.json +++ b/backend/.sqlx/query-2768622b76ad92c05f4f44d997aff285707e1a43ce85e5bb8e87849d78a0637f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT schema, description, format_extension\n FROM resource_type\n WHERE workspace_id = $1 AND name = $2", + "query": "SELECT schema, description, format_extension, is_fileset\n FROM resource_type\n WHERE workspace_id = $1 AND name = $2", "describe": { "columns": [ { @@ -17,6 +17,11 @@ "ordinal": 2, "name": "format_extension", "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_fileset", + "type_info": "Bool" } ], "parameters": { @@ -28,8 +33,9 @@ "nullable": [ true, true, - true + true, + false ] }, - "hash": "7bc9fc05dbd162866bef1fdd3e7faeb50429881ed1bc962903f06e4b3d5f8d44" + "hash": "2768622b76ad92c05f4f44d997aff285707e1a43ce85e5bb8e87849d78a0637f" } diff --git a/backend/.sqlx/query-ffedbb3a2676a6d7b71f81f89109a02a8dba90d40144e942527f8a3fc36dfbc1.json b/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json similarity index 54% rename from backend/.sqlx/query-ffedbb3a2676a6d7b71f81f89109a02a8dba90d40144e942527f8a3fc36dfbc1.json rename to backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json index c58dfad63a..400c8d9ee6 100644 --- a/backend/.sqlx/query-ffedbb3a2676a6d7b71f81f89109a02a8dba90d40144e942527f8a3fc36dfbc1.json +++ b/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, now())", + "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, now())", "describe": { "columns": [], "parameters": { @@ -10,10 +10,11 @@ "Jsonb", "Text", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "ffedbb3a2676a6d7b71f81f89109a02a8dba90d40144e942527f8a3fc36dfbc1" + "hash": "5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00" } diff --git a/backend/.sqlx/query-61b37cb4db6e60c2d35f7d23db5afbe04e040a8dcd1d93afaaaa320665c8779a.json b/backend/.sqlx/query-61b37cb4db6e60c2d35f7d23db5afbe04e040a8dcd1d93afaaaa320665c8779a.json deleted file mode 100644 index 64fb5a6673..0000000000 --- a/backend/.sqlx/query-61b37cb4db6e60c2d35f7d23db5afbe04e040a8dcd1d93afaaaa320665c8779a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH prev_sd AS (\n DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock\n ) INSERT INTO debounce_stale_data (job_id, to_relock)\n VALUES ($2, array_cat((SELECT to_relock FROM prev_sd), $3))\n ON CONFLICT (job_id) DO UPDATE SET to_relock = EXCLUDED.to_relock\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "61b37cb4db6e60c2d35f7d23db5afbe04e040a8dcd1d93afaaaa320665c8779a" -} diff --git a/backend/.sqlx/query-7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e.json b/backend/.sqlx/query-7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e.json deleted file mode 100644 index 7d45d000ab..0000000000 --- a/backend/.sqlx/query-7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension\n FROM resource_type\n WHERE workspace_id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "7abd579d3ec97853ac36cc8dad29013eb133a28cd848bf8fdf9571b2ee402a3e" -} diff --git a/backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json b/backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json index a7a5b812f7..29f4baacc2 100644 --- a/backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json +++ b/backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json @@ -37,6 +37,11 @@ "ordinal": 6, "name": "format_extension", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_fileset", + "type_info": "Bool" } ], "parameters": { @@ -51,7 +56,8 @@ true, true, true, - true + true, + false ] }, "hash": "7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234" diff --git a/backend/.sqlx/query-842775bcf91d747abb11ffe9c98fa1208595e012590606ef6667ea3a78105883.json b/backend/.sqlx/query-842775bcf91d747abb11ffe9c98fa1208595e012590606ef6667ea3a78105883.json new file mode 100644 index 0000000000..7e57557c96 --- /dev/null +++ b/backend/.sqlx/query-842775bcf91d747abb11ffe9c98fa1208595e012590606ef6667ea3a78105883.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT name, format_extension, is_fileset FROM resource_type WHERE (format_extension IS NOT NULL OR is_fileset = true) AND (workspace_id = $1 OR workspace_id = 'admins')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "format_extension", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_fileset", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "842775bcf91d747abb11ffe9c98fa1208595e012590606ef6667ea3a78105883" +} diff --git a/backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json b/backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json index c403f6a428..4871744e03 100644 --- a/backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json +++ b/backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json @@ -37,6 +37,11 @@ "ordinal": 6, "name": "format_extension", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_fileset", + "type_info": "Bool" } ], "parameters": { @@ -51,7 +56,8 @@ true, true, true, - true + true, + false ] }, "hash": "b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33" diff --git a/backend/.sqlx/query-cf1cef7e0fe2e7e3db96b0ec005360361b9eec023a6fc2a4a7a917f59d86af4d.json b/backend/.sqlx/query-cf1cef7e0fe2e7e3db96b0ec005360361b9eec023a6fc2a4a7a917f59d86af4d.json deleted file mode 100644 index 2556ac5580..0000000000 --- a/backend/.sqlx/query-cf1cef7e0fe2e7e3db96b0ec005360361b9eec023a6fc2a4a7a917f59d86af4d.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "format_extension", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "cf1cef7e0fe2e7e3db96b0ec005360361b9eec023a6fc2a4a7a917f59d86af4d" -} diff --git a/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json b/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json index 73c1fd7b2c..ce532af385 100644 --- a/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json +++ b/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json @@ -37,6 +37,11 @@ "ordinal": 6, "name": "format_extension", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_fileset", + "type_info": "Bool" } ], "parameters": { @@ -49,7 +54,8 @@ true, true, true, - true + true, + false ] }, "hash": "eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12" diff --git a/backend/migrations/20260223000000_resource_type_is_fileset.down.sql b/backend/migrations/20260223000000_resource_type_is_fileset.down.sql new file mode 100644 index 0000000000..7d78ea6ac5 --- /dev/null +++ b/backend/migrations/20260223000000_resource_type_is_fileset.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource_type DROP COLUMN is_fileset; diff --git a/backend/migrations/20260223000000_resource_type_is_fileset.up.sql b/backend/migrations/20260223000000_resource_type_is_fileset.up.sql new file mode 100644 index 0000000000..5c9c309001 --- /dev/null +++ b/backend/migrations/20260223000000_resource_type_is_fileset.up.sql @@ -0,0 +1 @@ +ALTER TABLE resource_type ADD COLUMN is_fileset BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 0e3d3257df..01db7590f5 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -137,7 +137,7 @@ raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at( FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char) FK: (workspace_id) -> workspace(id) -resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char) +resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool) FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) diff --git a/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql b/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql index 1d267256cd..1e04195ae0 100644 --- a/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql +++ b/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql @@ -54,6 +54,21 @@ INSERT INTO resource (workspace_id, path, value, description, resource_type, ext VALUES ('test-workspace', 'u/test-user/scalar_var_resource', '"$var:u/test-user/db_password"', 'Scalar var ref', 'string', '{}', 'test-user'); +-- === fileset resource type test data === + +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, is_fileset) +VALUES ('test-workspace', 'test_fileset', '{}', + 'Test fileset type', 'test-user', true); + +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, format_extension) +VALUES ('test-workspace', 'test_file', '{"type": "object", "properties": {"content": {"type": "string"}}}', + 'Test file type', 'test-user', 'txt'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/fileset_resource', + '{"config.yaml": "key: value", "data/input.json": "{\"items\": []}"}', + 'A fileset resource', 'test_fileset', '{}', 'test-user'); + -- === mcp_tools test data === INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 6294c67308..22dab5d418 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -69,8 +69,12 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(resp.status(), 404); // --- get_value_interpolated --- - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/simple_resource").await; + let resp = authed_get( + port, + "get_value_interpolated", + "u/test-user/simple_resource", + ) + .await; assert_eq!(resp.status(), 200); assert_eq!( resp.json::().await?, @@ -78,8 +82,12 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // $var: interpolation - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/resource_with_var").await; + let resp = authed_get( + port, + "get_value_interpolated", + "u/test-user/resource_with_var", + ) + .await; assert_eq!(resp.status(), 200); assert_eq!( resp.json::().await?, @@ -87,8 +95,12 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // $res: interpolation - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/resource_with_res").await; + let resp = authed_get( + port, + "get_value_interpolated", + "u/test-user/resource_with_res", + ) + .await; assert_eq!(resp.status(), 200); assert_eq!( resp.json::().await?, @@ -96,8 +108,7 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // mixed $var: and $res: refs - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/resource_mixed").await; + let resp = authed_get(port, "get_value_interpolated", "u/test-user/resource_mixed").await; assert_eq!(resp.status(), 200); assert_eq!( resp.json::().await?, @@ -105,8 +116,12 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // chained $res: -> $var: - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/chained_resource").await; + let resp = authed_get( + port, + "get_value_interpolated", + "u/test-user/chained_resource", + ) + .await; assert_eq!(resp.status(), 200); assert_eq!( resp.json::().await?, @@ -114,8 +129,7 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // null value - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/null_resource").await; + let resp = authed_get(port, "get_value_interpolated", "u/test-user/null_resource").await; assert_eq!(resp.status(), 200); assert_eq!( resp.json::().await?, @@ -123,8 +137,7 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // not found - let resp = - authed_get(port, "get_value_interpolated", "u/test-user/nonexistent").await; + let resp = authed_get(port, "get_value_interpolated", "u/test-user/nonexistent").await; assert_eq!(resp.status(), 404); // array passthrough @@ -162,7 +175,9 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { "expected at least 10 resources from fixture, got {}", list.len() ); - assert!(list.iter().any(|r| r["path"] == "u/test-user/simple_resource")); + assert!(list + .iter() + .any(|r| r["path"] == "u/test-user/simple_resource")); // list with resource_type filter let resp = authed(client().get(format!("{base}/list?resource_type=mcp_server"))) @@ -259,9 +274,11 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(body["description"], "Updated description"); // --- update_value --- - let resp = authed( - client().post(resource_url(port, "update_value", "u/test-user/new_resource")), - ) + let resp = authed(client().post(resource_url( + port, + "update_value", + "u/test-user/new_resource", + ))) .json(&json!({"value": {"url": "https://final.com"}})) .send() .await @@ -275,35 +292,44 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { ); // --- delete --- - let resp = authed( - client().delete(resource_url(port, "delete", "u/test-user/new_resource")), - ) - .send() - .await - .unwrap(); + let resp = authed(client().delete(resource_url(port, "delete", "u/test-user/new_resource"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "exists", "u/test-user/new_resource").await; assert_eq!(resp.json::().await?, false); // delete nonexistent -> 404 - let resp = authed( - client().delete(resource_url(port, "delete", "u/test-user/new_resource")), - ) - .send() - .await - .unwrap(); + let resp = authed(client().delete(resource_url(port, "delete", "u/test-user/new_resource"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 404); // --- file_resource_type_to_file_ext_map --- - let resp = authed(client().get(format!( - "{base}/file_resource_type_to_file_ext_map" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("{base}/file_resource_type_to_file_ext_map"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); - resp.json::().await?; + let ext_map = resp.json::().await?; + // Verify the map includes fileset type info with is_fileset flag (no format_extension) + let fileset_info = &ext_map["test_fileset"]; + assert_eq!(fileset_info["format_extension"], serde_json::Value::Null); + assert_eq!(fileset_info["is_fileset"], true); + // Verify non-fileset file type + let file_info = &ext_map["test_file"]; + assert_eq!(file_info["format_extension"], "txt"); + assert_eq!(file_info["is_fileset"], false); + + // --- fileset resource value --- + let resp = authed_get(port, "get_value", "u/test-user/fileset_resource").await; + assert_eq!(resp.status(), 200); + let fileset_val = resp.json::().await?; + assert_eq!(fileset_val["config.yaml"], "key: value"); + assert_eq!(fileset_val["data/input.json"], "{\"items\": []}"); // --- resource types --- @@ -384,17 +410,68 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(body["description"], "Updated type desc"); // type/delete - let resp = authed( - client().delete(resource_url(port, "type/delete", "new_test_type")), - ) - .send() - .await - .unwrap(); + let resp = authed(client().delete(resource_url(port, "type/delete", "new_test_type"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "type/exists", "new_test_type").await; assert_eq!(resp.json::().await?, false); + // --- fileset resource type CRUD --- + + // type/get for fileset type - verify is_fileset is returned + let resp = authed_get(port, "type/get", "test_fileset").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["name"], "test_fileset"); + assert_eq!(body["is_fileset"], true); + assert_eq!(body["format_extension"], serde_json::Value::Null); + + // type/get for non-fileset type - verify is_fileset is false + let resp = authed_get(port, "type/get", "test_db").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["is_fileset"], false); + + // type/create fileset type (no format_extension needed) + let resp = authed(client().post(format!("{base}/type/create"))) + .json(&json!({ + "name": "new_fileset_type", + "description": "A fileset type", + "schema": {}, + "is_fileset": true + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + let resp = authed_get(port, "type/get", "new_fileset_type").await; + let body = resp.json::().await?; + assert_eq!(body["is_fileset"], true); + assert_eq!(body["format_extension"], serde_json::Value::Null); + + // type/update - set is_fileset on existing type + let resp = authed(client().post(resource_url(port, "type/update", "new_fileset_type"))) + .json(&json!({"is_fileset": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "type/get", "new_fileset_type").await; + let body = resp.json::().await?; + assert_eq!(body["is_fileset"], false); + + // cleanup + let resp = authed(client().delete(resource_url(port, "type/delete", "new_fileset_type"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + Ok(()) } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index aa1cd45d42..9146ce2274 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -30,7 +30,6 @@ use uuid::Uuid; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; -use windmill_types::s3::LargeFileStorage; use windmill_common::users::username_to_permissioned_as; use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; @@ -55,6 +54,7 @@ use windmill_dep_map::scoped_dependency_map::{ DependencyDependent, DependencyMap, ScopedDependencyMap, }; use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject}; +use windmill_types::s3::LargeFileStorage; use hyper::StatusCode; use serde::{Deserialize, Serialize}; @@ -3010,8 +3010,8 @@ async fn clone_resource_types( target_workspace_id: &str, ) -> Result<()> { sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension) - SELECT $2, name, schema, description, edited_at, created_by, format_extension + "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset) + SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", source_workspace_id, @@ -5254,7 +5254,7 @@ async fn compare_two_resource_types( ) -> Result { // Get resource type from each workspace let source_resource_type = sqlx::query!( - "SELECT schema, description, format_extension + "SELECT schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1 AND name = $2", source_workspace_id, @@ -5264,7 +5264,7 @@ async fn compare_two_resource_types( .await?; let target_resource_type = sqlx::query!( - "SELECT schema, description, format_extension + "SELECT schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1 AND name = $2", fork_workspace_id, @@ -5280,6 +5280,7 @@ async fn compare_two_resource_types( if source.schema != target.schema || source.description != target.description || source.format_extension != target.format_extension + || source.is_fileset != target.is_fileset { has_changes = true; } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f2796962c8..46131043d5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5244,10 +5244,19 @@ paths: - $ref: "#/components/parameters/WorkspaceId" responses: "200": - description: map from resource type to file ext + description: map from resource type to file resource info content: application/json: - schema: {} + schema: + type: object + additionalProperties: + type: object + properties: + format_extension: + type: string + nullable: true + is_fileset: + type: boolean /w/{workspace}/resources/type/delete/{path}: delete: @@ -19885,6 +19894,8 @@ components: format: date-time format_extension: type: string + is_fileset: + type: boolean required: - name @@ -19894,6 +19905,8 @@ components: schema: {} description: type: string + is_fileset: + type: boolean Schedule: type: object diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 07fcd6c93b..269edf2612 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -98,6 +98,7 @@ pub struct ResourceType { pub created_by: Option, pub edited_at: Option>, pub format_extension: Option, + pub is_fileset: bool, } #[derive(Deserialize)] @@ -106,12 +107,14 @@ pub struct CreateResourceType { pub schema: Option, pub description: Option, pub format_extension: Option, + pub is_fileset: Option, } #[derive(Deserialize)] pub struct EditResourceType { pub schema: Option, pub description: Option, + pub is_fileset: Option, } #[derive(FromRow, Serialize, Deserialize)] @@ -1209,29 +1212,38 @@ async fn update_resource_value( Ok(format!("value of resource {} updated", path)) } +#[derive(Serialize)] +pub struct FileResourceTypeInfo { + pub format_extension: Option, + pub is_fileset: bool, +} + async fn file_resource_ext_to_resource_type( Extension(db): Extension, Path(w_id): Path, -) -> JsonResult> { - #[derive(Serialize, sqlx::FromRow)] +) -> JsonResult> { + #[derive(sqlx::FromRow)] struct LocalFileResourceExtension { name: String, format_extension: Option, + is_fileset: bool, } let r = sqlx::query_as!(LocalFileResourceExtension, " - SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')", w_id) + SELECT name, format_extension, is_fileset FROM resource_type WHERE (format_extension IS NOT NULL OR is_fileset = true) AND (workspace_id = $1 OR workspace_id = 'admins')", w_id) .fetch_all(&db) .await?; - let hashmap: HashMap = r + let hashmap: HashMap = r .into_iter() - .filter_map(|entry| { - if let Some(format_extension) = entry.format_extension { - Some((entry.name, format_extension)) - } else { - None - } + .map(|entry| { + ( + entry.name, + FileResourceTypeInfo { + format_extension: entry.format_extension, + is_fileset: entry.is_fileset, + }, + ) }) .collect(); @@ -1331,16 +1343,25 @@ async fn create_resource_type( check_rt_path_conflict(&mut tx, &w_id, &resource_type.name).await?; + let is_fileset = resource_type.is_fileset.unwrap_or(false); + + if is_fileset && resource_type.format_extension.is_some() { + return Err(Error::BadRequest( + "A fileset resource type cannot have a format_extension".to_string(), + )); + } + sqlx::query!( "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, now())", + (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, now())", w_id, resource_type.name, resource_type.schema, resource_type.description, authed.username, resource_type.format_extension, + is_fileset, ) .execute(&mut *tx) .await?; @@ -1501,6 +1522,9 @@ async fn update_resource_type( if let Some(ndesc) = ns.description { sqlb.set_str("description", ndesc); } + if let Some(is_fileset) = ns.is_fileset { + sqlb.set("is_fileset", if is_fileset { "TRUE" } else { "FALSE" }); + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; diff --git a/cli/package-lock.json b/cli/package-lock.json index 872bed83d9..80027c2490 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -6,16 +6,11 @@ "": { "name": "wmill-dev", "dependencies": { - "@ayonli/jsext": "^1.9.0", "@cliffy/ansi": "npm:@jsr/cliffy__ansi@1.0.0", "@cliffy/command": "npm:@jsr/cliffy__command@1.0.0", "@cliffy/prompt": "npm:@jsr/cliffy__prompt@1.0.0", "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", - "@std/encoding": "npm:@jsr/std__encoding@1.0.10", - "@std/log": "npm:@jsr/std__log@0.224.14", - "@std/path": "npm:@jsr/std__path@1.1.4", - "@std/yaml": "npm:@jsr/std__yaml@1.0.10", - "@windmill-labs/shared-utils": "npm:@jsr/windmill-labs__shared-utils@1.0.12", + "@windmill-labs/shared-utils": "^1.0.12", "diff": "^5.2.0", "esbuild": "0.24.2", "get-port": "7.1.0", @@ -23,6 +18,7 @@ "minimatch": "^10.0.0", "open": "^10.0.0", "svelte": "^5.45.2", + "tar-stream": "^3.1.7", "windmill-parser-wasm-csharp": "*", "windmill-parser-wasm-go": "*", "windmill-parser-wasm-java": "*", @@ -44,25 +40,11 @@ "devDependencies": { "@types/diff": "^5.2.3", "@types/node": "^22.0.0", + "@types/tar-stream": "^3.1.4", "@types/ws": "^8.5.0", "typescript": "^5.7.0" } }, - "node_modules/@ayonli/jsext": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@ayonli/jsext/-/jsext-1.9.0.tgz", - "integrity": "sha512-hIu6lQhoLr5e26lmt+vzopuZffaAyb623r4+8HlN/rhXgm2ywHslzk7UHiATdfDbfPjBARkB6cfXjVEi3aav6g==", - "license": "MIT", - "dependencies": { - "iconv-lite": "^0.6.3", - "sudo-prompt": "^9.2.1", - "ws": "^8.17.0", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=14.18" - } - }, "node_modules/@cliffy/ansi": { "name": "@jsr/cliffy__ansi", "version": "1.0.0", @@ -623,15 +605,6 @@ "resolved": "https://npm.jsr.io/~/11/@jsr/std__fmt/1.0.9.tgz", "integrity": "sha512-YFJJMozmORj2K91c5J9opWeh0VUwrd+Mwb7Pr0FkVCAKVLu2UhT4LyvJqWiyUT+eF+MdfqQ9F7RtQj4bXn9Smw==" }, - "node_modules/@jsr/std__fs": { - "version": "1.0.23", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__fs/1.0.23.tgz", - "integrity": "sha512-e8jspB3M44E5YhWiLCTqibBBTwVmxQaHN06WvFa/elAKm5E/LfAe8Hj5XGNC8P7a0MIPASlNJsnF1bgO/g+aqg==", - "dependencies": { - "@jsr/std__internal": "^1.0.12", - "@jsr/std__path": "^1.1.4" - } - }, "node_modules/@jsr/std__internal": { "version": "1.0.12", "resolved": "https://npm.jsr.io/~/11/@jsr/std__internal/1.0.12.tgz", @@ -671,38 +644,6 @@ "@jsr/std__regexp": "^1.0.1" } }, - "node_modules/@std/encoding": { - "name": "@jsr/std__encoding", - "version": "1.0.10", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__encoding/1.0.10.tgz", - "integrity": "sha512-WK2njnDTyKefroRNk2Ooq7GStp6Y0ccAvr4To+Z/zecRAGe7+OSvH9DbiaHpAKwEi2KQbmpWMOYsdNt+TsdmSw==" - }, - "node_modules/@std/log": { - "name": "@jsr/std__log", - "version": "0.224.14", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__log/0.224.14.tgz", - "integrity": "sha512-EHT7E0plakyzk/gxMrwqUf3YGCCxN3Is25QrEh7toYA7qwj46R4qY7cIaDEKy8QqI5JHOFHwWXOClcPK6goIoQ==", - "dependencies": { - "@jsr/std__fmt": "^1.0.5", - "@jsr/std__fs": "^1.0.11", - "@jsr/std__io": "^0.225.2" - } - }, - "node_modules/@std/path": { - "name": "@jsr/std__path", - "version": "1.1.4", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__path/1.1.4.tgz", - "integrity": "sha512-SK4u9H6NVTfolhPdlvdYXfNFefy1W04AEHWJydryYbk+xqzNiVmr5o7TLJLJFqwHXuwMRhwrn+mcYeUfS0YFaA==", - "dependencies": { - "@jsr/std__internal": "^1.0.12" - } - }, - "node_modules/@std/yaml": { - "name": "@jsr/std__yaml", - "version": "1.0.10", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__yaml/1.0.10.tgz", - "integrity": "sha512-1WIM023Kvi48pvPE3UO5YcieambLgywUooLhAkkaObIcMB77F/YP2ILdl+vNfik+vElkl9znmuST9AZo8mbCpA==" - }, "node_modules/@stoplight/ordered-object-literal": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@stoplight/ordered-object-literal/-/ordered-object-literal-1.0.5.tgz", @@ -784,6 +725,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/tar-stream": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-3.1.4.tgz", + "integrity": "sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -852,6 +803,20 @@ "node": ">= 0.4" } }, + "node_modules/b4a": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.0.tgz", + "integrity": "sha512-qRuSmNSkGQaHwNbM7J78Wwy+ghLEYF1zNrSeMxj4Kgw6y33O3mXcQ6Ie9fRvfU/YnxWkOchPXbaLb73TkIsfdg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.3.tgz", @@ -861,6 +826,20 @@ "node": "20 || >=22" } }, + "node_modules/bare-events": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, "node_modules/brace-expansion": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", @@ -1013,12 +992,27 @@ "@jridgewell/sourcemap-codec": "^1.4.15" } }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", @@ -1047,18 +1041,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", @@ -1263,12 +1245,6 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, "node_modules/set-immediate-shim": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", @@ -1278,6 +1254,17 @@ "node": ">=0.10.0" } }, + "node_modules/streamx": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -1287,13 +1274,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/sudo-prompt": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-9.2.1.tgz", - "integrity": "sha512-Mu7R0g4ig9TUuGSxJavny5Rv0egCEtpZRNMrZaYS1vxkiIxGiGUwoezU3LazIQ+KE04hTrTfNPgxU5gzi7F5Pw==", - "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", - "license": "MIT" - }, "node_modules/svelte": { "version": "5.53.2", "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.2.tgz", @@ -1321,6 +1301,26 @@ "node": ">=18" } }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1484,15 +1484,6 @@ "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.4.tgz", "integrity": "sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ==", "license": "MIT" - }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } } } } diff --git a/cli/src/commands/resource/resource.ts b/cli/src/commands/resource/resource.ts index 400130f9b8..e28a479f13 100644 --- a/cli/src/commands/resource/resource.ts +++ b/cli/src/commands/resource/resource.ts @@ -1,5 +1,6 @@ -import { stat, writeFile } from "node:fs/promises"; +import { stat, writeFile, readdir, readFile } from "node:fs/promises"; import { stringify as yamlStringify } from "yaml"; +import nodePath from "node:path"; import { GlobalOptions, @@ -27,6 +28,24 @@ export interface ResourceFile { is_oauth?: boolean; // deprecated } +async function readFilesetDirectory(dirPath: string): Promise> { + const result: Record = {}; + async function walk(currentPath: string, prefix: string) { + const entries = await readdir(currentPath, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = nodePath.join(currentPath, entry.name); + const relPath = prefix ? prefix + "/" + entry.name : entry.name; + if (entry.isDirectory()) { + await walk(entryPath, relPath); + } else if (entry.isFile()) { + result[relPath] = await readFile(entryPath, "utf-8"); + } + } + } + await walk(dirPath, ""); + return result; +} + export async function pushResource( workspace: string, remotePath: string, @@ -46,7 +65,10 @@ export async function pushResource( // Helper function to resolve inline content const resolveInlineContent = async () => { - if (localResource.value["content"]?.startsWith("!inline ")) { + if (typeof localResource.value === "string" && localResource.value.startsWith("!inline_fileset ")) { + const dirPath = localResource.value.split(" ")[1]; + localResource.value = await readFilesetDirectory(dirPath.replaceAll("/", SEP)); + } else if (localResource.value["content"]?.startsWith("!inline ")) { const basePath = localResource.value["content"].split(" ")[1]; // If we're processing a branch-specific metadata file, read from branch-specific resource file diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index bf2ecf7826..0f6d90c182 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -38,6 +38,7 @@ import { deepEqual, fetchRemoteVersion, isFileResource, + isFilesetResource, isRawAppFile, isWorkspaceDependencies, } from "../../utils/utils.ts"; @@ -484,11 +485,53 @@ export function extractInlineScriptsForApps( return []; } +type FileResourceTypeInfo = { format_extension: string | null; is_fileset: boolean }; + +function parseFileResourceTypeMap( + raw: Record, +): { formatExtMap: Record; filesetMap: Record } { + const formatExtMap: Record = {}; + const filesetMap: Record = {}; + for (const [k, v] of Object.entries(raw)) { + if (typeof v === "string") { + formatExtMap[k] = v; + filesetMap[k] = false; + } else { + if (v.format_extension) { + formatExtMap[k] = v.format_extension; + } + filesetMap[k] = v.is_fileset ?? false; + } + } + return { formatExtMap, filesetMap }; +} + +async function findFilesetResourceFile(changePath: string): Promise { + // Extract the base path before .fileset/ + const filesetIdx = changePath.indexOf(".fileset" + SEP); + if (filesetIdx === -1) { + throw new Error(`Not a fileset resource path: ${changePath}`); + } + const basePath = changePath.substring(0, filesetIdx); + const candidates = [basePath + ".resource.json", basePath + ".resource.yaml"]; + + for (const candidate of candidates) { + try { + const s = await stat(candidate); + if (s.isFile()) return candidate; + } catch { + // not found, try next + } + } + throw new Error(`No resource metadata file found for fileset resource: ${changePath}`); +} + function ZipFSElement( zip: JSZip, useYaml: boolean, defaultTs: "bun" | "deno", resourceTypeToFormatExtension: Record, + resourceTypeToIsFileset: Record, ignoreCodebaseChanges: boolean, ): DynFSElement { async function _internal_file( @@ -860,10 +903,17 @@ function ZipFSElement( log.error(`Failed to parse resource.yaml at path: ${p}`); throw error; } + const resourceType = parsed["resource_type"]; const formatExtension = - resourceTypeToFormatExtension[parsed["resource_type"]]; + resourceTypeToFormatExtension[resourceType]; + const isFileset = resourceTypeToIsFileset[resourceType] ?? false; - if (formatExtension) { + if (isFileset) { + parsed["value"] = + "!inline_fileset " + + removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") + + ".fileset"; + } else if (formatExtension) { parsed["value"]["content"] = "!inline " + removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") + @@ -918,10 +968,37 @@ function ZipFSElement( log.error(`Failed to parse resource file content at path: ${p}`); throw error; } + const resourceType = parsed["resource_type"]; const formatExtension = - resourceTypeToFormatExtension[parsed["resource_type"]]; + resourceTypeToFormatExtension[resourceType]; + const isFileset = resourceTypeToIsFileset[resourceType] ?? false; - if (formatExtension) { + if (isFileset && typeof parsed["value"] === "object" && parsed["value"] !== null) { + const filesetBasePath = + removeSuffix(finalPath, ".resource.json") + ".fileset"; + // Push directory entry for the fileset + r.push({ + isDirectory: true, + path: filesetBasePath, + async *getChildren() { + for (const [relPath, fileContent] of Object.entries(parsed["value"])) { + if (typeof fileContent === "string") { + yield { + isDirectory: false, + path: path.join(filesetBasePath, relPath), + async *getChildren() {}, + async getContentText() { + return fileContent; + }, + }; + } + } + }, + async getContentText() { + throw new Error("Cannot get content of directory"); + }, + }); + } else if (formatExtension) { const fileContent: string = parsed["value"]["content"]; if (typeof fileContent === "string") { r.push({ @@ -1058,6 +1135,7 @@ export async function elementsToMap( const path = entry.path; if ( !isFileResource(path) && + !isFilesetResource(path) && !isRawAppFile(path) && !isWorkspaceDependencies(path) ) { @@ -1103,7 +1181,7 @@ export async function elementsToMap( } } - if (skips.skipResources && isFileResource(path)) continue; + if (skips.skipResources && (isFileResource(path) || isFilesetResource(path))) continue; const ext = json ? ".json" : ".yaml"; if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; @@ -1715,10 +1793,14 @@ export async function pull( ); let resourceTypeToFormatExtension: Record = {}; + let resourceTypeToIsFileset: Record = {}; try { - resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({ + const raw = (await wmill.fileResourceTypeToFileExtMap({ workspace: workspace.workspaceId, - })) as Record; + })) as Record; + const parsed = parseFileResourceTypeMap(raw); + resourceTypeToFormatExtension = parsed.formatExtMap; + resourceTypeToIsFileset = parsed.filesetMap; } catch { // ignore } @@ -1745,6 +1827,7 @@ export async function pull( !opts.json, opts.defaultTs ?? "bun", resourceTypeToFormatExtension, + resourceTypeToIsFileset, true, ); @@ -2241,10 +2324,14 @@ export async function push( ), ); let resourceTypeToFormatExtension: Record = {}; + let resourceTypeToIsFileset: Record = {}; try { - resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({ + const raw = (await wmill.fileResourceTypeToFileExtMap({ workspace: workspace.workspaceId, - })) as Record; + })) as Record; + const parsed = parseFileResourceTypeMap(raw); + resourceTypeToFormatExtension = parsed.formatExtMap; + resourceTypeToIsFileset = parsed.filesetMap; } catch { // ignore } @@ -2269,6 +2356,7 @@ export async function push( !opts.json, opts.defaultTs ?? "bun", resourceTypeToFormatExtension, + resourceTypeToIsFileset, false, ); @@ -2587,6 +2675,39 @@ export async function push( continue; } } + if (isFilesetResource(change.path)) { + const resourceFilePath = await findFilesetResourceFile(change.path); + if (!alreadySynced.includes(resourceFilePath)) { + alreadySynced.push(resourceFilePath); + + const newObj = parseFromPath( + resourceFilePath, + await readFile(resourceFilePath, "utf-8"), + ); + + let serverPath = resourceFilePath; + const currentBranch = cachedBranchForPush; + + if (currentBranch && isBranchSpecificFile(resourceFilePath)) { + serverPath = fromBranchSpecificPath( + resourceFilePath, + currentBranch, + ); + } + + await pushResource( + workspace.workspaceId, + serverPath, + undefined, + newObj, + resourceFilePath, + ); + if (stateTarget) { + await writeFile(stateTarget, change.after, "utf-8"); + } + continue; + } + } const oldObj = parseFromPath(change.path, change.before); const newObj = parseFromPath(change.path, change.after); @@ -2619,7 +2740,8 @@ export async function push( change.path.endsWith(".script.json") || change.path.endsWith(".script.yaml") || change.path.endsWith(".lock") || - isFileResource(change.path) + isFileResource(change.path) || + isFilesetResource(change.path) ) { continue; } else if ( diff --git a/cli/src/core/specific_items.ts b/cli/src/core/specific_items.ts index aa3f6652fc..cfd806d9c6 100644 --- a/cli/src/core/specific_items.ts +++ b/cli/src/core/specific_items.ts @@ -1,6 +1,6 @@ import { minimatch } from "minimatch"; import { getCurrentGitBranch, isGitRepository } from "../utils/git.ts"; -import { isFileResource } from "../utils/utils.ts"; +import { isFileResource, isFilesetResource } from "../utils/utils.ts"; import { SyncOptions } from "./conf.ts"; import { TRIGGER_TYPES } from "../types.ts"; @@ -165,7 +165,7 @@ export function isItemTypeConfigured(path: string, specificItems: SpecificItemsC return specificItems.settings !== undefined; } - if (isFileResource(path)) { + if (isFileResource(path) || isFilesetResource(path)) { return specificItems.resources !== undefined; } @@ -219,6 +219,14 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig } } + if (isFilesetResource(path)) { + const basePathMatch = path.match(/^(.+?)\.fileset[/\\]/); + if (basePathMatch && specificItems.resources) { + const basePath = basePathMatch[1] + '.resource.yaml'; + return matchesPatterns(basePath, specificItems.resources); + } + } + return false; } diff --git a/cli/src/main.ts b/cli/src/main.ts old mode 100644 new mode 100755 diff --git a/cli/src/types.ts b/cli/src/types.ts index 382f7f82af..69592664f8 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -14,7 +14,7 @@ import { pushResourceType } from "./commands/resource-type/resource-type.ts"; import { pushVariable } from "./commands/variable/variable.ts"; import { yamlOptions } from "./commands/sync/sync.ts"; import { showDiffs } from "./core/conf.ts"; -import { deepEqual, isFileResource, isWorkspaceDependencies } from "./utils/utils.ts"; +import { deepEqual, isFileResource, isFilesetResource, isWorkspaceDependencies } from "./utils/utils.ts"; import { pushSchedule } from "./commands/schedule/schedule.ts"; import { pushWorkspaceUser } from "./commands/user/user.ts"; import { pushGroup } from "./commands/user/user.ts"; @@ -333,7 +333,7 @@ export function getTypeStrFromPath( ) { return typeEnding; } else { - if (isFileResource(p)) { + if (isFileResource(p) || isFilesetResource(p)) { return "resource"; } throw new Error("Could not infer type of path " + JSON.stringify(parsed)); diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 02a8c1a643..766b372a0a 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -154,6 +154,11 @@ export function isFileResource(path: string): boolean { ); } +/** Matches children inside a .fileset/ directory, not the directory itself. */ +export function isFilesetResource(path: string): boolean { + return path.includes(".fileset/") || path.includes(".fileset\\"); +} + export function isRawAppFile(path: string): boolean { return isRawAppPath(path); } diff --git a/cli/test/utils_unit.test.ts b/cli/test/utils_unit.test.ts index f72ba23173..7bc3985a9c 100644 --- a/cli/test/utils_unit.test.ts +++ b/cli/test/utils_unit.test.ts @@ -4,7 +4,7 @@ */ import { expect, test, describe } from "bun:test"; -import { deepEqual, isFileResource, toCamel, capitalize } from "../src/utils/utils.ts"; +import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize } from "../src/utils/utils.ts"; import { getTypeStrFromPath, removeType, @@ -156,6 +156,32 @@ describe("isFileResource", () => { }); }); +// ============================================================================= +// isFilesetResource +// ============================================================================= + +describe("isFilesetResource", () => { + test("detects fileset resource paths (unix separator)", () => { + expect(isFilesetResource("f/test/my_config.fileset/config.yaml")).toBe(true); + expect(isFilesetResource("u/admin/templates.fileset/path/to/file.txt")).toBe(true); + }); + + test("detects fileset resource paths (windows separator)", () => { + expect(isFilesetResource("f\\test\\my_config.fileset\\config.yaml")).toBe(true); + }); + + test("rejects non-fileset paths", () => { + expect(isFilesetResource("f/test/my_resource.resource.yaml")).toBe(false); + expect(isFilesetResource("f/test/my_file.resource.file.txt")).toBe(false); + expect(isFilesetResource("f/test/my_script.ts")).toBe(false); + }); + + test("rejects paths ending with .fileset (no child file)", () => { + // The directory itself is not a fileset resource file - only children are + expect(isFilesetResource("f/test/my_config.fileset")).toBe(false); + }); +}); + // ============================================================================= // removeType // ============================================================================= @@ -241,6 +267,11 @@ describe("getTypeStrFromPath", () => { expect(getTypeStrFromPath("devs.group.yaml")).toBe("group"); }); + test("detects fileset resource files as resource type", () => { + expect(getTypeStrFromPath("f/test/my_config.fileset/config.yaml")).toBe("resource"); + expect(getTypeStrFromPath("u/admin/templates.fileset/path/to/file.txt")).toBe("resource"); + }); + test("throws for unknown type", () => { expect(() => getTypeStrFromPath("f/test/unknown.xyz.yaml")).toThrow(); }); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4aac239774..4f7ce29b87 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -835,7 +835,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", "integrity": "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -847,7 +846,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -858,7 +856,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1348,7 +1345,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.0.tgz", "integrity": "sha512-Fq6DJW+Bb5jaWE69/qOE0D1TUN9+6uWhCeZpdnSBk14pjLcCWR7Q8n49PTSPHazM37JqrsdpEthXy2xn6jWWiA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1503,7 +1499,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1520,7 +1515,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1537,7 +1531,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1554,7 +1547,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1571,7 +1563,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1588,7 +1579,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1605,7 +1595,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1622,7 +1611,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1639,7 +1627,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1656,7 +1643,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1673,7 +1659,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1690,7 +1675,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1707,7 +1691,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2313,7 +2296,6 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7193,7 +7175,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7692,7 +7674,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7713,7 +7694,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7734,7 +7714,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7755,7 +7734,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7776,7 +7754,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7797,7 +7774,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7818,7 +7794,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7839,7 +7814,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7860,7 +7834,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7881,7 +7854,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7902,7 +7874,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12529,21 +12500,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index a557b15b4a..d8429a5ebf 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -1,5 +1,6 @@ + +{#if !hideHeader} +
+ Files +
+ + +
+
+{/if} +
+ {#if showRoot} + + {/if} + {#each fileTree as node (node.path)} + (pathToEdit = path)} + onCancelEdit={() => { + pathToEdit = undefined + pendingNewFilePath = undefined + }} + /> + {/each} + {#if extraNodes} + {#each extraNodes as node (node.path)} + + {/each} + {/if} +
diff --git a/frontend/src/lib/components/FilesetEditor.svelte b/frontend/src/lib/components/FilesetEditor.svelte new file mode 100644 index 0000000000..96956e038a --- /dev/null +++ b/frontend/src/lib/components/FilesetEditor.svelte @@ -0,0 +1,120 @@ + + +
+
+ +
+
+ {#if selectedDisplayKey != null} +
+ {selectedDisplayKey} +
+ {#key selectedFileKey} + + {/key} + {:else} +
+ Select a file or add a new one +
+ {/if} +
+
diff --git a/frontend/src/lib/components/IconedResourceType.svelte b/frontend/src/lib/components/IconedResourceType.svelte index 212d0fdd73..056af6c320 100644 --- a/frontend/src/lib/components/IconedResourceType.svelte +++ b/frontend/src/lib/components/IconedResourceType.svelte @@ -1,5 +1,5 @@ {#snippet action()}
-
- - -
+ +
{/snippet} -
- {#each fileTree as node (node.path)} - (pathToEdit = path)} - onCancelEdit={() => { - pathToEdit = undefined - pendingNewFilePath = undefined - }} - /> - {/each} - -
+ diff --git a/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte b/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte index 6227431e67..27f19928f1 100644 --- a/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte +++ b/frontend/src/lib/components/schema/EditableSchemaWrapper.svelte @@ -1,31 +1,40 @@ -{#if !resourceIsTextFile} +{#if resourceMode === 'schema'}
{/if} -{#if resourceIsTextFile} +{#if resourceMode === 'file'}
- {#if editResourceType.formatExtension} + {#if editResourceType.isFileset} + + This resource type represents a collection of files. The schema cannot be edited. + + {:else if editResourceType.formatExtension} This resource type represents a plain text file with a .{editResourceType.formatExtension}
@@ -895,7 +952,8 @@ //@ts-ignore schema: linkedRt.schema, description: linkedRt.description ?? '', - formatExtension: linkedRt.format_extension + formatExtension: linkedRt.format_extension, + isFileset: linkedRt.is_fileset ?? false } resourceTypeViewer?.openDrawer?.() } else { @@ -910,6 +968,7 @@ name={resource_type} after={true} formatExtension={resourceNameToFileExt(resource_type)} + isFileset={resourceNameIsFileset(resource_type)} /> @@ -1096,7 +1155,7 @@ {#if resourceTypes} - {#each resourceTypes as { name, description, schema, canWrite, format_extension }} + {#each resourceTypes as { name, description, schema, canWrite, format_extension, is_fileset }}