starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-23 08:00:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into di/data-tables

Browse Source
This commit is contained in:
Diego Imbert
2025-12-01 12:52:48 +01:00
parent e62794b64e 717db01a16
commit 0e52c4007d
223 changed files with 12821 additions and 4215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-test.yml
+1
View File
@@ -83,6 +83,7 @@ jobs:
RUST_LOG: info
RUST_LOG_STYLE: never
CARGO_NET_GIT_FETCH_WITH_CLI: true
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
run: |
deno --version && bun -v && go version && python3 --version
cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd ..
CHANGELOG.md
+37
View File
@@ -1,5 +1,42 @@
# Changelog
## [1.587.1](https://github.com/windmill-labs/windmill/compare/v1.587.0...v1.587.1) (2025-12-01)
### Bug Fixes
* fix public apps by custom url ([9e7be4b](https://github.com/windmill-labs/windmill/commit/9e7be4b55efdc912c39994091a0db0d70ed7e83c))
## [1.587.0](https://github.com/windmill-labs/windmill/compare/v1.586.0...v1.587.0) (2025-11-30)
### Features
* **aichat:** stream tool arguments ([#7244](https://github.com/windmill-labs/windmill/issues/7244)) ([8d6936a](https://github.com/windmill-labs/windmill/commit/8d6936ae4a8577983405d95ab75f99822f15da3d))
* workspace dependencies ([#7124](https://github.com/windmill-labs/windmill/issues/7124)) ([d38c96d](https://github.com/windmill-labs/windmill/commit/d38c96db369bf0a9a0640e11d7fee16605a6775a))
### Bug Fixes
* **cli:** cli behave as expected in forked workspaces ([58ef965](https://github.com/windmill-labs/windmill/commit/58ef965e20ec25456da23b27fe16b8abab1889f6))
* **git-sync:** initialize repo with gitBranches set ([49b2ea6](https://github.com/windmill-labs/windmill/commit/49b2ea65316be3cf0b95af4b2e69c10c01f35d1a))
## [1.586.0](https://github.com/windmill-labs/windmill/compare/v1.585.1...v1.586.0) (2025-11-27)
### Features
* add license key expiration warning on workers page ([#7225](https://github.com/windmill-labs/windmill/issues/7225)) ([d876c2c](https://github.com/windmill-labs/windmill/commit/d876c2c31c1183226e47443c5fb2f5885647d303))
* **app:** Add progress bar app component ([#7242](https://github.com/windmill-labs/windmill/issues/7242)) ([267171f](https://github.com/windmill-labs/windmill/commit/267171f2c9b1639ade8bf717d7f50d55ec2b9767))
### Bug Fixes
* **bun:** do not add builtin to lockfiles ([e3b5975](https://github.com/windmill-labs/windmill/commit/e3b59752bd0a3f278465c783a0508c5394b58119))
* **cli:** support better esm mode for codebases ([e8fd36e](https://github.com/windmill-labs/windmill/commit/e8fd36e2e7578e21aeccb094bc4526c7fa4ff70c))
* **cli:** update jszip to 3.8.0 ([d22d8b7](https://github.com/windmill-labs/windmill/commit/d22d8b7af020afbf2f448047ceee0e9c7d46b3f0))
* **frontend:** check resource type name conflict in frontend ([#7237](https://github.com/windmill-labs/windmill/issues/7237)) ([fc1a52c](https://github.com/windmill-labs/windmill/commit/fc1a52c1b3bc4f3077de862162d65f81360484b7))
## [1.585.1](https://github.com/windmill-labs/windmill/compare/v1.585.0...v1.585.1) (2025-11-26)
Dockerfile
+1
View File
@@ -100,6 +100,7 @@ ARG POWERSHELL_VERSION=7.5.0
ARG POWERSHELL_DEB_VERSION=7.5.0-1
ARG KUBECTL_VERSION=1.28.7
ARG HELM_VERSION=3.14.3
# NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte
ARG GO_VERSION=1.25.0
ARG APP=/usr/src/app
ARG WITH_POWERSHELL=true
backend/.sqlx/query-04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM v2_job",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493"
}
backend/.sqlx/query-05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at)\n SELECT $1, language, name, description, content, archived, created_at\n FROM workspace_dependencies \n WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c"
}
backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "hash",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd"
}
backend/.sqlx/query-0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d.json
Generated
-24
View File
@@ -1,24 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false
]
},
"hash": "0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d"
}
backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json
Generated
+47
View File
@@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_dependencies\n SET archived = true\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": []
},
"hash": "0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0"
}
backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n importer_path,\n importer_kind::text as \"importer_kind!\", -- sqlx thinks this is nullable somehow, so enfore with !\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "importer_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "importer_kind!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "importer_node_ids",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null,
null
]
},
"hash": "1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708"
}
backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json
Generated
+96
View File
@@ -0,0 +1,96 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, archived, description, workspace_id, created_at\n FROM workspace_dependencies\n WHERE id = $1 AND workspace_id = $2\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041"
}
backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_dependencies\n SET archived = true \n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "description",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false
]
},
"hash": "37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b"
}
backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json
Generated
+47
View File
@@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": []
},
"hash": "5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab"
}
backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json
Generated
+57
View File
@@ -0,0 +1,57 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5) \n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
},
"Text"
]
},
"nullable": [
false
]
},
"hash": "5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0"
}
backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json
Generated
+65
View File
@@ -0,0 +1,65 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, hash, language AS \"language: ScriptLang\" FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "hash",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false
]
},
"hash": "5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea"
}
backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json → backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar, codebase LIKE '%.esm%' as is_esm FROM script WHERE hash = $1 LIMIT 1",
"describe": {
"columns": [
{
@@ -68,6 +68,11 @@
"ordinal": 6,
"name": "use_tar",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "is_esm",
"type_info": "Bool"
}
],
"parameters": {
@@ -82,8 +87,9 @@
true,
true,
false,
null,
null
]
},
"hash": "a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a"
"hash": "7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662"
}
backend/.sqlx/query-9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0.json
Generated
-66
View File
@@ -1,66 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "display_name",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owners",
"type_info": "VarcharArray"
},
{
"ordinal": 4,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "summary",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "edited_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
true,
true
]
},
"hash": "9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0"
}
backend/.sqlx/query-a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6"
}
backend/.sqlx/query-abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d.json
Generated
-42
View File
@@ -1,42 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "extra_perms",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
true,
false
]
},
"hash": "abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d"
}
backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false
]
},
"hash": "bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d"
}
backend/.sqlx/query-c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n imported_path,\n COUNT(DISTINCT importer_path) as \"count!\"\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = ANY($2)\n GROUP BY imported_path\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "imported_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"TextArray"
]
},
"nullable": [
false,
null
]
},
"hash": "c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c"
}
backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json
Generated
+95
View File
@@ -0,0 +1,95 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, created_at, archived, name, description, workspace_id, content, language AS \"language: ScriptLang\"\n FROM workspace_dependencies\n WHERE archived = false AND workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 2,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d"
}
backend/.sqlx/query-d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name desc",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce"
}
backend/.sqlx/query-e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM v2_job_queue",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39"
}
backend/.sqlx/query-ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4.json
Generated
-35
View File
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT importer_path, importer_kind::text, array_agg(importer_node_id) as importer_node_ids FROM dependency_map\n WHERE imported_path = $1\n AND workspace_id = $2\n GROUP BY importer_path, importer_kind",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "importer_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "importer_kind",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "importer_node_ids",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null,
null
]
},
"hash": "ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4"
}
backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json
Generated
+128
View File
@@ -0,0 +1,128 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, description, archived, workspace_id, created_at\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc"
}
backend/.sqlx/query-f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f.json
Generated
+51
View File
@@ -0,0 +1,51 @@
{
"db_name": "PostgreSQL",
"query": "SELECT language AS \"language: ScriptLang\" FROM script WHERE path = 'u/admin/hub_sync'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f"
}
backend/Cargo.lock
Generated
+54 -45
View File
@@ -2008,9 +2008,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.47"
version = "1.2.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd405d82c84ff7f35739f175f67d8b9fb7687a0e84ccdc78bd3568839827cf07"
checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -4797,9 +4797,9 @@ dependencies = [
[[package]]
name = "dlopen2_derive"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4"
checksum = "95f4a04e1bfbfa4835a6073177aafb95ead4de0722dbb339195fdc7e0a09599b"
dependencies = [
"proc-macro2",
"quote",
@@ -6538,13 +6538,13 @@ dependencies = [
[[package]]
name = "hostname"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65"
checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd"
dependencies = [
"cfg-if",
"libc",
"windows-link 0.1.3",
"windows-link 0.2.1",
]
[[package]]
@@ -11232,9 +11232,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.13.0"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a"
checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c"
dependencies = [
"web-time",
"zeroize",
@@ -14073,9 +14073,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
version = "0.1.41"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647"
dependencies = [
"log",
"pin-project-lite",
@@ -14188,9 +14188,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.20"
version = "0.3.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
dependencies = [
"matchers",
"nu-ansi-term",
@@ -15162,7 +15162,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"aws-sdk-config",
@@ -15193,6 +15193,7 @@ dependencies = [
"reqwest 0.12.24",
"rustls 0.23.35",
"serde",
"serde_derive",
"serde_json",
"serde_yml",
"sha1",
@@ -15224,7 +15225,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"argon2",
@@ -15345,7 +15346,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -15360,7 +15361,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"chrono",
"lazy_static",
@@ -15374,7 +15375,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"axum",
@@ -15393,7 +15394,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15439,6 +15440,8 @@ dependencies = [
"opentelemetry-otlp",
"opentelemetry-semantic-conventions",
"opentelemetry_sdk",
"pep440_rs",
"phf 0.11.3",
"pin-project-lite",
"prometheus",
"quick_cache",
@@ -15451,6 +15454,7 @@ dependencies = [
"semver 1.0.27",
"serde",
"serde_json",
"serde_yml",
"sha2 0.10.9",
"size",
"sqlx",
@@ -15483,7 +15487,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"regex",
"serde",
@@ -15498,7 +15502,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"bytes",
@@ -15522,19 +15526,23 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"itertools 0.14.0",
"lazy_static",
"pep440_rs",
"proc-macro2",
"quote",
"regex",
"serde",
"serde_derive",
"serde_yml",
"syn 2.0.111",
]
[[package]]
name = "windmill-parser"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15543,7 +15551,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15555,7 +15563,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"serde_json",
@@ -15567,7 +15575,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"gosyn",
@@ -15579,7 +15587,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15591,7 +15599,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"serde_json",
@@ -15603,7 +15611,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -15614,7 +15622,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15625,7 +15633,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15637,7 +15645,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15654,13 +15662,14 @@ dependencies = [
"serde_json",
"sqlx",
"toml",
"tracing",
"windmill-common",
"windmill-parser",
]
[[package]]
name = "windmill-parser-ruby"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15674,7 +15683,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15691,7 +15700,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15705,7 +15714,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15723,7 +15732,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"getrandom 0.2.16",
@@ -15748,7 +15757,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"serde",
@@ -15759,7 +15768,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15796,7 +15805,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -15806,7 +15815,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.585.1"
version = "1.587.1"
dependencies = [
"anyhow",
"async-once-cell",
@@ -16603,18 +16612,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.30"
version = "0.8.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c"
checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.30"
version = "0.8.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5"
checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a"
dependencies = [
"proc-macro2",
"quote",
backend/Cargo.toml
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.585.1"
version = "1.587.1"
authors.workspace = true
edition.workspace = true
@@ -34,7 +34,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.585.1"
version = "1.587.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -138,6 +138,7 @@ prometheus = { workspace = true, optional = true }
uuid.workspace = true
gethostname.workspace = true
serde_json.workspace = true
serde_derive.workspace = true
serde_yml.workspace = true
serde.workspace = true
deno_core = { workspace = true, optional = true }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
99828e4a3a27af428a45d1e6e65092ca15082b1e
2e6cce1bcb9deb750276ffbde4c7ca1f22e70c62
backend/migrations/20251106152104_raw_requirements.down.sql
+10
View File
@@ -0,0 +1,10 @@
-- Drop indexes first (though dropping table will cascade)
DROP INDEX IF EXISTS workspace_dependencies_id_workspace ;
DROP INDEX IF EXISTS workspace_dependencies_workspace_lang_name_archived_idx ;
DROP INDEX IF EXISTS workspace_dependencies_workspace_archived_idx;
DROP INDEX IF EXISTS one_non_archived_per_null_name_language_constraint;
DROP INDEX IF EXISTS one_non_archived_per_name_language_constraint;
-- Drop table and sequence
DROP TABLE IF EXISTS workspace_dependencies;
DROP SEQUENCE IF EXISTS workspace_dependencies_id_seq CASCADE;
backend/migrations/20251106152104_raw_requirements.up.sql
+23
View File
@@ -0,0 +1,23 @@
CREATE SEQUENCE IF NOT EXISTS workspace_dependencies_id_seq;
CREATE TABLE IF NOT EXISTS workspace_dependencies(
id BIGINT DEFAULT nextval('workspace_dependencies_id_seq') PRIMARY KEY,
name VARCHAR(255), -- If NULL - it's global
content TEXT NOT NULL,
language SCRIPT_LANG NOT NULL,
description text NOT NULL DEFAULT '',
archived BOOLEAN NOT NULL DEFAULT false,
workspace_id character varying(50) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
);
-- Make any query that tries to create non-linear history fail
CREATE UNIQUE INDEX IF NOT EXISTS one_non_archived_per_name_language_constraint ON workspace_dependencies(name, language, workspace_id) WHERE archived = false AND name IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS one_non_archived_per_null_name_language_constraint ON workspace_dependencies(language, workspace_id) WHERE archived = false AND name IS NULL;
-- Performance indexes for common query patterns
-- For the list query (filtering by workspace_id and archived)
CREATE INDEX IF NOT EXISTS workspace_dependencies_workspace_archived_idx ON workspace_dependencies(workspace_id, archived) WHERE archived = false;
CREATE INDEX IF NOT EXISTS workspace_dependencies_workspace_lang_name_archived_idx ON workspace_dependencies(workspace_id, language, name, archived);
CREATE INDEX IF NOT EXISTS workspace_dependencies_id_workspace ON workspace_dependencies(id, workspace_id);
backend/migrations/20251129112655_extend_oauth_refresh_token.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20251129112655_extend_oauth_refresh_token.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TABLE account ALTER COLUMN refresh_token TYPE VARCHAR(10000);
backend/parsers/windmill-parser-py-imports/Cargo.toml
+1
View File
@@ -30,3 +30,4 @@ async-recursion.workspace = true
toml.workspace = true
serde.workspace = true
pep440_rs.workspace = true
tracing.workspace = true
backend/parsers/windmill-parser-py-imports/src/lib.rs
+240 -206
View File
@@ -27,7 +27,11 @@ use rustpython_parser::{
use sqlx::{Pool, Postgres};
use windmill_common::{
error::{self, to_anyhow},
worker::PythonAnnotations,
worker::{
split_python_requirements, try_parse_locked_python_version_from_requirements,
PythonAnnotations,
},
workspace_dependencies::{RawWorkspaceDependencies, WorkspaceDependenciesPrefetched},
};
fn replace_import(x: String) -> String {
@@ -170,7 +174,6 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<NImport>>
// This is needed because we've split off the real main function above
let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string()))
})?;
@@ -259,6 +262,8 @@ pub async fn parse_python_imports(
path: &str,
db: &Pool<Postgres>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
locked_v: &mut Option<pep440_rs::Version>,
raw_workspace_dependencies_o: &Option<RawWorkspaceDependencies>,
) -> error::Result<(Vec<String>, Option<String>)> {
let mut compile_error_hint: Option<String> = None;
let mut imports = parse_python_imports_inner(
@@ -268,8 +273,9 @@ pub async fn parse_python_imports(
db,
&mut vec![],
version_specifiers,
// &mut version_specifier.and_then(|_| Some(path.to_owned())),
&mut None
&mut None,
locked_v,
raw_workspace_dependencies_o,
)
.await?
.into_values()
@@ -323,11 +329,22 @@ async fn parse_python_imports_inner(
already_visited: &mut Vec<String>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
path_where_annotated_pyv: &mut Option<String>,
locked_v: &mut Option<pep440_rs::Version>,
raw_workspace_dependencies_o: &Option<RawWorkspaceDependencies>,
) -> error::Result<HashMap<String, NImportResolved>> {
tracing::debug!("Parsing python imports for path: {}", path);
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
tracing::debug!(
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}",
py310,
py311,
py312,
py313
);
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
if perform {
tracing::debug!("Adding version specifier: {}", unparsed);
pep440_rs::VersionSpecifiers::from_str(unparsed.as_str())
.ok()
.map(|vs| version_specifiers.extend(vs.to_vec()));
@@ -341,10 +358,9 @@ async fn parse_python_imports_inner(
for x in code.lines() {
if x.starts_with("# py:") || x.starts_with("#py:") {
push_version_specifiers(
true,
x.replace('#', "").replace("py:", "").trim().to_owned(),
)?;
let version_spec = x.replace('#', "").replace("py:", "").trim().to_owned();
tracing::debug!("Found inline python version specifier: {}", version_spec);
push_version_specifiers(true, version_spec)?;
} else if !x.starts_with('#') {
break;
}
@@ -369,12 +385,40 @@ async fn parse_python_imports_inner(
// dependencies: Vec<String>,
// }
let find_requirements = code.lines().find_position(|x| {
x.starts_with("#requirements:")
|| x.starts_with("# requirements:")
|| x.starts_with("# /// script")
});
let mut final_imports = HashMap::new();
tracing::debug!("Extracting workspace dependencies for workspace: {}", w_id);
let wdp = WorkspaceDependenciesPrefetched::extract(
code,
windmill_common::scripts::ScriptLang::Python3,
w_id,
raw_workspace_dependencies_o,
path,
db.into(),
)
.await?;
if let Some(c) = wdp.get_python()? {
*locked_v = extract_nimports_from_content(&c, &mut final_imports);
}
if wdp.is_manual() {
tracing::debug!(
"Workspace dependencies mode is Manual, returning {} imports",
final_imports.len()
);
return Ok(final_imports);
}
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("# /// script"));
tracing::debug!(
"Looking for script metadata block, found: {}",
find_requirements.is_some()
);
if let Some((pos, item)) = find_requirements {
tracing::debug!("Found script metadata block at position {}: {}", pos, item);
let mut requirements = HashMap::new();
if item.starts_with("# /// script") {
let mut incorrect = false;
@@ -384,7 +428,7 @@ async fn parse_python_imports_inner(
.map_while(|x| {
incorrect = !x.starts_with('#');
if incorrect || x.starts_with("# ///") {
None
Option::None
} else {
x.get(1..)
}
@@ -392,9 +436,11 @@ async fn parse_python_imports_inner(
.join("\n")
.parse::<toml::Table>()
.map_err(to_anyhow)?;
tracing::debug!("Parsed script metadata: {:?}", metadata);
{
if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) {
tracing::debug!("Found requires-python in metadata: {}", v);
push_version_specifiers(true, v.to_owned())?;
}
};
@@ -403,9 +449,15 @@ async fn parse_python_imports_inner(
.get("dependencies")
.and_then(|dependencies| dependencies.as_array())
.inspect(|list| {
tracing::debug!("Found {} dependencies in script metadata", list.len());
for dependency_v in list.into_iter() {
let requirement = dependency_v.as_str().unwrap_or("ERROR").to_owned();
let key = extract_pkg_name(&requirement);
tracing::debug!(
"Adding dependency from metadata: {} (key: {})",
requirement,
key
);
requirements.insert(
key.clone(),
NImportResolved::Pin {
@@ -418,217 +470,199 @@ async fn parse_python_imports_inner(
);
}
});
} else {
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
let key = extract_pkg_name(&requirement);
requirements.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin {
pkg: requirement.clone(),
path: Default::default(),
}],
key,
},
);
})
})
})
.collect_vec();
}
Ok(requirements)
} else {
let find_extra_requirements = code.lines().find_position(|x| {
x.starts_with("#extra_requirements:") || x.starts_with("# extra_requirements:")
});
let mut imports: HashMap<String, NImportResolved> = HashMap::new();
if let Some((pos, _)) = find_extra_requirements {
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
let key = extract_pkg_name(&requirement);
imports.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin {
pkg: requirement,
path: Default::default(),
}],
key,
},
);
})
})
})
.collect_vec();
}
tracing::debug!(
"Returning {} requirements from script metadata",
requirements.len()
);
return Ok(requirements);
}
// Will get unsorted vector of imports found in current script
let mut nimports = parse_code_for_imports(code, path)?;
tracing::debug!("Found {} imports in code", nimports.len());
// Will get unsorted vector of imports found in current script
let mut nimports = parse_code_for_imports(code, path)?;
// It is important to note, that sorting is important and will always result in this pattern:
// 1. All Repins go first
// 2. All Pins go second
// 3. All Auto go third
// 4. All relative imports go the last
//
// This way we make sure all repins are resolved before (re)pins inside imported relative scripts.
nimports.sort();
tracing::debug!("Processing imports in sorted order");
// It is important to note, that sorting is important and will always result in this pattern:
// 1. All Repins go first
// 2. All Pins go second
// 3. All Auto go third
// 4. All relative imports go the last
//
// This way we make sure all repins are resolved before (re)pins inside imported relative scripts.
nimports.sort();
for n in nimports.into_iter() {
let mut nested = match n {
NImport::Relative(rpath) => {
let code = sqlx::query_scalar!(
r#"
for n in nimports.into_iter() {
let mut nested = match n {
NImport::Relative(rpath) => {
let code = sqlx::query_scalar!(
r#"
SELECT content FROM script WHERE path = $1 AND workspace_id = $2
AND archived = false ORDER BY created_at DESC LIMIT 1
"#,
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string());
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string());
if already_visited.contains(&rpath) {
vec![]
if already_visited.contains(&rpath) {
vec![]
} else {
already_visited.push(rpath.clone());
// Because the algo goes depth first, this function will never return relative import
// This why we can safely assume later, that there is no relative imports
parse_python_imports_inner(
&code,
w_id,
&rpath,
db,
already_visited,
version_specifiers,
path_where_annotated_pyv,
locked_v,
raw_workspace_dependencies_o,
)
.await?
.into_values()
.collect_vec()
}
}
NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }],
NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }],
NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }],
};
// Nested should also be sorted for the same reason
nested.sort();
tracing::debug!("Processing {} nested imports", nested.len());
// At this point there should be no NImport::Relative in `nested`
for imp in nested {
let key = match imp.clone() {
NImportResolved::Pin { key, .. } => key,
NImportResolved::Repin { key, .. } => key,
NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg),
};
tracing::debug!("Resolving import with key: {}", key);
// Handled cases:
//
// 1.
// Error: Imported windmill scripts have different pins
//
// auto
// ├── pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// ├── pin:1
// └── pin:1
//
// Fix 2:
//
// repin:1
// ├── pin:2
// └── pin:1
//
// 2.
// Error: Imported windmill scripts have different pins
//
// pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// └── pin:1
//
// Fix 2:
//
// repin:2
// └── pin:1
//
// 3. repins allowed to be repinned again
//
// repin:2
// └── repin:1
//
match imp.clone() {
NImportResolved::Repin { .. } => {
if let Some(existing_import) = final_imports.get(&key) {
match existing_import {
// replace
p if matches!(
p,
NImportResolved::Pin { .. } | NImportResolved::Auto { .. }
) =>
{
final_imports.insert(key, imp);
}
// do nothing (older repins have greater precedence)
NImportResolved::Repin { .. } => {}
// Should not be possible
_ => {
return Err(anyhow::anyhow!(
"Internal error: cannot resolve requirement pins",
)
.into());
}
}
} else {
already_visited.push(rpath.clone());
// Because the algo goes depth first, this function will never return relative import
// This why we can safely assume later, that there is no relative imports
parse_python_imports_inner(
&code,
w_id,
&rpath,
db,
already_visited,
version_specifiers,
path_where_annotated_pyv,
)
.await?
.into_values()
.collect_vec()
final_imports.insert(key, imp.clone());
}
}
NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }],
NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }],
NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }],
};
// Nested should also be sorted for the same reason
nested.sort();
// At this point there should be no NImport::Relative in `nested`
for imp in nested {
let key = match imp.clone() {
NImportResolved::Pin { key, .. } => key,
NImportResolved::Repin { key, .. } => key,
NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg),
};
// Handled cases:
//
// 1.
// Error: Imported windmill scripts have different pins
//
// auto
// ├── pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// ├── pin:1
// └── pin:1
//
// Fix 2:
//
// repin:1
// ├── pin:2
// └── pin:1
//
// 2.
// Error: Imported windmill scripts have different pins
//
// pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// └── pin:1
//
// Fix 2:
//
// repin:2
// └── pin:1
//
// 3. repins allowed to be repinned again
//
// repin:2
// └── repin:1
//
match imp.clone() {
NImportResolved::Repin { .. } => {
if let Some(existing_import) = imports.get(&key) {
match existing_import {
// replace
p if matches!(
p,
NImportResolved::Pin { .. } | NImportResolved::Auto { .. }
) =>
{
imports.insert(key, imp);
}
// do nothing (older repins have greater precedence)
NImportResolved::Repin { .. } => {}
// Should not be possible
_ => {
return Err(anyhow::anyhow!(
"Internal error: cannot resolve requirement pins",
)
.into());
}
NImportResolved::Pin { pins: new_pins, .. } => {
if let Some(existing_import) = final_imports.get_mut(&key) {
match existing_import {
// Check if pin is the same version, if same, do nothing, if not error
NImportResolved::Pin { pins: existing_pins, .. } => {
existing_pins.extend(new_pins)
}
} else {
imports.insert(key, imp.clone());
}
}
NImportResolved::Pin { pins: new_pins, .. } => {
if let Some(existing_import) = imports.get_mut(&key) {
match existing_import {
// Check if pin is the same version, if same, do nothing, if not error
NImportResolved::Pin { pins: existing_pins, .. } => {
existing_pins.extend(new_pins)
}
// do nothing
NImportResolved::Repin { .. } => {}
// Replace with new pin
NImportResolved::Auto { .. } => {
imports.insert(key, imp);
}
// do nothing
NImportResolved::Repin { .. } => {}
// Replace with new pin
NImportResolved::Auto { .. } => {
final_imports.insert(key, imp);
}
} else {
imports.insert(key, imp.clone());
}
} else {
final_imports.insert(key, imp.clone());
}
NImportResolved::Auto { .. } => {
if !imports.contains_key(&key) {
imports.insert(key, imp);
}
}
NImportResolved::Auto { .. } => {
if !final_imports.contains_key(&key) {
final_imports.insert(key, imp);
}
}
}
}
Ok(imports)
}
tracing::debug!(
"Finished processing imports, returning {} final imports",
final_imports.len()
);
Ok(final_imports)
}
fn extract_nimports_from_content(
content: &str,
hm: &mut HashMap<String, NImportResolved>,
) -> Option<pep440_rs::Version> {
let lines = split_python_requirements(content);
let locked_version = try_parse_locked_python_version_from_requirements(&lines);
for requirement in lines {
let key = extract_pkg_name(&requirement);
hm.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin { pkg: requirement, path: Default::default() }],
key,
},
);
}
locked_version
}
const STDIMPORTS: [&str; 303] = [
backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql
Vendored
+99
View File
@@ -2581,3 +2581,102 @@ import innerdifffolder
'',
'',
'f/foobar/bar', -28028598712388159, 'python3', '');
--
-- Name: workspace_dependencies_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres
--
CREATE SEQUENCE public.workspace_dependencies_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.workspace_dependencies_id_seq OWNER TO postgres;
--
-- Name: workspace_dependencies_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres
--
SELECT pg_catalog.setval('public.workspace_dependencies_id_seq', 7, true);
--
-- Name: SEQUENCE workspace_dependencies_id_seq; Type: ACL; Schema: public; Owner: postgres
--
GRANT ALL ON SEQUENCE public.workspace_dependencies_id_seq TO windmill_user;
GRANT ALL ON SEQUENCE public.workspace_dependencies_id_seq TO windmill_admin;
--
-- Name: workspace_dependencies; Type: TABLE; Schema: public; Owner: postgres
--
CREATE TABLE public.workspace_dependencies (
id bigint DEFAULT nextval('public.workspace_dependencies_id_seq'::regclass) NOT NULL,
name character varying(255),
content text NOT NULL,
language public.script_lang NOT NULL,
description text DEFAULT ''::text NOT NULL,
archived boolean DEFAULT false NOT NULL,
workspace_id character varying(50) NOT NULL,
created_at timestamp with time zone DEFAULT now() NOT NULL
);
ALTER TABLE public.workspace_dependencies OWNER TO postgres;
--
-- Name: workspace_dependencies workspace_dependencies_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
--
ALTER TABLE ONLY public.workspace_dependencies
ADD CONSTRAINT workspace_dependencies_pkey PRIMARY KEY (id);
--
-- Name: one_non_archived_per_name_language_constraint; Type: INDEX; Schema: public; Owner: postgres
--
CREATE UNIQUE INDEX one_non_archived_per_name_language_constraint ON public.workspace_dependencies USING btree (name, language, workspace_id) WHERE ((archived = false) AND (name IS NOT NULL));
--
-- Name: one_non_archived_per_null_name_language_constraint; Type: INDEX; Schema: public; Owner: postgres
--
CREATE UNIQUE INDEX one_non_archived_per_null_name_language_constraint ON public.workspace_dependencies USING btree (language, workspace_id) WHERE ((archived = false) AND (name IS NULL));
--
-- Name: workspace_dependencies_workspace_archived_idx; Type: INDEX; Schema: public; Owner: postgres
--
CREATE INDEX workspace_dependencies_workspace_archived_idx ON public.workspace_dependencies USING btree (workspace_id, archived) WHERE (archived = false);
--
-- Name: workspace_dependencies_workspace_lang_name_idx; Type: INDEX; Schema: public; Owner: postgres
--
CREATE INDEX workspace_dependencies_workspace_lang_name_idx ON public.workspace_dependencies USING btree (workspace_id, language, name);
--
-- Name: TABLE workspace_dependencies; Type: ACL; Schema: public; Owner: postgres
--
GRANT SELECT,INSERT,REFERENCES,DELETE,TRIGGER,TRUNCATE,UPDATE ON TABLE public.workspace_dependencies TO windmill_user;
GRANT SELECT,INSERT,REFERENCES,DELETE,TRIGGER,TRUNCATE,UPDATE ON TABLE public.workspace_dependencies TO windmill_admin;
--
-- PostgreSQL database dump complete
--
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+30 -6
View File
@@ -18,8 +18,16 @@ def main():
pass
";
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut vec![],
&mut None,
&None,
)
.await?;
// println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
@@ -51,8 +59,16 @@ def main():
pass
";
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut vec![],
&mut None,
&None,
)
.await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
@@ -74,8 +90,16 @@ def main():
";
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut vec![],
&mut None,
&None,
)
.await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
backend/src/main.rs
+3 -3
View File
@@ -271,8 +271,8 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
let job_id = Uuid::new_v4();
let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id);
create_dir_all(&job_dir)?;
if let Some(lockfile) = res.lockfile {
let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?;
if let Some(lock) = res.lockfile {
let _ = windmill_worker::prepare_job_dir(&lock, &job_dir).await?;
let envs = windmill_worker::get_common_bun_proc_envs(None).await;
let _ = windmill_worker::install_bun_lockfile(
&mut 0,
@@ -292,7 +292,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
if let Err(e) = windmill_worker::prebundle_bun_script(
&res.content,
Some(&lockfile),
&lock,
&path,
&job_id,
"admins",
backend/tests/relative_imports.rs → backend/tests/debouncing.rs
+3 -485
View File
@@ -1,7 +1,9 @@
// TODO: move all related logic here (if anything left anywhere in codebase)
mod common;
#[cfg(feature = "test_job_debouncing")]
use windmill_api_client::types::NewScript;
#[cfg(feature = "test_job_debouncing")]
fn quick_ns(
content: &str,
language: windmill_api_client::types::ScriptLang,
@@ -43,490 +45,6 @@ fn quick_ns(
}
}
mod dependency_map {
use super::quick_ns;
use sqlx::{Pool, Postgres};
use tokio_stream::StreamExt;
use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer};
async fn init(db: Pool<Postgres>) -> (windmill_api_client::Client, u16, ApiServer) {
init_client(db).await
}
async fn _clear_dmap(db: &Pool<Postgres>) {
sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'")
.execute(db)
.await
.unwrap();
}
/// Corrects map according to provided replacements.
/// Only changes importer_path and/or id
/// Does not affect imported_path nor kind!
fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> {
CORRECT_DMAP
.clone()
.into_iter()
.map(|e| {
let mut r = (
e.0.to_owned(),
e.1.to_owned(),
e.2.to_owned(),
e.3.to_owned(),
);
for (from, to) in &replacements {
r = (
r.0.replace(from, to),
r.1, // Kind should be immutable
r.2, // Imported path should be immutable
// We do not modify script contents in test, so we can assume scripts always import the same path
// Modification of kind or imported path considered to be incorrect.
r.3.replace(from, to),
);
}
r
})
.collect()
}
async fn assert_dmap(
db: &Pool<Postgres>,
importer: Option<String>,
expected: Vec<(
impl Into<String>,
impl Into<String>,
impl Into<String>,
impl Into<String>,
)>,
) {
let dmap = sqlx::query_as::<_, (String, String, String, String)>(
"SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)",
)
.bind(importer)
.fetch_all(db)
.await
.unwrap();
assert_eq!(
dmap,
expected
.into_iter()
.map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into()))
.collect::<Vec<(String, String, String, String)>>()
);
}
lazy_static::lazy_static! {
pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![
("f/rel/branch", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/branch", ""),
("f/rel/root_script", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/leaf_2", ""),
("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"),
("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"),
("f/rel/root_app", "app", "f/rel/branch", "youcanpressme")];
}
// TODO:
// Test that checks that we can run rebuild_dmap multiple times in tests.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_correctness(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
// rebuild map
assert!(super::common::rebuild_dmap(&client).await);
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_lock(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
// Spawn first rebuild
let handle = {
let client = client.clone();
tokio::spawn(async move { super::common::rebuild_dmap(&client).await })
};
// Immidiately spawn another
let res = client
.client()
.post(format!(
"{}/w/test-workspace/workspaces/rebuild_dependency_map",
client.baseurl()
))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
// Should tell us there is already rebuilt in progress
// Or if it is too fast we will be able to trigger it second time
assert!(&res == "There is already one task pending, try again later." || &res == "Success");
assert!(handle.await.unwrap());
Ok(())
}
// If you deploy from cli and you use raw requirements you don't want the script be included in dmap
// Otherwise script will be overwritten once any relative import is updated
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_with_requirements_txt(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script",
Some("# from requirements.txt".to_string()),
Some("000000000005165B".into()),
),
)
.await
.unwrap();
assert_dmap(
&db,
Some("f/rel/root_script".into()),
vec![
("f/rel/root_script", "script", "f/rel/branch", ""),
("f/rel/root_script", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/leaf_2", ""),
],
)
.await;
tokio::time::sleep(std::time::Duration::from_secs(13)).await;
assert_dmap(
&db,
Some("f/rel/root_script".into()),
Vec::<(String, String, String, String)>::new(),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_without_requirements_txt(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script",
// We still want to pass lock to it.
Some("# py311".to_string()),
Some("000000000005165B".into()),
),
)
.await
.unwrap();
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
// tokio::time::sleep(std::time::Duration::from_secs(13)).await;
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
// Consider simple one. Only referenced directly. No deep connections
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_2(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf3';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_2_renamed",
None,
Some("0000000000051659".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
// Consider hard one. Referenced deeply and exists in double references.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_1(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf1';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_1_renamed",
None,
Some("0000000000051658".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_branch(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.leaf_1 import main as lf_1;
def main():
return lf_1();
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/branch_renamed",
None,
Some("000000000005165A".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing branches SHOULD change dependency map
// Though it should only change branch item in dmap when it is importer.
// All entries when branch is imported should not change.
let mut corrected_dmap = CORRECT_DMAP.clone();
// Corresponds to importer path of branch entry
corrected_dmap[0].0 = "f/rel/branch_renamed";
assert_dmap(&db, None, corrected_dmap).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_script(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script_renamed",
None,
Some("000000000005165B".into()),
),
)
.await
.unwrap();
let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(&db, None, corrected_dmap.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value};
let (client, port, _s) = init(db.clone()).await;
let flow = fetch_version(&db, 1443253234253454).await.unwrap();
let res = client
.client()
.post(format!(
"{}/w/test-workspace/flows/update/{}",
client.baseurl(),
"f/rel/root_flow" // encode_path()
))
.json(&NewFlow {
path: "f/rel/root_flow_renamed".into(),
summary: "".into(),
description: None,
value: to_raw_value(
&serde_json::from_str::<serde_json::Value>(
&serde_json::to_string(flow.value())
.unwrap()
.replace("nstep1", "Foxes")
.replace("nstep2_2", "like")
.replace("nstep_4_1", "Emeralds"),
)
.unwrap(),
),
schema: None,
draft_only: None,
tag: None,
dedicated_worker: None,
timeout: None,
deployment_message: None,
visible_to_runner_only: None,
on_behalf_of_email: None,
ws_error_handler_muted: None,
})
.send()
.await
.unwrap();
assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed");
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_flow", "f/rel/root_flow_renamed"),
("nstep1", "Foxes"),
("nstep2_2", "like"),
("nstep_4_1", "Emeralds"),
]),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_app(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
let app_value: String =
sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2")
.fetch_one(&db)
.await
.unwrap()
.unwrap();
// TODO: There is:
// 1. update app
// 2. create app
// 3. update app raw
// Ideally all of them should be handled
let res = client
.client()
.post(format!(
"{}/w/test-workspace/apps/update/{}",
client.baseurl(),
"f/rel/root_app" // encode_path()
))
.json(&windmill_api::EditApp {
path: Some("f/rel/root_app_renamed".into()),
summary: None,
value: serde_json::from_str(
&app_value
.replace("dontpressmeplz", "Apps")
.replace("youcanpressme", "Work"),
)
.unwrap(),
policy: None,
deployment_message: None,
custom_path: None,
})
.send()
.await
.unwrap();
assert_eq!(
res.text().await.unwrap(),
"app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")"
);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_app", "f/rel/root_app_renamed"),
("dontpressmeplz", "Apps"),
("youcanpressme", "Work"),
]),
)
.await;
Ok(())
}
}
#[cfg(feature = "test_job_debouncing")]
mod dependency_job_debouncing {
async fn trigger_djob_for(
backend/tests/dependency_map.rs
+535
View File
@@ -0,0 +1,535 @@
use sqlx::{Pool, Postgres};
use tokio_stream::StreamExt;
use windmill_api_client::types::NewScript;
mod common;
use common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer};
mod dependency_map {
use super::*;
fn quick_ns(
content: &str,
language: windmill_api_client::types::ScriptLang,
path: &str,
lock: Option<String>,
parent_hash: Option<String>,
) -> NewScript {
NewScript {
content: content.into(),
language,
lock,
parent_hash,
path: path.into(),
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: None,
description: "".to_string(),
draft_only: None,
envs: vec![],
is_template: None,
kind: None,
summary: "".to_string(),
tag: None,
schema: std::collections::HashMap::new(),
ws_error_handler_muted: Some(false),
priority: None,
delete_after_use: None,
timeout: None,
restart_unless_cancelled: None,
deployment_message: None,
concurrency_key: None,
visible_to_runner_only: None,
no_main_func: None,
codebase: None,
has_preprocessor: None,
on_behalf_of_email: None,
assets: vec![],
}
}
async fn init(db: Pool<Postgres>) -> (windmill_api_client::Client, u16, ApiServer) {
init_client(db).await
}
async fn _clear_dmap(db: &Pool<Postgres>) {
sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'")
.execute(db)
.await
.unwrap();
}
/// Corrects map according to provided replacements.
/// Only changes importer_path and/or id
/// Does not affect imported_path nor kind!
fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> {
CORRECT_DMAP
.clone()
.into_iter()
.map(|e| {
let mut r = (
e.0.to_owned(),
e.1.to_owned(),
e.2.to_owned(),
e.3.to_owned(),
);
for (from, to) in &replacements {
r = (
r.0.replace(from, to),
r.1, // Kind should be immutable
r.2, // Imported path should be immutable
// We do not modify script contents in test, so we can assume scripts always import the same path
// Modification of kind or imported path considered to be incorrect.
r.3.replace(from, to),
);
}
r
})
.collect()
}
async fn assert_dmap(
db: &Pool<Postgres>,
importer: Option<String>,
expected: Vec<(
impl Into<String>,
impl Into<String>,
impl Into<String>,
impl Into<String>,
)>,
) {
let mut dmap = sqlx::query_as::<_, (String, String, String, String)>(
"SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)",
)
.bind(importer)
.fetch_all(db)
.await
.unwrap();
let mut expected = expected
.into_iter()
.map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into()))
.collect::<Vec<(String, String, String, String)>>();
dmap.sort();
expected.sort();
assert_eq!(dmap, expected);
}
lazy_static::lazy_static! {
pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![
("f/rel/root_script", "script", "dependencies/test.requirements.in", ""),
("f/rel/root_flow", "flow", "dependencies/test.requirements.in", "nstep1"),
("f/rel/root_app", "app", "dependencies/test.requirements.in", "dontpressmeplz"),
("f/rel/branch", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/branch", ""),
("f/rel/root_script", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/leaf_2", ""),
("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"),
("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"),
("f/rel/root_app", "app", "f/rel/branch", "youcanpressme"),
// Default
("f/rel/leaf_1", "script", "dependencies/requirements.in", ""),
("f/rel/leaf_2", "script", "dependencies/requirements.in", ""),
("f/rel/branch", "script", "dependencies/requirements.in", ""),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "failure"),
("f/rel/root_flow", "flow", "dependencies/package.json", "nstep2_1"),
("f/rel/root_flow", "flow", "dependencies/package.json", "nstep3_2"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep2_2"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep3_1"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep4_1"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep5_1"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "preprocessor"),
("f/rel/root_app", "app", "dependencies/requirements.in", "pressmeplz"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "qtool1"),
("f/rel/root_app", "app", "dependencies/requirements.in", "youcanpressme")
];
}
// TODO:
// Test that checks that we can run rebuild_dmap multiple times in tests.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_correctness(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
// rebuild map
assert!(common::rebuild_dmap(&client).await);
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_lock(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
// Spawn first rebuild
let handle = {
let client = client.clone();
tokio::spawn(async move { common::rebuild_dmap(&client).await })
};
// Immidiately spawn another
let res = client
.client()
.post(format!(
"{}/w/test-workspace/workspaces/rebuild_dependency_map",
client.baseurl()
))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
// Should tell us there is already rebuilt in progress
// Or if it is too fast we will be able to trigger it second time
assert!(&res == "There is already one task pending, try again later." || &res == "Success");
assert!(handle.await.unwrap());
Ok(())
}
// If you deploy from cli and you use raw requirements you don't want the script be included in dmap
// Otherwise script will be overwritten once any relative import is updated
// #[cfg(feature = "python")]
// #[sqlx::test(fixtures("base", "dependency_map"))]
// async fn relative_imports_test_with_legacy(db: Pool<Postgres>) -> anyhow::Result<()> {
// let (client, _port, _s) = init(db.clone()).await;
// client
// .create_script(
// "test-workspace",
// &quick_ns(
// "
// from f.rel.branch import main as br;
// from f.rel.leaf_1 import main as lf_1;
// from f.rel.leaf_2 import main as lf_2;
// def main():
// return [br(), lf_1(), lf_2];
// ",
// windmill_api_client::types::ScriptLang::Python3,
// "f/rel/root_script",
// Some("# from requirements.txt".to_string()),
// Some("000000000005165B".into()),
// ),
// )
// .await
// .unwrap();
// assert_dmap(
// &db,
// Some("f/rel/root_script".into()),
// vec![
// ("f/rel/root_script", "script", "f/rel/branch", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_1", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_2", ""),
// ],
// )
// .await;
// tokio::time::sleep(std::time::Duration::from_secs(13)).await;
// assert_dmap(
// &db,
// Some("f/rel/root_script".into()),
// vec![
// ("f/rel/root_script", "script", "f/rel/branch", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_1", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_2", ""),
// ],
// )
// .await;
// Ok(())
// }
// Consider simple one. Only referenced directly. No deep connections
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_2(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf3';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_2_renamed",
None,
Some("0000000000051659".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(
&db,
None,
corrected_dmap(vec![("f/rel/leaf_2", "f/rel/leaf_2_renamed")]),
)
.await;
Ok(())
}
// Consider hard one. Referenced deeply and exists in double references.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_1(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf1';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_1_renamed",
None,
Some("0000000000051658".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(
&db,
None,
corrected_dmap(vec![("f/rel/leaf_1", "f/rel/leaf_1_renamed")]),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_branch(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.leaf_1 import main as lf_1;
def main():
return lf_1();
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/branch_renamed",
None,
Some("000000000005165A".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing branches SHOULD change dependency map
// Though it should only change branch item in dmap when it is importer.
// All entries when branch is imported should not change.
let corrected_dmap = CORRECT_DMAP
.clone()
.iter_mut()
.map(|el| {
if el.0 == "f/rel/branch" {
el.0 = "f/rel/branch_renamed";
}
*el
})
.collect::<Vec<_>>();
assert_dmap(&db, None, corrected_dmap).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_script(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
# requirements: test
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script_renamed",
None,
Some("000000000005165B".into()),
),
)
.await
.unwrap();
let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(&db, None, corrected_dmap.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value};
let (client, port, _s) = init(db.clone()).await;
let flow = fetch_version(&db, 1443253234253454).await.unwrap();
let res = client
.client()
.post(format!(
"{}/w/test-workspace/flows/update/{}",
client.baseurl(),
"f/rel/root_flow" // encode_path()
))
.json(&NewFlow {
path: "f/rel/root_flow_renamed".into(),
summary: "".into(),
description: None,
value: to_raw_value(
&serde_json::from_str::<serde_json::Value>(
&serde_json::to_string(flow.value())
.unwrap()
.replace("nstep1", "Foxes")
.replace("nstep2_2", "like")
.replace("nstep_4_1", "Emeralds"),
)
.unwrap(),
),
schema: None,
draft_only: None,
tag: None,
dedicated_worker: None,
timeout: None,
deployment_message: None,
visible_to_runner_only: None,
on_behalf_of_email: None,
ws_error_handler_muted: None,
})
.send()
.await
.unwrap();
assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed");
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_flow", "f/rel/root_flow_renamed"),
("nstep1", "Foxes"),
("nstep2_2", "like"),
("nstep_4_1", "Emeralds"),
]),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_app(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
let app_value: String =
sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2")
.fetch_one(&db)
.await
.unwrap()
.unwrap();
// TODO: There is:
// 1. update app
// 2. create app
// 3. update app raw
// Ideally all of them should be handled
let res = client
.client()
.post(format!(
"{}/w/test-workspace/apps/update/{}",
client.baseurl(),
"f/rel/root_app" // encode_path()
))
.json(&windmill_api::EditApp {
path: Some("f/rel/root_app_renamed".into()),
summary: None,
value: serde_json::from_str(
&app_value
.replace("dontpressmeplz", "Apps")
.replace("youcanpressme", "Work"),
)
.unwrap(),
policy: None,
deployment_message: None,
custom_path: None,
})
.send()
.await
.unwrap();
assert_eq!(
res.text().await.unwrap(),
"app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")"
);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_app", "f/rel/root_app_renamed"),
("dontpressmeplz", "Apps"),
("youcanpressme", "Work"),
]),
)
.await;
Ok(())
}
}
backend/tests/fixtures/dependency_map.sql
Vendored
+35 -3
View File
File diff suppressed because one or more lines are too long
backend/tests/fixtures/hub_sync_blacklist.sql
Vendored
+15
View File
@@ -0,0 +1,15 @@
-- Fixture for testing hub_sync blacklist from workspace dependencies
-- hub_sync and apps are already built-in via migrations
-- Insert a simple Bun script that should be affected by workspace dependencies
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'admins',
'test-user',
'
export async function main() {
return "Simple bun script";
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'Simple bun script',
'',
'u/admin/simple_bun', 700001, 'bun', '');
backend/tests/fixtures/workspace_dependencies.sql
Vendored
+33
View File
@@ -0,0 +1,33 @@
-- NOTE: Applied after workspace_dependencies_leafs.sql
-- Python script that uses relative imports
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
from f.leafs.python import main as python_leaf
def main():
return {"python_import": python_leaf()}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/python_importer', 500005, 'python3', '');
-- TypeScript script that uses relative imports
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
import { main as tsLeaf } from "./leafs/ts";
export async function main() {
return { ts_import: await tsLeaf() };
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/ts_importer', 500006, 'nativets', '');
-- Dependency map entries
INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/python_importer', 'script', 'f/leafs/python', '');
INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/ts_importer', 'script', 'f/leafs/ts', '');
backend/tests/fixtures/workspace_dependencies_leafs.sql
Vendored
+54
View File
@@ -0,0 +1,54 @@
-- Basic leaf scripts in different languages
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
export async function main() {
return "TypeScript leaf";
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/ts', 500001, 'nativets', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
package main
import "fmt"
func main() {
fmt.Println("Go leaf")
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/go', 500002, 'go', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
def main():
return "Python leaf"',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/python', 500003, 'python3', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
<?php
function main() {
return "PHP leaf";
}
?>',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/php', 500004, 'php', '');
backend/tests/python_jobs.rs
+12 -4
View File
@@ -25,7 +25,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.11.11", "tiny==0.1.3"],
vec!["# workspace-dependencies-mode: manual\n# py: 3.11.11","tiny==0.1.3"],
)
.await?;
Ok(())
@@ -55,7 +55,11 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.11.11", "bottle==0.13.2", "tiny==0.1.2"],
vec![
"# workspace-dependencies-mode: extra\n# py: 3.11.11",
"bottle==0.13.2",
"tiny==0.1.2",
],
)
.await?;
}
@@ -79,7 +83,11 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.11.11", "simplejson==3.20.1", "tiny==0.1.3"],
vec![
"# workspace-dependencies-mode: extra\n# py: 3.11.11",
"simplejson==3.20.1",
"tiny==0.1.3",
],
)
.await?;
Ok(())
@@ -108,7 +116,7 @@ def main():
content,
ScriptLang::Python3,
vec![
"# py: 3.11.11",
"# workspace-dependencies-mode: extra\n# py: 3.11.11",
"bottle==0.13.2",
"microdot==2.2.0",
"simplejson==3.19.3",
backend/tests/workspace_dependencies.rs
+194
View File
@@ -0,0 +1,194 @@
mod common;
mod workspace_dependencies {
use crate::common::in_test_worker;
use crate::common::init_client;
use crate::common::listen_for_completed_jobs;
use sqlx::{Pool, Postgres};
use tokio_stream::StreamExt;
use windmill_common::scripts::ScriptLang;
use windmill_worker::workspace_dependencies::NewWorkspaceDependencies;
mod deps {
pub const REQUIREMENTS_IN: &'static str = "tiny==0.1.3";
// pub const GO_MOD: &'static str = r##"
// module example.com/project
// go 1.20
// require github.com/gin-gonic/gin v1.8.1
// "##;
pub const PACKAGE_JSON: &'static str = r##"
{
"name": "example-project",
"version": "1.0.0",
"dependencies": {
"express": "^4.17.1"
}
}
"##;
pub const COMPOSER_JSON: &'static str = r##"
{
"name": "example/project",
"require": {
"monolog/monolog": "^2.3"
}
}
"##;
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "workspace_dependencies_leafs"))]
#[ignore]
async fn basic_manual_named(db: Pool<Postgres>) -> anyhow::Result<()> {
let ((_client, port, _s), db, mut completed) = (
init_client(db.clone()).await,
&db,
listen_for_completed_jobs(&db).await,
);
for (idx, (l, c)) in [
(ScriptLang::Python3, deps::REQUIREMENTS_IN),
(ScriptLang::Bun, deps::PACKAGE_JSON),
(ScriptLang::Php, deps::COMPOSER_JSON),
// (ScriptLang::Go, deps::GO_MOD),
]
.iter()
.enumerate()
{
let id = NewWorkspaceDependencies {
workspace_id: "test-workspace".into(),
language: *l,
content: (*c).into(),
name: Some("test".to_owned()),
description: None,
}
.create("", "", "", db)
.await
.unwrap();
assert_eq!(idx + 1, id as usize);
}
// Wait for 4 jobs.
// Creating those dependencies will trigger redeployment of all scripts in workspace_dependencies_leafs.sql
in_test_worker(
db,
async {
completed.next().await;
completed.next().await;
completed.next().await;
// completed.next().await;
},
port,
)
.await;
// Verify all scripts have correct locks
// let mut langs = vec![];
// for r in sqlx::query!(
// r#"SELECT language AS "language: ScriptLang",lock FROM script WHERE archived = false"#
// )
// .fetch_all(db)
// .await
// .unwrap()
// {
// match r.language {
// ScriptLang::Python3 => assert_eq!("", &r.lock.unwrap()),
// ScriptLang::Go => todo!(),
// ScriptLang::Bun => todo!(),
// ScriptLang::Bunnative => todo!(),
// ScriptLang::Php => todo!(),
// _ => panic!("Unsupported language"),
// }
// langs.push(r.language);
// }
// langs.sort();
// // Just tiny additional verification for peace of mind.
// assert_eq!(langs.as_slice(), &[]);
Ok(())
}
#[sqlx::test(fixtures("base", "hub_sync_blacklist"))]
async fn hub_sync_blacklist_from_workspace_deps(db: Pool<Postgres>) -> anyhow::Result<()> {
let ((_client, port, _s), db, mut completed) = (
init_client(db.clone()).await,
&db,
listen_for_completed_jobs(&db).await,
);
// Verify built-in fixtures exist
// Check that the setup_app exists
let app_exists =
sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')")
.fetch_one(db)
.await
.unwrap();
assert!(app_exists.unwrap(), "Expected g/all/setup_app to exist");
// Check that hub_sync script exists and is a Bun script
let hub_sync_lang = sqlx::query_scalar!(
r#"SELECT language AS "language: ScriptLang" FROM script WHERE path = 'u/admin/hub_sync'"#
)
.fetch_one(db)
.await
.unwrap();
assert_eq!(
hub_sync_lang,
ScriptLang::Bun,
"Expected hub_sync to be a Bun script"
);
// Create unnamed (default) workspace dependencies for Bun
let _id = NewWorkspaceDependencies {
workspace_id: "admins".into(),
language: ScriptLang::Bun,
content: deps::PACKAGE_JSON.into(),
name: None, // No name = default workspace dependencies
description: None,
}
.create("", "", "", db)
.await
.unwrap();
// Wait for exactly 1 job (only u/admin/simple_bun, not hub_sync)
let job_id = in_test_worker(db, async { completed.next().await }, port)
.await
.expect("Expected one job to complete");
// Query the job's runnable_path
let runnable_path =
sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", job_id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(
runnable_path,
Some("u/admin/simple_bun".to_string()),
"Expected job runnable_path to be 'u/admin/simple_bun' (hub_sync should be blacklisted)"
);
// Assert total job count is 1
let job_count = sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job")
.fetch_one(db)
.await
.unwrap();
assert_eq!(job_count, Some(1), "Expected exactly one job total");
// Assert v2_job_queue is empty
let queue_count = sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue")
.fetch_one(db)
.await
.unwrap();
assert_eq!(queue_count, Some(0), "Expected job queue to be empty");
Ok(())
}
}
backend/windmill-api/openapi.yaml
+249 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.585.1
version: 1.587.1
title: Windmill API
contact:
@@ -2260,6 +2260,56 @@ paths:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/get_dependents/{imported_path}:
get:
summary: get dependents of an imported path
operationId: getDependents
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: imported_path
in: path
required: true
schema:
type: string
description: The imported path to get dependents for
responses:
"200":
description: list of dependents
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/DependencyDependent"
/w/{workspace}/workspaces/get_dependents_amounts:
post:
summary: get dependents amounts for multiple imported paths
operationId: getDependentsAmounts
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: List of imported paths to get dependents counts for
required: true
content:
application/json:
schema:
type: array
items:
type: string
responses:
"200":
description: list of dependents amounts
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/DependentsAmount"
/w/{workspace}/workspaces/get_dependency_map:
get:
@@ -5455,6 +5505,126 @@ paths:
schema:
type: boolean
/w/{workspace}/workspace_dependencies/create:
post:
summary: create workspace dependencies
operationId: createWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: New workspace dependencies
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/NewWorkspaceDependencies"
responses:
"201":
description: workspace dependencies created
content:
text/plain:
schema:
type: string
/w/{workspace}/workspace_dependencies/archive/{language}:
post:
summary: archive workspace dependencies (require admin)
operationId: archiveWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: language
in: path
required: true
schema:
$ref: "#/components/schemas/ScriptLang"
- name: name
in: query
required: false
schema:
type: string
responses:
"200":
description: result
content:
application/json:
schema: {}
/w/{workspace}/workspace_dependencies/delete/{language}:
post:
summary: delete workspace dependencies (require admin)
operationId: deleteWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: language
in: path
required: true
schema:
$ref: "#/components/schemas/ScriptLang"
- name: name
in: query
required: false
schema:
type: string
responses:
"200":
description: result
content:
application/json:
schema: {}
/w/{workspace}/workspace_dependencies/list:
get:
summary: list all workspace dependencies
operationId: listWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: All workspace dependencies
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/WorkspaceDependencies"
/w/{workspace}/workspace_dependencies/get_latest/{language}:
get:
summary: get latest workspace dependencies by language and name
operationId: getLatestWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: language
in: path
required: true
schema:
$ref: "#/components/schemas/ScriptLang"
- name: name
in: query
required: false
schema:
type: string
responses:
"200":
description: Latest workspace dependencies
content:
application/json:
schema:
$ref: "#/components/schemas/WorkspaceDependencies"
/w/{workspace}/scripts/archive/p/{path}:
post:
summary: archive script by path
@@ -15837,6 +16007,52 @@ components:
items:
$ref: "#/components/schemas/Alert"
WorkspaceDependencies:
type: object
properties:
id:
type: integer
archived:
type: boolean
name:
type: string
description:
type: string
content:
type: string
language:
$ref: "#/components/schemas/ScriptLang"
workspace_id:
type: string
created_at:
type: string
format: date-time
required:
- workspace_id
- language
- created_at
- content
- id
- archived
NewWorkspaceDependencies:
type: object
properties:
workspace_id:
type: string
language:
$ref: "#/components/schemas/ScriptLang"
name:
type: string
description:
type: string
content:
type: string
required:
- workspace_id
- language
- content
Script:
type: object
properties:
@@ -19054,6 +19270,38 @@ components:
type: string
nullable: true
DependencyDependent:
type: object
properties:
importer_path:
type: string
importer_kind:
type: string
enum:
- script
- flow
- app
importer_node_ids:
type: array
items:
type: string
nullable: true
required:
- importer_path
- importer_kind
DependentsAmount:
type: object
properties:
imported_path:
type: string
count:
type: integer
format: int64
required:
- imported_path
- count
WorkspaceInvite:
type: object
properties:
backend/windmill-api/src/apps.rs
+10 -6
View File
@@ -174,6 +174,7 @@ pub struct AppWithLastVersion {
pub extra_perms: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub custom_path: Option<String>,
pub raw_app: bool,
}
#[derive(Serialize, FromRow)]
@@ -510,7 +511,7 @@ async fn get_app(
sqlx::query_as::<_, AppWithLastVersionAndStarred>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by, favorite.path IS NOT NULL as starred
app_version.created_at, app_version.created_by, favorite.path IS NOT NULL as starred, app_version.raw_app
FROM app
JOIN app_version
ON app_version.id = app.versions[array_upper(app.versions, 1)]
@@ -530,7 +531,7 @@ async fn get_app(
sqlx::query_as::<_, AppWithLastVersionAndStarred>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by, NULL as starred
app_version.created_at, app_version.created_by, NULL as starred, app_version.raw_app
FROM app, app_version
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
@@ -557,7 +558,7 @@ async fn get_app_lite(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value,
app_version.created_at, app_version.created_by, NULL as starred
app_version.created_at, app_version.created_by, NULL as starred, app_version.raw_app
FROM app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
@@ -596,7 +597,8 @@ async fn get_app_w_draft(
app_version.created_at,
app_version.created_by,
app.draft_only,
draft.value AS "draft"
draft.value AS "draft",
app_version.raw_app
FROM app
INNER JOIN app_version
ON app_version.id = app.versions[array_upper(app.versions, 1)]
@@ -739,7 +741,8 @@ async fn get_app_by_id(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
app_version.created_at, app_version.created_by, app_version.raw_app
FROM app, app_version
WHERE app_version.id = $1 AND app.id = app_version.app_id AND app.workspace_id = $2",
)
.bind(&id)
@@ -766,7 +769,8 @@ async fn get_public_app_by_secret(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
app_version.created_at, app_version.created_by from app, app_version
app_version.created_at, app_version.created_by, app_version.raw_app
FROM app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
backend/windmill-api/src/jobs.rs
+64 -31
View File
@@ -25,6 +25,7 @@ use std::str::FromStr;
use std::time::Instant;
use tokio::io::AsyncReadExt;
use tower::ServiceBuilder;
use url::Url;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
use windmill_common::auth::is_super_admin_email;
use windmill_common::auth::TOKEN_PREFIX_LEN;
@@ -40,6 +41,9 @@ use windmill_common::jobs::{
use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat};
use windmill_common::utils::{RunnableKind, WarnAfterExt};
use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR};
use windmill_common::workspace_dependencies::{
RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES,
};
use windmill_common::DYNAMIC_INPUT_CACHE;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
use windmill_common::{email_oss::send_email_html, server::load_smtp_config};
@@ -6159,14 +6163,17 @@ async fn run_bundle_preview_script(
Ok((StatusCode::CREATED, job_id.unwrap().to_string()))
}
#[derive(Deserialize)]
#[derive(Deserialize, Debug)]
pub struct RunDependenciesRequest {
pub raw_scripts: Vec<RawScriptForDependencies>,
pub entrypoint: String,
#[serde(default)]
pub raw_workspace_dependencies: Option<RawWorkspaceDependencies>,
#[serde(default)]
pub raw_deps: Option<String>,
}
#[derive(Deserialize, Clone)]
#[derive(Deserialize, Clone, Debug)]
pub struct RawScriptForDependencies {
pub script_path: String,
pub raw_code: Option<String>,
@@ -6191,41 +6198,49 @@ async fn run_dependencies_job(
));
}
if req.raw_deps.is_some() {
return Err(error::Error::MigrationNeeded {
feature: "cli is outdated".into(),
version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(),
guide_url: Url::from_str(
"https://www.windmill.dev/docs/core_concepts/workspace_dependencies/migration",
)?,
});
}
// Check if workers support workspace dependencies feature
if req.raw_workspace_dependencies.is_some() {
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
.await?;
}
if req.raw_scripts.len() != 1 || req.raw_scripts[0].script_path != req.entrypoint {
return Err(error::Error::internal_err(
"For now only a single raw script can be passed to this endpoint, and the entrypoint should be set to the script path".to_string(),
));
}
let raw_script = req.raw_scripts[0].clone();
let script_path = raw_script.script_path;
let ehm = HashMap::new();
let raw_code = raw_script.raw_code.unwrap_or_else(|| "".to_string());
let language = raw_script.language;
let (args, raw_code) = if let Some(deps) = req.raw_deps {
let mut hm = HashMap::new();
hm.insert(
"raw_deps".to_string(),
JsonRawValue::from_string("true".to_string()).unwrap(),
);
if language == ScriptLang::Bun {
let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code);
hm.insert(
"npm_mode".to_string(),
JsonRawValue::from_string(annotation.npm.to_string()).unwrap(),
);
}
(PushArgs { extra: Some(hm), args: &ehm }, deps)
} else {
(PushArgs::from(&ehm), raw_code)
};
let RawScriptForDependencies {
// unwrap
script_path,
raw_code,
language,
} = req.raw_scripts[0].clone();
let mut hm = HashMap::new();
req.raw_workspace_dependencies
.map(|v| hm.insert("raw_workspace_dependencies".to_owned(), to_raw_value(&v)));
let (uuid, tx) = push(
&db,
PushIsolationLevel::IsolatedRoot(db.clone()),
&w_id,
JobPayload::RawScriptDependencies { script_path, content: raw_code, language },
args,
JobPayload::RawScriptDependencies {
script_path,
content: raw_code.unwrap_or_default(),
language,
},
PushArgs { extra: Some(hm), args: &HashMap::new() },
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -6261,6 +6276,9 @@ async fn run_dependencies_job(
pub struct RunFlowDependenciesRequest {
pub path: String,
pub flow_value: FlowValue,
#[serde(default)]
pub raw_workspace_dependencies: Option<RawWorkspaceDependencies>,
#[serde(default)]
pub raw_deps: Option<HashMap<String, String>>,
}
@@ -6282,13 +6300,28 @@ async fn run_flow_dependencies_job(
));
}
// Create args HashMap with skip_flow_update and raw_deps if present
if req.raw_deps.is_some() {
return Err(error::Error::MigrationNeeded {
feature: "cli is outdated".into(),
version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(),
guide_url: Url::from_str(
"https://www.windmill.dev/docs/core_concepts/workspace_dependencies/migration",
)?,
});
}
// Check if workers support workspace dependencies feature
if req.raw_workspace_dependencies.is_some() {
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
.await?;
}
// Create args HashMap with skip_flow_update and raw_workspace_dependencies if present
let mut args_map = HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]);
// Add raw_deps to args if present
if let Some(ref raw_deps) = req.raw_deps {
args_map.insert("raw_deps".to_string(), to_raw_value(raw_deps));
}
// Add raw_workspace_dependencies to args if present
req.raw_workspace_dependencies
.map(|v| args_map.insert("raw_workspace_dependencies".to_string(), to_raw_value(&v)));
let (uuid, tx) = push(
&db,
backend/windmill-api/src/lib.rs
+5
View File
@@ -110,6 +110,7 @@ mod openapi;
#[cfg(all(feature = "private", feature = "parquet"))]
pub mod s3_proxy_ee;
mod s3_proxy_oss;
mod workspace_dependencies;
mod approvals;
#[cfg(all(feature = "enterprise", feature = "private"))]
@@ -455,6 +456,10 @@ pub async fn run_server(
.nest("/drafts", drafts::workspaced_service())
.nest("/favorites", favorite::workspaced_service())
.nest("/flows", flows::workspaced_service())
.nest(
"/workspace_dependencies",
workspace_dependencies::workspaced_service(),
)
.nest(
"/flow_conversations",
flow_conversations::workspaced_service(),
backend/windmill-api/src/scripts.rs
+12 -13
View File
@@ -41,11 +41,11 @@ use windmill_audit::ActionKind;
use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap};
use windmill_common::{
assets::{AssetUsageKind, AssetWithAltAccessType, clear_asset_usage, insert_asset_usage},
assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType},
error::to_anyhow,
s3_helpers::upload_artifact_to_store,
scripts::hash_script,
utils::{WarnAfterExt, paginate_without_limits},
utils::{paginate_without_limits, WarnAfterExt},
worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING},
};
@@ -60,9 +60,7 @@ use windmill_common::{
ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred,
},
users::username_to_permissioned_as,
utils::{
not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath,
},
utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath},
worker::to_raw_value,
HUB_BASE_URL,
};
@@ -409,7 +407,7 @@ async fn create_snapshot_script(
if name == "script" {
let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap();
let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar"));
let use_esm = ns.codebase.as_ref().is_some_and(|x| x.contains(".esm"));
let (new_hash, ntx, hdm) = create_script_internal(
ns,
w_id.clone(),
@@ -419,8 +417,14 @@ async fn create_snapshot_script(
webhook.clone(),
)
.await?;
let nh = new_hash.to_string();
script_hash = Some(if is_tar { format!("{nh}.tar") } else { nh });
let mut nh = new_hash.to_string();
if use_esm {
nh = format!("{nh}.esm");
}
if is_tar {
nh = format!("{nh}.tar");
}
script_hash = Some(nh);
tx = Some(ntx);
handle_deployment_metadata = hdm;
}
@@ -1039,14 +1043,10 @@ async fn create_script_internal<'c>(
let permissioned_as2 = permissioned_as.clone();
let script_path2 = script_path.clone();
let parent_path = p_path_opt.clone();
let lock = ns.lock.clone();
let deployment_message = ns.deployment_message.clone();
let content = ns.content.clone();
let language = ns.language.clone();
tokio::spawn(async move {
// TODO: I don't think we want this. We might want to send dependency job. But skip any calculations if lock is already present.
// It will allow us to make code more consistent and predictable.
// wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
if let Err(e) = process_relative_imports(
@@ -1062,7 +1062,6 @@ async fn create_script_internal<'c>(
&authed2.email,
&authed2.username,
&permissioned_as2,
lock,
)
.await
{
backend/windmill-api/src/workspace_dependencies.rs
+145
View File
@@ -0,0 +1,145 @@
use axum::{
extract::{Path, Query},
routing::{get, post},
Extension, Json, Router,
};
use http::StatusCode;
use serde::Deserialize;
use windmill_common::{
error::{self, JsonResult},
scripts::ScriptLang,
users::username_to_permissioned_as,
utils::require_admin,
workspace_dependencies::WorkspaceDependencies,
DB,
};
use windmill_worker::{
scoped_dependency_map, trigger_dependents_to_recompute_dependencies,
workspace_dependencies::NewWorkspaceDependencies,
};
use crate::db::ApiAuthed;
pub fn workspaced_service() -> Router {
Router::new()
.route("/create", post(create))
.route("/list", get(list))
.route("/archive/:language", post(archive))
.route("/get_latest/:language", get(get_latest))
.route("/delete/:language", post(delete))
}
#[axum::debug_handler]
async fn create(
authed: ApiAuthed,
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Json(nwd): Json<NewWorkspaceDependencies>,
) -> error::Result<(StatusCode, String)> {
tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
Ok((
StatusCode::CREATED,
format!(
"{}",
nwd.create(
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
&db
)
.await?
),
))
}
#[axum::debug_handler]
async fn list(
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<WorkspaceDependencies>> {
tracing::info!(workspace_id = %w_id, "list workspace dependencies");
Ok(Json(WorkspaceDependencies::list(&w_id, &db).await?))
}
#[derive(Deserialize)]
pub(super) struct NameQuery {
name: Option<String>,
}
#[axum::debug_handler]
pub(super) async fn get_latest(
Extension(db): Extension<DB>,
Path((w_id, language)): Path<(String, ScriptLang)>,
Query(params): Query<NameQuery>,
) -> JsonResult<Option<WorkspaceDependencies>> {
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "get latest workspace dependencies");
Ok(Json(
WorkspaceDependencies::get_latest(params.name, language, &w_id, db.into()).await?,
))
}
#[axum::debug_handler]
async fn archive(
authed: ApiAuthed,
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, language)): Path<(String, ScriptLang)>,
Query(params): Query<NameQuery>,
) -> error::Result<()> {
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let db = &db;
WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?;
trigger_dependents_to_recompute_dependencies(
&w_id,
scoped_dependency_map::ScopedDependencyMap::get_dependents(
WorkspaceDependencies::to_path(&params.name, language)?.as_str(),
&w_id,
db,
)
.await?,
None,
None,
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
db,
vec![],
)
.await
}
#[axum::debug_handler]
async fn delete(
authed: ApiAuthed,
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, language)): Path<(String, ScriptLang)>,
Query(params): Query<NameQuery>,
) -> error::Result<()> {
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let db = &db;
WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?;
trigger_dependents_to_recompute_dependencies(
&w_id,
scoped_dependency_map::ScopedDependencyMap::get_dependents(
WorkspaceDependencies::to_path(&params.name, language)?.as_str(),
&w_id,
db,
)
.await?,
None,
None,
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
db,
vec![],
)
.await
}
backend/windmill-api/src/workspaces.rs
+100 -3
View File
@@ -53,7 +53,9 @@ use windmill_common::{
utils::{paginate, rd_string, require_admin, Pagination},
};
use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject};
use windmill_worker::scoped_dependency_map::{DependencyMap, ScopedDependencyMap};
use windmill_worker::scoped_dependency_map::{
DependencyDependent, DependencyMap, ScopedDependencyMap,
};
#[cfg(feature = "enterprise")]
use windmill_common::utils::require_admin_or_devops;
@@ -84,6 +86,8 @@ pub fn workspaced_service() -> Router {
.route("/delete_invite", post(delete_invite))
.route("/rebuild_dependency_map", post(rebuild_dependency_map))
.route("/get_dependency_map", get(get_dependency_map))
.route("/get_dependents/*imported_path", get(get_dependents))
.route("/get_dependents_amounts", post(get_dependents_amounts))
.route("/get_settings", get(get_settings))
.route("/get_deploy_to", get(get_deploy_to))
.route("/edit_slack_command", post(edit_slack_command))
@@ -2686,8 +2690,11 @@ async fn clone_workspace_data(
clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?;
// Clone workspace runnable dependencies and dependency map
clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?;
clone_workspace_runnable_dependencies(tx, source_workspace_id, target_workspace_id).await?;
// TODO: Enable when git sync is implemented for workspace dependencies.
// // Clone workspace dependencies
// clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?;
Ok(())
}
@@ -3150,7 +3157,7 @@ async fn clone_raw_apps(
Ok(())
}
async fn clone_workspace_dependencies(
async fn clone_workspace_runnable_dependencies(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
target_workspace_id: &str,
@@ -3182,6 +3189,27 @@ async fn clone_workspace_dependencies(
Ok(())
}
#[allow(dead_code)]
async fn clone_workspace_dependencies(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
target_workspace_id: &str,
) -> Result<()> {
// Clone workspace_runnable_dependencies
sqlx::query!(
"INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at)
SELECT $1, language, name, description, content, archived, created_at
FROM workspace_dependencies
WHERE workspace_id = $2",
target_workspace_id,
source_workspace_id
)
.execute(&mut **tx)
.await?;
Ok(())
}
async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result<String> {
return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string()));
}
@@ -3716,6 +3744,75 @@ async fn rebuild_dependency_map(
ScopedDependencyMap::rebuild_map(&w_id, &db).await
}
#[axum::debug_handler]
async fn get_dependents(
Extension(db): Extension<DB>,
Path((w_id, imported_path)): Path<(String, String)>,
_authed: ApiAuthed,
) -> JsonResult<Vec<DependencyDependent>> {
tracing::debug!(
workspace_id = %w_id,
imported_path = %imported_path,
"API: Getting dependents for imported path"
);
let dependents = ScopedDependencyMap::get_dependents(&imported_path, &w_id, &db).await?;
tracing::debug!(
workspace_id = %w_id,
imported_path = %imported_path,
dependents_count = dependents.len(),
"API: Found dependents: {:?}",
dependents
);
Ok(Json(dependents))
}
#[derive(Serialize, Debug)]
struct DependentsAmount {
imported_path: String,
count: i64,
}
#[axum::debug_handler]
async fn get_dependents_amounts(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(imported_paths): Json<Vec<String>>,
) -> JsonResult<Vec<DependentsAmount>> {
tracing::debug!(
workspace_id = %w_id,
imported_paths = ?imported_paths,
"API: Getting dependents amounts for imported paths"
);
let results = sqlx::query_as!(
DependentsAmount,
r#"
SELECT
imported_path,
COUNT(DISTINCT importer_path) as "count!"
FROM dependency_map
WHERE workspace_id = $1 AND imported_path = ANY($2)
GROUP BY imported_path
"#,
w_id,
&imported_paths
)
.fetch_all(&db)
.await?;
tracing::debug!(
workspace_id = %w_id,
results_count = results.len(),
"API: Found dependents amounts: {:?}",
results
);
Ok(Json(results))
}
#[derive(Deserialize)]
struct ChangeWorkspaceName {
new_name: String,
backend/windmill-api/src/workspaces_export.rs
+44 -3
View File
@@ -44,6 +44,7 @@ use axum::{
use http::HeaderName;
use itertools::Itertools;
use windmill_common::utils::require_admin;
use windmill_common::variables::decrypt;
use windmill_common::{
db::UserDB,
@@ -52,6 +53,7 @@ use windmill_common::{
schedule::Schedule,
scripts::{Schema, Script, ScriptLang},
variables::{build_crypt, ExportableListableVariable},
workspace_dependencies::WorkspaceDependencies,
};
use hyper::header;
@@ -177,6 +179,7 @@ pub(crate) struct ArchiveQueryParams {
include_groups: Option<bool>,
include_settings: Option<bool>,
include_key: Option<bool>,
include_workspace_dependencies: Option<bool>,
default_ts: Option<String>,
}
@@ -211,6 +214,7 @@ where
"error",
"last_server_ping",
"server_id",
"raw_app",
],
ignore_keys.unwrap_or(vec![]),
]
@@ -309,11 +313,20 @@ pub(crate) async fn tarball_workspace(
include_groups,
include_settings,
include_key,
include_workspace_dependencies,
default_ts,
}): Query<ArchiveQueryParams>,
) -> Result<([(HeaderName, String); 2], impl IntoResponse)> {
// require_admin(authed.is_admin, &authed.username)?;
tracing::info!(
"tarball_workspace called for workspace {}: include_workspace_dependencies={:?}, skip_variables={:?}, skip_resources={:?}",
w_id,
include_workspace_dependencies,
skip_variables,
skip_resources
);
let mut tx = user_db.begin(&authed).await?;
let tmp_dir = TempDir::new_in("/tmp/windmill/")?;
@@ -529,8 +542,8 @@ pub(crate) async fn tarball_workspace(
let apps = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)] AND app_version.raw_app IS false
app_version.created_at, app_version.created_by, app_version.raw_app from app, app_version
WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]
AND (app.draft_only IS NULL OR app.draft_only = false)",
)
.bind(&w_id)
@@ -539,12 +552,40 @@ pub(crate) async fn tarball_workspace(
for app in apps {
let app_str = &to_string_without_metadata(&app, false, None).unwrap();
let kind = if app.raw_app { "raw_app" } else { "app" };
archive
.write_to_archive(&app_str, &format!("{}.app.json", app.path))
.write_to_archive(&app_str, &format!("{}.{}.json", app.path, kind))
.await?;
}
}
if include_workspace_dependencies.unwrap_or(false)
&& require_admin(authed.is_admin, &authed.username).is_ok()
{
tracing::info!("Including workspace dependencies in tarball export");
let workspace_dependencies = WorkspaceDependencies::list(&w_id, &db).await?;
tracing::info!(
"Found {} workspace dependencies",
workspace_dependencies.len()
);
for dep in workspace_dependencies {
// let dep_str = &to_string_without_metadata(&dep, false, None).unwrap();
let filename = WorkspaceDependencies::to_path(&dep.name, dep.language)?;
tracing::info!(
"Adding workspace dependency: name={:?}, language={:?}, filename={}",
dep.name,
dep.language,
filename
);
archive.write_to_archive(&dep.content, &filename).await?;
}
} else {
tracing::info!(
"Skipping workspace dependencies: include_workspace_dependencies={:?}",
include_workspace_dependencies
);
}
if include_schedules.unwrap_or(false) {
let schedules = sqlx::query_as::<_, Schedule>(
"SELECT * FROM schedule
backend/windmill-common/Cargo.toml
+3
View File
@@ -33,6 +33,7 @@ thiserror.workspace = true
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_yml.workspace = true
chrono.workspace = true
chrono-tz.workspace = true
hex.workspace = true
@@ -69,6 +70,7 @@ aws-smithy-types.workspace = true
base64.workspace = true
bitflags.workspace = true
once_cell.workspace = true
phf.workspace = true
aws-smithy-types-convert = { workspace = true, optional = true }
aws-sdk-rds = { workspace = true, optional = true }
@@ -92,6 +94,7 @@ strum_macros.workspace = true
url.workspace = true
urlencoding.workspace = true
async-recursion.workspace = true
pep440_rs.workspace = true
semver.workspace = true
croner = "2.2.0"
backend/windmill-common/src/cache.rs
+9 -6
View File
@@ -631,7 +631,8 @@ pub mod script {
envs AS \"envs: Vec<String>\", \
schema AS \"schema: String\", \
schema_validation AS \"schema_validation: bool\", \
codebase LIKE '%.tar' as use_tar \
codebase LIKE '%.tar' as use_tar, \
codebase LIKE '%.esm%' as is_esm \
FROM script WHERE hash = $1 LIMIT 1",
hash.0
)
@@ -647,12 +648,14 @@ pub mod script {
language: r.language,
envs: r.envs,
codebase: if let Some(use_tar) = r.use_tar {
let sh = hash.to_string();
if use_tar {
Some(format!("{sh}.tar"))
} else {
Some(sh)
let mut sh = hash.to_string();
if r.is_esm.unwrap_or(false) {
sh = format!("{sh}.esm");
}
if use_tar {
sh = format!("{sh}.tar");
}
Some(sh)
} else {
None
},
backend/windmill-common/src/error.rs
+6
View File
@@ -86,6 +86,12 @@ pub enum Error {
Generic(StatusCode, String),
#[error("{feature} is unavailable due to some workers being behind. Do not use the feature or make sure all workers run at least {min_version}")]
WorkersAreBehind { feature: String, min_version: String },
#[error(
"Breaking change was introduced in v{version} ({feature}). Follow this migration guide: {guide_url}"
)]
MigrationNeeded { version: String, feature: String, guide_url: url::Url },
#[error("{0} is unavailable. It is possible for this worker to be behind.")]
FeatureUnavailable(String),
}
impl Error {
backend/windmill-common/src/flows.rs
+7 -1
View File
@@ -246,7 +246,13 @@ impl FlowValue {
}
}
}
BranchOne { branches, .. } | BranchAll { branches, .. } => {
BranchOne { default, branches, .. } => {
Self::traverse_leafs(default.iter().collect(), cb)?;
for branch in branches {
Self::traverse_leafs(branch.modules.iter().collect(), cb)?;
}
}
BranchAll { branches, .. } => {
for branch in branches {
Self::traverse_leafs(branch.modules.iter().collect(), cb)?;
}
backend/windmill-common/src/global_settings.rs
+15
View File
@@ -133,3 +133,18 @@ pub async fn load_value_from_global_settings(
.map(|x| x.value);
Ok(r)
}
pub async fn set_value_in_global_settings(
db: &Pool<Postgres>,
setting_name: &str,
value: serde_json::Value,
) -> error::Result<()> {
sqlx::query!(
"INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
setting_name,
value
)
.execute(db)
.await?;
Ok(())
}
backend/windmill-common/src/jobs.rs
+1 -1
View File
@@ -414,7 +414,7 @@ pub enum JobPayload {
/// Dependency Job, exposed with API. Requirements can be predefined
RawScriptDependencies {
script_path: String,
/// Will reflect raw requirements content (e.g. requirements.txt)
/// Will reflect raw requirements content (e.g. requirements.in)
content: String,
language: ScriptLang,
},
backend/windmill-common/src/lib.rs
+1 -1
View File
@@ -55,7 +55,7 @@ pub mod job_metrics;
pub mod job_s3_helpers_ee;
#[cfg(feature = "parquet")]
pub mod job_s3_helpers_oss;
pub mod lockfiles;
pub mod workspace_dependencies;
#[cfg(feature = "private")]
pub mod git_sync_ee;
backend/windmill-common/src/schema.rs
+1 -9
View File
@@ -345,15 +345,7 @@ fn find_annotation(comm_lit: &str, annotation: &str, code: &str) -> bool {
pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
let annotation = "schema_validation";
use ScriptLang::*;
let comment = match lang {
Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
Rust => "//!",
// for related places search: ADD_NEW_LANG
};
find_annotation(comment, annotation, code)
find_annotation(&lang.as_comment_lit(), annotation, code)
}
#[derive(Serialize, Deserialize, Debug, Clone)]
backend/windmill-common/src/scripts.rs
+83 -1
View File
@@ -17,6 +17,7 @@ use crate::{
assets::AssetWithAltAccessType,
error::{to_anyhow, Error},
utils::http_get_from_hub,
workspace_dependencies::WorkspaceDependenciesAnnotatedRefs,
DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION,
};
@@ -25,12 +26,26 @@ use anyhow::Context;
use backon::ConstantBuilder;
use backon::{BackoffBuilder, Retryable};
use itertools::Itertools;
use regex::Regex;
use serde::de::Error as _;
use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
use crate::utils::StripPath;
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, Default)]
#[derive(
Serialize,
Deserialize,
Debug,
PartialEq,
Copy,
Clone,
Hash,
Eq,
sqlx::Type,
Default,
Ord,
PartialOrd,
)]
#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum ScriptLang {
@@ -90,6 +105,71 @@ impl ScriptLang {
// for related places search: ADD_NEW_LANG
}
}
pub fn as_dependencies_filename(&self) -> Option<String> {
use ScriptLang::*;
Some(
match self {
Bun | Bunnative => "package.json",
Python3 => "requirements.in",
// Go => "go.mod",
Php => "composer.json",
_ => return None,
}
.to_owned(),
)
}
pub fn as_comment_lit(&self) -> String {
use ScriptLang::*;
match self {
Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//",
Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
Rust => "//!",
// for related places search: ADD_NEW_LANG
}
.to_owned()
}
pub fn extract_workspace_dependencies_annotated_refs(
&self,
code: &str,
runnable_path: &str,
) -> Option<WorkspaceDependenciesAnnotatedRefs<String>> {
use ScriptLang::*;
lazy_static::lazy_static! {
static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap();
}
match self {
// TODO: Maybe use regex
Bun | Bunnative => WorkspaceDependenciesAnnotatedRefs::parse(
"//",
"package_json",
code,
None,
runnable_path,
),
Python3 => WorkspaceDependenciesAnnotatedRefs::parse(
"#",
"requirements",
code,
Some(&RE_PYTHON),
runnable_path,
),
Go => {
WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path)
}
Php => WorkspaceDependenciesAnnotatedRefs::parse(
"//",
"composer_json",
code,
None,
runnable_path,
),
_ => return None,
}
}
}
impl FromStr for ScriptLang {
@@ -739,12 +819,14 @@ pub struct ClonedScript {
pub old_script: NewScript,
pub new_hash: i64,
}
// TODO: What if dependency job fails, there is script with NULL in the lock
pub async fn clone_script<'c>(
base_hash: ScriptHash,
w_id: &str,
deployment_message: Option<String>,
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
) -> crate::error::Result<ClonedScript> {
// TODO:!
let s = sqlx::query_as::<_, Script>(
"SELECT * FROM script WHERE hash = $1 AND workspace_id = $2 AND archived = false FOR UPDATE",
)
backend/windmill-common/src/worker.rs
+121 -1
View File
@@ -264,6 +264,10 @@ lazy_static::lazy_static! {
.unwrap_or(false);
pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
/// Global flag indicating if all workers support workspace dependencies feature (>= 1.583.0)
/// This flag is updated during worker initialization by checking the minimum version across all workers
/// When false, creation of workspace dependencies is forbidden and extraction of external workspace dependencies will error
pub static ref MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
/// Global flag indicating if all workers support the debouncing feature (>= 1.566.0)
/// Debouncing consolidates multiple dependency job requests within a time window to avoid redundant work
/// This flag is updated during worker initialization by checking the minimum version across all workers
@@ -505,7 +509,10 @@ pub const ROOT_CACHE_DIR: &str = concatcp!(TMP_DIR, "/cache/");
pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result<File> {
let path = format!("{}/{}", dir, path);
let mut file = File::create(&path)?;
let mut file = File::create(&path).map_err(|e| {
tracing::error!("Failed to create file at {path}: {:?}", &e);
e
})?;
file.write_all(content.as_bytes())?;
file.flush()?;
Ok(file)
@@ -1264,6 +1271,10 @@ pub async fn update_min_version(conn: &Connection) -> bool {
tracing::info!("Minimal worker version: {min_version}");
}
// Workspace dependencies feature requires minimum version across all workers
*MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES.write().await = min_version
>= Version::parse(crate::workspace_dependencies::MIN_VERSION_WORKSPACE_DEPENDENCIES)
.unwrap();
// Debouncing feature requires minimum version 1.566.0 across all workers
// This ensures all workers can handle debounce keys and stale data accumulation
*MIN_VERSION_SUPPORTS_DEBOUNCING.write().await = min_version >= Version::new(1, 566, 0);
@@ -1949,6 +1960,115 @@ pub fn to_raw_value_owned(result: serde_json::Value) -> Box<RawValue> {
.unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap())
}
pub fn split_python_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
requirements
.as_ref()
.lines()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.map(String::from)
.collect()
}
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
#[repr(u32)]
pub enum PyVAlias {
Py310 = 10,
#[default]
Py311,
Py312,
Py313,
}
impl Into<pep440_rs::Version> for PyVAlias {
fn into(self) -> pep440_rs::Version {
pep440_rs::Version::new([self.major() as u64, self as u64])
}
}
impl Into<u32> for PyVAlias {
fn into(self) -> u32 {
self.major() * 100 + self as u32
}
}
impl PyVAlias {
pub fn all<T: From<PyVAlias>>() -> Vec<T> {
use PyVAlias::*;
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
}
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
fn major(&self) -> u32 {
use PyVAlias::*;
match self {
Py310 | Py311 | Py312 | Py313 => 3,
// Py400 | Py401 => 4
}
}
/// Converts numeric format to alias
/// Example:
/// 310u32 (in) -> PyVAlias::Py310 (out)
pub fn try_from_v1<T: ToString>(numeric: T) -> Option<Self> {
use PyVAlias::*;
match numeric.to_string().as_str() {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
_ => None,
}
}
}
/// Parse lockfile for assigned python version.
/// If not found returns None
pub fn try_parse_locked_python_version_from_requirements<S: AsRef<str>>(
requirements_lines: &[S],
) -> Option<pep440_rs::Version> {
let parse_version = |s: &str| -> Option<pep440_rs::Version> {
// Possible inputs:
// V2:
// # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0
//
// V1:
// # py311 or #py311
let version_unparsed = s
.to_owned()
// Remove whitespaces. That leaves us with:
// V2: #py:3.11.0
// V1: #py311
//
// Remove #
// V2: py:3.11.0
// V1: py311
//
// Remove :
// V2: py3.11.0
// V1: py311
.replace([' ', '#', ':'], "")
// Remove "py"
// V2: 3.11.0
// V1: 311
.replace("py", "");
// We will support reading V1 syntax, but it will be overwritten next deploy
PyVAlias::try_from_v1(&version_unparsed)
.map(PyVAlias::into)
.or(pep440_rs::Version::from_str(&version_unparsed)
.ok()
.map(pep440_rs::Version::into))
};
requirements_lines
.iter()
.find(|s| {
let s = s.as_ref();
s.starts_with("#py") || s.starts_with("# py")
})
.map(S::as_ref)
.and_then(parse_version)
}
#[cfg(test)]
mod tests {
use super::*;
backend/windmill-common/src/workspace_dependencies.rs
+1193
View File
File diff suppressed because it is too large Load Diff
backend/windmill-macros/Cargo.toml
+7
View File
@@ -11,6 +11,9 @@ proc-macro = true
proc-macro2.workspace = true
quote.workspace = true
syn.workspace = true
serde.workspace = true
serde_yml.workspace = true
serde_derive.workspace = true
# Dependencies for tests
[dev-dependencies]
@@ -18,3 +21,7 @@ syn.workspace = true
lazy_static.workspace = true
itertools.workspace = true
regex.workspace = true
serde.workspace = true
serde_yml.workspace = true
serde_derive.workspace = true
pep440_rs.workspace = true
backend/windmill-macros/tests/annotations.rs
+96
View File
@@ -3,6 +3,7 @@ mod annotations_tests {
extern crate windmill_macros;
use itertools::Itertools;
// use pep440_rs::Version;
use windmill_macros::annotations;
// Previous implementation.
@@ -166,4 +167,99 @@ mod annotations_tests {
assert_eq!(expected, Annotations::parse(cont));
}
}
// // #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// // #[annotations("#")]
// // pub struct SerAnnotations {
// // pub ann1: bool,
// // pub stt: String,
// // }
// #[test]
// fn non_bool_1() {
// let cont = r#"#ann1, stt: "hey""#;
// // non-bool take entire line, so you can't have one normal and than parsed.
// let a = SerAnnotations { ann1: false, stt: "".to_owned() };
// assert_eq!(a, SerAnnotations::parse(cont));
// }
// #[test]
// fn non_bool_2() {
// let cont = "#ann1, \n#stt: hey";
// let a = SerAnnotations { ann1: true, stt: "hey".to_owned() };
// assert_eq!(a, SerAnnotations::parse(cont));
// }
// #[test]
// fn non_bool_different_idents() {
// #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// #[annotations("#")]
// pub struct A {
// pub s: String,
// }
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s:hey"));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s : hey"));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s :hey"));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s : hey "));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("# s : hey "));
// assert_eq!(A { s: "".to_owned() }, A::parse(" # s : hey "));
// }
// #[test]
// fn non_bool_unparseable_last() {
// #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// #[annotations("#")]
// pub struct A {
// pub v: i32,
// }
// let cont = "#v: 1\n#v: non int";
// let a = A {
// v: 1, // Should still be first, second unparsable one should have no affect on existing values
// };
// assert_eq!(a, A::parse(cont));
// }
// #[test]
// fn non_bool_different_types() {
// #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// #[annotations("#")]
// pub struct A {
// pub s: String,
// pub i: i32,
// pub o: Option<String>,
// pub a: Vec<i32>,
// pub v: Option<pep440_rs::Version>, // Custom deser
// pub e: E,
// }
// #[derive(
// serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq, Clone, Default, Debug,
// )]
// pub enum E {
// #[default]
// One,
// Two(String),
// Three,
// }
// assert_eq!(
// A {
// s: "foo".to_owned(),
// i: 33,
// o: None,
// a: vec![1, 2, 3],
// v: Some(Version::new([1, 0, 0])),
// e: E::Three
// },
// A::parse(
// "#
// #s: foo
// #i: 33
// #o:
// #a: [1, 2, 3]
// #v: 1.0.0
// #e: Three
// "
// )
// );
// }
}
backend/windmill-queue/src/jobs.rs
+2 -9
View File
@@ -38,7 +38,6 @@ use windmill_common::add_time;
use windmill_common::auth::JobPerms;
#[cfg(feature = "benchmark")]
use windmill_common::bench::BenchmarkIter;
use windmill_common::lockfiles::is_generated_from_raw_requirements;
use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL};
use windmill_common::utils::{configure_client, now_from_db};
use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY};
@@ -2659,13 +2658,6 @@ impl PulledJobResult {
)
.await?;
if is_generated_from_raw_requirements(&Some(cloned_script.old_script.language), &cloned_script.old_script.lock.map(|v| v.to_string())) {
return Err(Error::BadRequest(format!(
"Script at path {} is generated from raw requirements, not overriding",
pulled_job.runnable_path()
)));
}
cloned_script.new_hash
}
JobKind::FlowDependencies => {
@@ -3632,7 +3624,7 @@ pub enum PushIsolationLevel<'c> {
}
impl<'c> PushIsolationLevel<'c> {
async fn into_tx(self) -> error::Result<Transaction<'c, Postgres>> {
pub async fn into_tx(self) -> error::Result<Transaction<'c, Postgres>> {
match self {
PushIsolationLevel::Isolated(db, authed) => Ok((db.begin(&authed).await?).into()),
PushIsolationLevel::IsolatedRoot(db) => Ok(db.begin().await?),
@@ -5867,3 +5859,4 @@ pub async fn get_same_worker_job(
))
})
}
backend/windmill-worker/src/ansible_executor.rs
+19 -13
View File
@@ -13,10 +13,10 @@ use tokio::process::Command;
use uuid::Uuid;
use windmill_common::{
error,
git_sync_oss::{prepend_token_to_github_url},
git_sync_oss::prepend_token_to_github_url,
worker::{
is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location,
Connection, WORKER_CONFIG,
is_allowed_file_location, split_python_requirements, to_raw_value, write_file,
write_file_at_user_defined_location, Connection, PyVAlias, WORKER_CONFIG,
},
};
use windmill_queue::MiniPulledJob;
@@ -34,8 +34,8 @@ use crate::{
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, PY_INSTALL_DIR, PyVAlias, TZ_ENV,
DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
PY_INSTALL_DIR, TZ_ENV,
};
use windmill_common::client::AuthedClient;
@@ -399,7 +399,7 @@ async fn handle_ansible_python_deps(
if requirements.len() > 0 {
let mut venv_path = handle_python_reqs(
crate::python_executor::split_requirements(requirements),
split_python_requirements(requirements),
job_id,
w_id,
mem_peak,
@@ -958,7 +958,11 @@ pub async fn handle_ansible_job(
#[cfg(feature = "enterprise")]
if is_github_app {
if let Connection::Sql(db) = conn {
let token = windmill_common::git_sync_oss::get_github_app_token_internal(db, &client.token).await?;
let token = windmill_common::git_sync_oss::get_github_app_token_internal(
db,
&client.token,
)
.await?;
secret_url = prepend_token_to_github_url(&secret_url, &token)?;
} else {
return Err(windmill_common::error::Error::BadRequest("Github App authentication is currently unavailable for agent workers. Contact the windmill team to request this feature".to_string()));
@@ -970,8 +974,12 @@ pub async fn handle_ansible_job(
let target_path = "delegate_git_repository".to_string();
let repo =
GitRepo { url: secret_url, commit: delegated_git_repo.commit.clone(), branch, target_path };
let repo = GitRepo {
url: secret_url,
commit: delegated_git_repo.commit.clone(),
branch,
target_path,
};
append_logs(
&job.id,
&job.workspace_id,
@@ -1241,10 +1249,8 @@ fi
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await?
} else {
let ansible_args: Vec<&str> = cmd_args.iter().map(|s| s.as_ref()).collect();
let mut ansible_cmd = build_command_with_isolation(
ANSIBLE_PLAYBOOK_PATH.as_str(),
&ansible_args,
);
let mut ansible_cmd =
build_command_with_isolation(ANSIBLE_PLAYBOOK_PATH.as_str(), &ansible_args);
ansible_cmd
.current_dir(job_dir)
.env_clear()
backend/windmill-worker/src/bun_executor.rs
+100 -91
View File
@@ -14,9 +14,9 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInf
use crate::{
common::{
create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file,
build_command_with_isolation, read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics,
StreamNotifier,
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
parse_npm_config, read_file, read_file_content, read_result, start_child_process,
write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier,
},
handle_child::handle_child,
BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH,
@@ -27,6 +27,7 @@ use windmill_common::{
client::AuthedClient,
s3_helpers::BundleFormat,
scripts::{id_to_codebase_info, CodebaseInfo},
workspace_dependencies::WorkspaceDependenciesPrefetched,
};
#[cfg(windows)]
@@ -40,7 +41,7 @@ use windmill_common::{
error::{self, Result},
get_latest_hash_for_path,
scripts::ScriptLang,
worker::{exists_in_cache, save_cache, to_raw_value, write_file, Connection, DISABLE_BUNDLING},
worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING},
DB,
};
@@ -101,7 +102,7 @@ pub async fn gen_bun_lockfile(
base_internal_url: &str,
worker_name: &str,
export_pkg: bool,
raw_deps: Option<String>,
workspace_dependencies: &WorkspaceDependenciesPrefetched,
npm_mode: bool,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<Option<String>> {
@@ -109,11 +110,11 @@ pub async fn gen_bun_lockfile(
let mut empty_deps = false;
if let Some(raw_deps) = raw_deps.as_ref() {
if let Some(package_json_content) = workspace_dependencies.get_bun()? {
gen_bunfig(job_dir).await?;
write_file(job_dir, "package.json", raw_deps.as_str())?;
write_file(job_dir, "package.json", package_json_content.as_str())?;
} else {
let _ = write_file(
write_file(
&job_dir,
"build.js",
&format!(
@@ -205,16 +206,7 @@ pub async fn gen_bun_lockfile(
let mut file = File::open(format!("{job_dir}/package.json")).await?;
let mut buf = String::default();
file.read_to_string(&mut buf).await?;
if raw_deps.is_some() {
let mut json_map: HashMap<String, Box<RawValue>> = serde_json::from_str(&buf)?;
json_map.insert(
"generatedFromPackageJson".to_string(),
to_raw_value(&"true".to_string()),
);
content = serde_json::to_string_pretty(&json_map)?;
} else {
content = buf;
}
content = buf;
}
if !npm_mode {
#[cfg(any(target_os = "linux", target_os = "macos"))]
@@ -612,13 +604,15 @@ struct PulledCodebase {
}
async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<PulledCodebase> {
let path = windmill_common::s3_helpers::bundle(&w_id, &id);
let CodebaseInfo { is_tar, is_esm } = id_to_codebase_info(id);
let bun_cache_path = format!(
"{}/{}",
"{}/{}.{}",
windmill_common::worker::ROOT_CACHE_NOMOUNT_DIR,
path
path,
if is_tar { "tar" } else { "js" }
);
let CodebaseInfo { is_tar, is_esm } = id_to_codebase_info(id);
let dst = format!(
"{job_dir}/{}",
if is_tar { "codebase.tar" } else { "main.js" }
@@ -639,9 +633,10 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<PulledCode
&& object_store.is_none()
{
let bun_cache_path = format!(
"{}/{}",
"{}/{}.{}",
*windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR,
path
path,
if is_tar { "tar" } else { "js" }
);
if std::fs::metadata(&bun_cache_path).is_ok() {
tracing::info!("loading {bun_cache_path} from standalone bundle cache");
@@ -700,7 +695,7 @@ fn extract_saved_codebase(
pub async fn prebundle_bun_script(
inner_content: &str,
lockfile: Option<&String>,
lock: &str,
script_path: &str,
job_id: &Uuid,
w_id: &str,
@@ -712,7 +707,7 @@ pub async fn prebundle_bun_script(
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<()> {
let (local_path, remote_path) =
compute_bundle_local_and_remote_path(inner_content, lockfile, script_path, db, w_id).await;
compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await;
if exists_in_cache(&local_path, &remote_path).await {
return Ok(());
}
@@ -776,16 +771,12 @@ async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) ->
pub async fn compute_bundle_local_and_remote_path(
inner_content: &str,
requirements_o: Option<&String>,
lock: &str,
script_path: &str,
db: Option<&DB>,
w_id: &str,
) -> (String, String) {
let mut input_src = format!(
"{}{}",
inner_content,
requirements_o.as_ref().map(|x| x.as_str()).unwrap_or("")
);
let mut input_src = format!("{inner_content}{lock}",);
if let Some(db) = db {
let relative_imports = crate::worker_lockfiles::extract_relative_imports(
@@ -841,7 +832,7 @@ async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> R
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_bun_job(
requirements_o: Option<&String>,
maybe_lock: MaybeLock,
codebase: Option<&String>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -862,16 +853,15 @@ pub async fn handle_bun_job(
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some()
&& !annotation.nobundling
&& !*DISABLE_BUNDLING
&& codebase.is_none()
{
let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = (
maybe_lock.get_lock(),
!annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(),
) {
let (local_path, remote_path) = match conn {
Connection::Sql(db) => {
compute_bundle_local_and_remote_path(
inner_content,
requirements_o,
lock,
job.runnable_path(),
Some(db),
&job.workspace_id,
@@ -937,59 +927,70 @@ pub async fn handle_bun_job(
if pulled_codebase.is_esm {
format = BundleFormat::Esm;
}
} else if let Some(reqs) = requirements_o.as_ref() {
let (pkg, lock, empty, is_binary) = split_lockfile(reqs);
if lock.is_none() && !annotation.npm {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{reqs}|", if is_binary {"b"} else {""})
));
}
let _ = write_file(job_dir, "package.json", pkg)?;
let lock = if annotation.npm { "" } else { lock.unwrap() };
if !empty {
if !annotation.npm {
let _ = write_lock(lock, job_dir, is_binary).await?;
}
install_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
job_dir,
worker_name,
common_bun_proc_envs.clone(),
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
}
} else {
// if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
let _ = gen_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
&client.token,
job.runnable_path(),
job_dir,
base_internal_url,
worker_name,
false,
None,
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
match &maybe_lock {
MaybeLock::Resolved { lock } => {
let (package_json, bun_lock, empty, is_binary) = split_lockfile(lock);
// }
if bun_lock.is_none() && !annotation.npm {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{lock}|", if is_binary {"b"} else {""})
));
}
write_file(job_dir, "package.json", package_json)?;
let bun_lock = if annotation.npm {
""
} else {
bun_lock.unwrap()
};
if !empty {
if !annotation.npm {
write_lock(bun_lock, job_dir, is_binary).await?;
}
install_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
job_dir,
worker_name,
common_bun_proc_envs.clone(),
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
}
}
MaybeLock::Unresolved { ref workspace_dependencies } => {
// if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
gen_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
&client.token,
job.runnable_path(),
job_dir,
base_internal_url,
worker_name,
false,
workspace_dependencies,
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
// }
}
}
}
if codebase.is_some() && format == BundleFormat::Cjs {
@@ -1181,7 +1182,7 @@ try {{
&& !annotation.nobundling
&& !*DISABLE_BUNDLING
&& !codebase.is_some()
&& (requirements_o.is_some() || annotation.native);
&& (maybe_lock.get_lock().is_some() || annotation.native);
let write_loader_f = async {
if build_cache {
@@ -1686,7 +1687,15 @@ pub async fn start_worker(
base_internal_url,
worker_name,
false,
None,
&WorkspaceDependenciesPrefetched::extract(
inner_content,
ScriptLang::Bun,
w_id,
&None,
&script_path,
db.into(),
)
.await?,
annotation.npm,
&mut None,
)
backend/windmill-worker/src/common.rs
+33
View File
@@ -24,6 +24,7 @@ use windmill_common::worker::{
to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType,
CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
};
use windmill_common::workspace_dependencies::WorkspaceDependenciesPrefetched;
use windmill_common::{
cache::{Cache, RawData},
error::{self, Error},
@@ -1368,3 +1369,35 @@ pub fn s3_mode_args_to_worker_data(
workspace_id: job.workspace_id.clone(),
}
}
#[derive(Debug)]
pub enum MaybeLock {
/// Deployed Scripts
Resolved { lock: String },
/// Previews
Unresolved { workspace_dependencies: WorkspaceDependenciesPrefetched },
}
impl MaybeLock {
pub fn map_unresolved<B, F>(&self, mut f: F) -> Option<B>
where
Self: Sized,
F: FnMut(&WorkspaceDependenciesPrefetched) -> B,
{
self.get_workspace_dependencies().map(|wd| f(wd))
}
pub fn get_workspace_dependencies(&self) -> Option<&WorkspaceDependenciesPrefetched> {
match self {
MaybeLock::Resolved { .. } => None,
MaybeLock::Unresolved { ref workspace_dependencies } => Some(workspace_dependencies),
}
}
pub fn get_lock(&self) -> Option<&String> {
match self {
MaybeLock::Resolved { ref lock } => Some(lock),
MaybeLock::Unresolved { .. } => None,
}
}
}
backend/windmill-worker/src/go_executor.rs
+80 -92
View File
@@ -1,4 +1,4 @@
use crate::PROXY_ENVS;
use crate::{common::MaybeLock, PROXY_ENVS};
use std::{collections::HashMap, fs::DirBuilder, process::Stdio};
use itertools::Itertools;
@@ -19,8 +19,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, read_result,
start_child_process, OccupancyMetrics,
build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables,
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV,
@@ -91,12 +91,12 @@ pub async fn handle_go_job(
parent_runnable_path: Option<String>,
inner_content: &str,
job_dir: &str,
requirements_o: Option<&String>,
shared_mount: &str,
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
occupation_metrics: &mut OccupancyMetrics,
maybe_lock: MaybeLock,
) -> Result<Box<RawValue>, Error> {
//go does not like executing modules at temp root
let job_dir = &format!("{job_dir}/go");
@@ -105,14 +105,7 @@ pub async fn handle_go_job(
.create(&job_dir)
.expect("could not create go job dir");
let hash = calculate_hash(&format!(
"{}{}v2",
inner_content,
requirements_o
.as_ref()
.map(|x| x.to_string())
.unwrap_or_default()
));
let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock));
let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR);
let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}");
let (cache, cache_logs) =
@@ -120,8 +113,8 @@ pub async fn handle_go_job(
let (skip_go_mod, skip_tidy) = if cache {
(true, true)
} else if let Some(requirements) = requirements_o {
gen_go_mod(inner_content, job_dir, &requirements).await?
} else if let Some(lock) = maybe_lock.get_lock() {
gen_go_mod(inner_content, job_dir, &lock).await?
} else {
(false, false)
};
@@ -133,6 +126,7 @@ pub async fn handle_go_job(
install_go_dependencies(
&job.id,
inner_content,
maybe_lock,
mem_peak,
canceled_by,
job_dir,
@@ -140,7 +134,6 @@ pub async fn handle_go_job(
true,
skip_go_mod,
skip_tidy,
false,
worker_name,
&job.workspace_id,
occupation_metrics,
@@ -430,20 +423,16 @@ func Run(req Req) (interface{{}}, error){{
read_result(job_dir, handle_result.result_stream).await
}
async fn gen_go_mod(
inner_content: &str,
job_dir: &str,
requirements: &str,
) -> error::Result<(bool, bool)> {
async fn gen_go_mod(inner_content: &str, job_dir: &str, lock: &str) -> error::Result<(bool, bool)> {
gen_go_mymod(inner_content, job_dir).await?;
let md = requirements.split_once(GO_REQ_SPLITTER);
let md = lock.split_once(GO_REQ_SPLITTER);
if let Some((req, sum)) = md {
write_file(job_dir, "go.mod", &req)?;
write_file(job_dir, "go.sum", &sum)?;
Ok((true, true))
} else {
write_file(job_dir, "go.mod", &requirements)?;
write_file(job_dir, "go.mod", &lock)?;
Ok((true, false))
}
}
@@ -454,92 +443,99 @@ use std::io::prelude::*;
pub async fn install_go_dependencies(
job_id: &Uuid,
code: &str,
maybe_lock: MaybeLock,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
conn: &Connection,
non_dep_job: bool,
// NOTE: this is impossible for skip_go_mod be `false` and maybe_lock be `Resolved`.
// TODO: make it comptime gurantee
skip_go_mod: bool,
has_sum: bool,
raw_deps: bool,
worker_name: &str,
w_id: &str,
occupation_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
let anns = GoAnnotations::parse(code);
if raw_deps {
let go_mod =
if let Some(module) = code.lines().find(|l| l.trim_start().starts_with("module ")) {
code.replace(module, "module mymod")
let hash_input = match maybe_lock {
MaybeLock::Resolved { ref lock } => lock.clone(),
MaybeLock::Unresolved { ref workspace_dependencies } => {
// NOTE: This will always be none, go workspace dependencies are disabled for now.
// read more on discord (internal):
// https://discord.com/channels/930051556043276338/1031563866641018910/1443541229349634189
if let Some(go_mod) = workspace_dependencies.get_go()? {
if !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
fs::write(format!("{job_dir}/go.mod"), &go_mod).await?;
}
go_mod
} else {
format!("module mymod\n{code}")
};
fs::write(format!("{job_dir}/go.mod"), go_mod).await?;
}
if !raw_deps && !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
let mut child_cmd = Command::new(GO_PATH.as_str());
child_cmd
.current_dir(job_dir)
.env_clear()
.args(vec!["mod", "init", "mymod"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
let mut child_cmd = Command::new(GO_PATH.as_str());
child_cmd
.current_dir(job_dir)
.env_clear()
.args(vec!["mod", "init", "mymod"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
child_cmd.env("GOPATH", windows_gopath());
#[cfg(unix)]
child_cmd.env("GOPATH", GO_CACHE_DIR);
#[cfg(windows)]
child_cmd.env("GOPATH", windows_gopath());
#[cfg(unix)]
child_cmd.env("GOPATH", GO_CACHE_DIR);
#[cfg(windows)]
set_windows_env_vars(&mut child_cmd);
let child_process = start_child_process(child_cmd, GO_PATH.as_str(), false).await?;
#[cfg(windows)]
set_windows_env_vars(&mut child_cmd);
let child_process =
start_child_process(child_cmd, GO_PATH.as_str(), false).await?;
handle_child(
job_id,
conn,
mem_peak,
canceled_by,
child_process,
false,
worker_name,
w_id,
"go init",
None,
false,
&mut Some(occupation_metrics),
None,
None,
)
.await?;
handle_child(
job_id,
conn,
mem_peak,
canceled_by,
child_process,
false,
worker_name,
w_id,
"go init",
None,
false,
&mut Some(occupation_metrics),
None,
None,
)
.await?;
for x in REQUIRE_PARSE.captures_iter(code) {
let mut file = OpenOptions::new()
.write(true)
.append(true)
.open(format!("{job_dir}/go.mod"))
.unwrap();
for x in REQUIRE_PARSE.captures_iter(code) {
let mut file = OpenOptions::new()
.write(true)
.append(true)
.open(format!("{job_dir}/go.mod"))
.unwrap();
writeln!(file, "require {}\n", &x[1])?;
writeln!(file, "require {}\n", &x[1])?;
}
}
if !has_sum {
calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
} else {
"".to_owned()
}
}
}
}
let mut new_lockfile = false;
let hash = if raw_deps {
calculate_hash(code)
} else if !has_sum {
calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
} else {
"".to_string()
};
let hash = format!(
"go{}-{}",
if anns.go1_22_compat { "1.22" } else { "" },
hash
calculate_hash(&hash_input)
);
let mut skip_tidy = has_sum;
let (mut new_lockfile, mut skip_tidy) = (false, has_sum);
if !has_sum {
if let Some(db) = conn.as_sql() {
@@ -561,15 +557,7 @@ pub async fn install_go_dependencies(
}
}
let mod_command = if skip_tidy ||
// If there is go.mod provided we want to use `download` only.
// Unlike `tidy` it does not modify local go.mod
raw_deps
{
"download"
} else {
"tidy"
};
let mod_command = if skip_tidy { "download" } else { "tidy" };
let mut child_cmd = Command::new(GO_PATH.as_str());
child_cmd
.current_dir(job_dir)
backend/windmill-worker/src/lib.rs
+2 -1
View File
@@ -70,6 +70,7 @@ mod worker;
mod worker_flow;
mod worker_lockfiles;
mod worker_utils;
pub mod workspace_dependencies;
pub use worker::*;
pub use worker_lockfiles::{
@@ -85,4 +86,4 @@ pub use bun_executor::{
pub use deno_executor::generate_deno_lock;
#[cfg(feature = "python")]
pub use python_versions::{PyV, PyVAlias};
pub use python_versions::PyV;
backend/windmill-worker/src/php_executor.rs
+27 -14
View File
@@ -7,7 +7,9 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command};
use uuid::Uuid;
use windmill_common::{
error::{self, to_anyhow, Result},
scripts::ScriptLang,
worker::{write_file, Connection},
workspace_dependencies::clean_lock_from_annotations,
};
use windmill_queue::MiniPulledJob;
@@ -17,11 +19,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, start_child_process, OccupancyMetrics,
get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics,
},
handle_child::handle_child,
COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
NSJAIL_PATH, PHP_PATH,
COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
};
use windmill_common::client::AuthedClient;
@@ -135,9 +136,23 @@ $args->{arg_name} = new {rt_name}($args->{arg_name});"
)
}
fn split_reqs_and_lock(content: &String) -> error::Result<(Option<String>, Option<String>)> {
let splitted = content.split(COMPOSER_LOCK_SPLIT).collect_vec();
if splitted.len() != 2 {
return Err(error::Error::ExecutionErr(format!(
"Invalid requirements, expected to find LOCK split pattern in reqs. Found: |{content}|"
)));
}
Ok((
Some(clean_lock_from_annotations(splitted[0], ScriptLang::Php)),
Some(splitted[1].to_string()),
))
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_php_job(
requirements_o: Option<&String>,
maybe_lock: MaybeLock,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
@@ -154,16 +169,14 @@ pub async fn handle_php_job(
) -> error::Result<Box<RawValue>> {
check_executor_binary_exists("php", PHP_PATH.as_str(), "php")?;
let (composer_json, composer_lock) = match requirements_o {
Some(reqs_and_lock) if !reqs_and_lock.is_empty() => {
let splitted = reqs_and_lock.split(COMPOSER_LOCK_SPLIT).collect_vec();
if splitted.len() != 2 {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find LOCK split pattern in reqs. Found: |{reqs_and_lock}|")
));
}
(Some(splitted[0].to_string()), Some(splitted[1].to_string()))
}
let (composer_json, composer_lock) = match &maybe_lock {
MaybeLock::Resolved { lock } if !lock.is_empty() => split_reqs_and_lock(lock)?,
MaybeLock::Unresolved { workspace_dependencies } => (
workspace_dependencies
.get_php()?
.or(parse_php_imports(inner_content)?),
None,
),
_ => (parse_php_imports(inner_content)?, None),
};
backend/windmill-worker/src/python_executor.rs
+30 -34
View File
@@ -30,7 +30,8 @@ use windmill_common::{
},
utils::calculate_hash,
worker::{
copy_dir_recursively, pad_string, write_file, Connection, PythonAnnotations, WORKER_CONFIG,
copy_dir_recursively, pad_string, split_python_requirements, write_file, Connection,
PyVAlias, PythonAnnotations, WORKER_CONFIG,
},
};
@@ -122,13 +123,13 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, read_result,
start_child_process, OccupancyMetrics, StreamNotifier,
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file,
read_result, start_child_process, OccupancyMetrics, StreamNotifier,
},
handle_child::handle_child,
worker_utils::ping_job_status,
PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
PyV, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
};
use windmill_common::client::AuthedClient;
@@ -566,7 +567,7 @@ pub async fn handle_python_job(
canceled_by,
&mut Some(occupancy_metrics),
precomputed_agent_info,
annotations,
annotations.clone(),
)
.await?;
@@ -838,10 +839,7 @@ mount {{
} else {
let args = vec!["-u", "-m", "wrapper"];
let mut python_cmd = build_command_with_isolation(
&python_path,
&args,
);
let mut python_cmd = build_command_with_isolation(&python_path, &args);
python_cmd
.current_dir(job_dir)
.env_clear()
@@ -1177,36 +1175,43 @@ async fn handle_python_deps(
let (pyv, resolved_lines) = match requirements_o {
// Deployed
Some(r) => {
let rl = split_requirements(r);
let rl = split_python_requirements(r);
(PyV::parse_from_requirements(&rl), rl)
}
// Preview
None => {
let (v, requirements_lines, error_hint) = match conn {
Connection::Sql(db) => {
let mut version_specifiers = vec![];
let (mut version_specifiers, mut locked_v) = (vec![], None);
let (r, h) = Box::pin(windmill_parser_py_imports::parse_python_imports(
inner_content,
w_id,
script_path,
db,
&mut version_specifiers,
&mut locked_v,
&None,
))
.await?;
let v = PyV::resolve(
version_specifiers,
job_id,
w_id,
annotations.py_select_latest,
Some(conn.clone()),
None,
None,
)
.await?;
let v = if let Some(v) = locked_v {
v.into()
} else {
PyV::resolve(
version_specifiers,
job_id,
w_id,
annotations.py_select_latest,
Some(conn.clone()),
None,
None,
)
.await?
};
(v, r, h)
}
Connection::Http(_) => match precomputed_agent_info {
Some(PrecomputedAgentInfo::Python {
requirements,
@@ -1237,7 +1242,7 @@ Returned from server: py_version - {:?}, py_version_v2 - {:?}
}
};
let r = split_requirements(requirements.unwrap_or_default());
let r = split_python_requirements(requirements.unwrap_or_default());
let h = None;
(v, r, h)
@@ -2113,15 +2118,6 @@ pub async fn handle_python_reqs(
};
}
pub fn split_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
requirements
.as_ref()
.lines()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.map(String::from)
.collect()
}
// Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed
fn get_result_postprocessor<'a>(skip: bool) -> &'a str {
if skip {
@@ -2157,7 +2153,7 @@ pub async fn start_worker(
killpill_rx: tokio::sync::broadcast::Receiver<()>,
client: windmill_common::client::AuthedClient,
) -> error::Result<()> {
use crate::{PyV, PyVAlias};
use crate::PyV;
tracing::info!("script path: {}", script_path);
let mut mem_peak: i32 = 0;
@@ -2317,7 +2313,7 @@ for line in sys.stdin:
proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string());
let py_version = if let Some(requirements) = requirements_o {
PyV::parse_from_requirements(&split_requirements(requirements.as_str()))
PyV::parse_from_requirements(&split_python_requirements(requirements.as_str()))
} else {
tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11");
PyVAlias::Py311.into()
backend/windmill-worker/src/python_versions.rs
+2 -97
View File
@@ -12,8 +12,7 @@ use tokio::{fs::DirBuilder, process::Command, sync::RwLock};
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT,
worker::Connection,
worker::{try_parse_locked_python_version_from_requirements, Connection, PyVAlias},
};
use anyhow::{anyhow, bail};
@@ -26,28 +25,6 @@ use crate::{
HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS,
};
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
#[repr(u32)]
pub enum PyVAlias {
Py310 = 10,
#[default]
Py311,
Py312,
Py313,
}
impl Into<pep440_rs::Version> for PyVAlias {
fn into(self) -> pep440_rs::Version {
pep440_rs::Version::new([self.major() as u64, self as u64])
}
}
impl Into<u32> for PyVAlias {
fn into(self) -> u32 {
self.major() * 100 + self as u32
}
}
impl From<PyV> for PyVAlias {
fn from(value: PyV) -> Self {
match value.release() {
@@ -66,34 +43,6 @@ impl From<PyV> for PyVAlias {
Self::default()
}
}
impl PyVAlias {
fn all<T: From<PyVAlias>>() -> Vec<T> {
use PyVAlias::*;
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
}
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
fn major(&self) -> u32 {
use PyVAlias::*;
match self {
Py310 | Py311 | Py312 | Py313 => 3,
// Py400 | Py401 => 4
}
}
/// Converts numeric format to alias
/// Example:
/// 310u32 (in) -> PyVAlias::Py310 (out)
pub(crate) fn try_from_v1<T: ToString>(numeric: T) -> Option<Self> {
use PyVAlias::*;
match numeric.to_string().as_str() {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
_ => None,
}
}
}
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
@@ -431,51 +380,7 @@ impl PyV {
/// Parse lockfile for assigned python version.
/// If not found returns None
pub fn try_parse_from_requirements<S: AsRef<str>>(requirements_lines: &[S]) -> Option<Self> {
let parse_version = |s: &str| -> Option<PyV> {
// Possible inputs:
// V2:
// # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0
//
// V1:
// # py311 or #py311
let version_unparsed = s
.to_owned()
// Remove whitespaces. That leaves us with:
// V2: #py:3.11.0
// V1: #py311
//
// Remove #
// V2: py:3.11.0
// V1: py311
//
// Remove :
// V2: py3.11.0
// V1: py311
.replace([' ', '#', ':'], "")
// Remove "py"
// V2: 3.11.0
// V1: 311
.replace("py", "");
// We will support reading V1 syntax, but it will be overwritten next deploy
PyVAlias::try_from_v1(&version_unparsed)
.map(PyVAlias::into)
.or(pep440_rs::Version::from_str(&version_unparsed)
.ok()
.map(pep440_rs::Version::into))
};
let index = if requirements_lines.get(0).map_or(false, |line| {
line.as_ref()
.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT)
}) {
1
} else {
0
};
requirements_lines
.get(index)
.map(S::as_ref)
.and_then(parse_version)
try_parse_locked_python_version_from_requirements(requirements_lines).map(PyV::from)
}
pub async fn get_python(
backend/windmill-worker/src/scoped_dependency_map.rs
+93 -62
View File
@@ -1,22 +1,24 @@
use serde::Serialize;
use sqlx::PgExecutor;
use tokio::sync::RwLock;
use windmill_common::{
apps::traverse_app_inline_scripts,
cache,
error::{Error, Result},
flows::{FlowModuleValue, FlowValue},
scripts::ScriptLang,
};
use std::collections::HashSet;
use crate::worker_lockfiles::extract_relative_imports;
use windmill_common::lockfiles::is_generated_from_raw_requirements;
use crate::worker_lockfiles::extract_referenced_paths;
// TODO: To be removed in future versions
lazy_static::lazy_static! {
pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok();
}
// TODO: Rename to DependencyRelation
#[derive(Serialize)]
pub struct DependencyMap {
pub workspace_id: Option<String>,
@@ -26,9 +28,17 @@ pub struct DependencyMap {
pub importer_node_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct DependencyDependent {
pub importer_path: String,
pub importer_kind: String,
pub importer_node_ids: Option<Vec<String>>,
}
#[derive(Debug)]
pub struct ScopedDependencyMap {
dmap: HashSet<(String, String)>,
/// (importer_node_id, imported_path)
to_delete: HashSet<(String, String)>,
w_id: String,
importer_path: String,
importer_kind: String,
@@ -72,7 +82,7 @@ RETURNING importer_node_id, imported_path
.fetch_all(executor)
.await?;
Ok(Self {
dmap: HashSet::from_iter(dmap.into_iter()),
to_delete: HashSet::from_iter(dmap.into_iter()),
w_id: w_id.to_owned(),
importer_path: importer_path.to_owned(),
importer_kind: importer_kind.to_owned(),
@@ -104,7 +114,7 @@ SELECT importer_node_id, imported_path
.await?;
Ok(Self {
dmap: HashSet::from_iter(dmap.into_iter()),
to_delete: HashSet::from_iter(dmap.into_iter()),
w_id: w_id.to_owned(),
importer_path: importer_path.to_owned(),
importer_kind: importer_kind.to_owned(),
@@ -115,22 +125,23 @@ SELECT importer_node_id, imported_path
/// Remove matching entries
pub(crate) async fn patch<'c>(
&mut self,
relative_imports: Option<Vec<String>>,
referenced_paths: Option<Vec<String>>,
node_id: String, // Flow Step/Node ID
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
self.patch_tx_ref(relative_imports, &node_id, &mut tx)
self.patch_tx_ref(referenced_paths, &node_id, &mut tx)
.await?;
Ok(tx)
}
pub(crate) async fn patch_tx_ref<'c>(
&mut self,
relative_imports: Option<Vec<String>>,
// NOTE: Referenced_paths should include all of the paths.
referenced_paths: Option<Vec<String>>,
node_id: &str, // Flow Step/Node ID
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
) -> Result<()> {
let Some(mut relative_imports) = relative_imports else {
let Some(mut referenced_paths) = referenced_paths else {
tracing::info!("relative imports are not found for: importer - {}, importer_node_id - {}, importer_kind - {}",
&self.importer_path,
&node_id,
@@ -149,9 +160,9 @@ SELECT importer_node_id, imported_path
// After all `reduce`'s called ScopedDependencyMap has only extra/orphan imports
// these are going to be clean up by calling [dissolve]
// NOTE: `retain` iterates over vec and remove the ones whose closures returned false.
relative_imports.retain(|imported_path| {
referenced_paths.retain(|imported_path| {
!self
.dmap
.to_delete
// As dmap is HashSet, removing is O(1) operation
// thus making entire process very efficient
// NOTE: `remove` returns true if item was removed and false if wasn't.
@@ -159,15 +170,15 @@ SELECT importer_node_id, imported_path
});
// As mentioned above, usually this will always be empty.
if !relative_imports.is_empty() {
if !referenced_paths.is_empty() {
tracing::info!("adding missing entries to dependency_map: importer_node_id - {}, importer_kind - {}, new_imported_paths - {:?}",
&node_id,
&self.importer_kind,
&relative_imports,
&referenced_paths,
);
}
for import in relative_imports {
for import in referenced_paths {
sqlx::query!(
"INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)
VALUES ($1, $2, $3::text::IMPORTER_KIND, $4, $5) ON CONFLICT DO NOTHING",
@@ -200,7 +211,7 @@ SELECT importer_node_id, imported_path
tracing::info!("dissolving dependency_map: {:?}", &self);
// We _could_ shove it into single query, but this query is rarely called AND let's keep it simple for redability.
for (importer_node_id, imported_path) in self.dmap.into_iter() {
for (importer_node_id, imported_path) in self.to_delete.into_iter() {
tracing::info!("cleaning orphan entry from dependency_map: importer_kind - {}, imported_path - {}, importer_node_id - {}",
&self.importer_kind,
&imported_path,
@@ -275,14 +286,14 @@ SELECT importer_node_id, imported_path
tx
}
/// Run if you want to rebuild maps on specific workspace.
/// Potentially takes much time
pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool<sqlx::Postgres>) -> Result<String> {
async fn inner<'c>(w_id: &str, db: &sqlx::Pool<sqlx::Postgres>) -> Result<String> {
// Scripts
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts");
for r in sqlx::query!(
"SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false",
pub(crate) async fn rebuild_map_unchecked<'c>(
w_id: &str,
db: &sqlx::Pool<sqlx::Postgres>,
) -> Result<String> {
// Scripts
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts");
for r in sqlx::query!(
r#"SELECT path, hash, language AS "language: ScriptLang" FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false"#,
w_id
)
.fetch_all(db)
@@ -292,29 +303,24 @@ SELECT importer_node_id, imported_path
let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "script", db).await?;
let mut tx = db.begin().await?;
if is_generated_from_raw_requirements(&smd.language, &sd.lock) {
// if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map
// because we do not want to have dependencies be recomputed automatically. Empty relative imports passed
// to update_script_dependency_map will clear the dependency map.
} else {
tx = dmap
.patch(
extract_relative_imports(&sd.code, &r.path, &smd.language),
"".into(),
tx,
)
.await?;
}
tx = dmap
.patch(
extract_referenced_paths(&sd.code, &r.path, smd.language),
"".into(),
tx,
)
.await?;
if !*WMDEBUG_NO_DMAP_DISSOLVE {
dmap.dissolve(tx).await.commit().await?;
}
tracing::info!(workspace_id = w_id, "Rebuilt for script {}", &r.path);
}
// Fetch only top level versions and paths
// It is not fetching value
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? {
// Fetch only top level versions and paths
// It is not fetching value
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? {
if let Some(version) = r.version {
// To reduce stress on db try to fetch from cache
// Since our flow versions are immutable it is safe to assume if we have cache for specific version/id it is up to date.
@@ -337,17 +343,15 @@ SELECT importer_node_id, imported_path
FlowValue::traverse_leafs(modules_to_check, &mut |fmv, id| {
match fmv {
// Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript.
FlowModuleValue::RawScript { content, language, lock ,.. } => {
if !is_generated_from_raw_requirements(&Some(*language), lock) {
to_process.push((
extract_relative_imports(
content,
&(r.path.clone() + "/flow"),
&Some(language.clone()),
),
id.clone(),
));
}
FlowModuleValue::RawScript { content, language, .. } => {
to_process.push((
extract_referenced_paths(
content,
&(r.path.clone() + "/flow"),
Some(*language),
),
id.clone(),
));
}
// But just in case we will also handle other cases.
FlowModuleValue::FlowScript { .. } => {
@@ -359,8 +363,8 @@ SELECT importer_node_id, imported_path
Ok(())
})?;
for (ri, id) in to_process {
tx = dmap.patch(ri, id, tx).await?;
for (rp, id) in to_process {
tx = dmap.patch(rp, id, tx).await?;
}
if !*WMDEBUG_NO_DMAP_DISSOLVE {
@@ -374,9 +378,9 @@ SELECT importer_node_id, imported_path
}
}
// Apps
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? {
// Apps
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? {
if let Some(version) = r.version {
// TODO: Use cache when implemented.
let value = sqlx::query_scalar!(
@@ -391,10 +395,10 @@ SELECT importer_node_id, imported_path
let mut to_process = vec![];
traverse_app_inline_scripts(&value, None, &mut |ais, id| {
to_process.push((
extract_relative_imports(
extract_referenced_paths(
&ais.content,
&(r.path.clone() + "/app"),
&ais.language,
ais.language,
),
id,
));
@@ -417,9 +421,11 @@ SELECT importer_node_id, imported_path
}
}
Ok("Success".into())
}
Ok("Success".into())
}
/// Run if you want to rebuild maps on specific workspace.
/// Potentially takes much time
pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool<sqlx::Postgres>) -> Result<String> {
lazy_static::lazy_static! {
pub static ref LOCKED: RwLock<bool> = RwLock::new(false);
}
@@ -438,9 +444,34 @@ SELECT importer_node_id, imported_path
}
*LOCKED.write().await = true;
let r = inner(w_id, db).await;
let r = Self::rebuild_map_unchecked(w_id, db).await;
*LOCKED.write().await = false;
r
}
}
/// Get dependents of any imported path - returns scripts/flows/apps that depend on it
pub async fn get_dependents<'c>(
imported_path: &str,
workspace_id: &str,
e: impl PgExecutor<'c>,
) -> Result<Vec<DependencyDependent>> {
sqlx::query_as!(
DependencyDependent,
r#"
SELECT
importer_path,
importer_kind::text as "importer_kind!", -- sqlx thinks this is nullable somehow, so enfore with !
array_agg(importer_node_id) as importer_node_ids
FROM dependency_map
WHERE workspace_id = $1 AND imported_path = $2
GROUP BY importer_path, importer_kind
"#,
workspace_id,
imported_path
)
.fetch_all(e)
.await
.map_err(Error::from)
}
}
backend/windmill-worker/src/worker.rs
+90 -19
View File
@@ -22,6 +22,8 @@ use windmill_common::scripts::is_special_codebase_hash;
use windmill_common::utils::report_critical_error;
use windmill_common::utils::retrieve_common_worker_prefix;
use windmill_common::worker::error_to_value;
use windmill_common::workspace_dependencies::RawWorkspaceDependencies;
use windmill_common::workspace_dependencies::WorkspaceDependenciesPrefetched;
use windmill_common::{
agent_workers::DECODED_AGENT_TOKEN,
apps::AppScriptId,
@@ -111,6 +113,7 @@ use tokio::{
use rand::Rng;
use crate::ai_executor::handle_ai_agent_job;
use crate::common::MaybeLock;
use crate::common::StreamNotifier;
use crate::{
agent_workers::{queue_init_job, queue_periodic_job},
@@ -155,10 +158,9 @@ use crate::ruby_executor::{handle_ruby_job, JobHandlerInput as JobHandlerInputRu
use crate::php_executor::handle_php_job;
#[cfg(feature = "python")]
use crate::{
python_executor::handle_python_job,
python_versions::{PyV, PyVAlias},
};
use crate::{python_executor::handle_python_job, python_versions::PyV};
#[cfg(feature = "python")]
use windmill_common::worker::PyVAlias;
#[cfg(feature = "python")]
use crate::ansible_executor::handle_ansible_job;
@@ -2859,6 +2861,16 @@ pub async fn handle_queued_job(
let mut column_order: Option<Vec<String>> = None;
let mut new_args: Option<HashMap<String, Box<RawValue>>> = None;
let mut has_stream = false;
let raw_workspace_dependencies_o = if job.kind.is_dependency() {
job.args
.as_ref()
.and_then(|x| x.get("raw_workspace_dependencies"))
.map(|v| v.get())
.and_then(|v| serde_json::from_str::<RawWorkspaceDependencies>(v).ok())
} else {
None
};
// Box::pin all async branches to prevent large match enum on stack
let result = match job.kind {
JobKind::Dependencies => match conn {
@@ -2875,6 +2887,7 @@ pub async fn handle_queued_job(
base_internal_url,
&client.token,
occupancy_metrics,
raw_workspace_dependencies_o,
))
.await
}
@@ -2898,6 +2911,7 @@ pub async fn handle_queued_job(
base_internal_url,
&client.token,
occupancy_metrics,
raw_workspace_dependencies_o,
))
.await
}
@@ -2919,6 +2933,7 @@ pub async fn handle_queued_job(
base_internal_url,
&client.token,
occupancy_metrics,
raw_workspace_dependencies_o,
))
.await
.map(|()| serde_json::from_str("{}").unwrap()),
@@ -3706,14 +3721,43 @@ mount {{
let envs = build_envs(envs.as_ref())?;
let Some(language) = language else {
return Err(Error::ExecutionErr(
"Require language to be not null".to_string(),
))?;
};
/// Resolves MaybeLock for languages that need workspace dependencies prefetching.
/// Only call this for Bun, Bunnative, Go, and Php.
async fn resolve_maybe_lock(
lock: &Option<String>,
code: &str,
language: ScriptLang,
workspace_id: &str,
runnable_path: &str,
conn: Connection,
) -> error::Result<MaybeLock> {
if let Some(lock) = lock.clone() {
Ok(MaybeLock::Resolved { lock })
} else {
Ok(MaybeLock::Unresolved {
workspace_dependencies: WorkspaceDependenciesPrefetched::extract(
code,
language,
workspace_id,
// TODO: implement
&None,
runnable_path,
conn,
)
.await?,
})
}
}
// Box::pin all language handlers to prevent large match enum on stack
let result: error::Result<Box<RawValue>> = match language {
None => {
return Err(Error::ExecutionErr(
"Require language to be not null".to_string(),
))?;
}
Some(ScriptLang::Python3) => {
ScriptLang::Python3 => {
#[cfg(not(feature = "python"))]
return Err(Error::internal_err(
"Python requires the python feature to be enabled".to_string(),
@@ -3780,8 +3824,17 @@ mount {{
"Inline execution is not yet supported for this language".to_string(),
));
}
let maybe_lock = resolve_maybe_lock(
&lock,
&code,
language,
&job.workspace_id,
job.runnable_path(),
conn.clone(),
)
.await?;
Box::pin(handle_bun_job(
lock.as_ref(),
maybe_lock,
codebase.as_ref(),
mem_peak,
canceled_by,
@@ -3808,6 +3861,15 @@ mount {{
"Inline execution is not yet supported for this language".to_string(),
));
}
let maybe_lock = resolve_maybe_lock(
&lock,
&code,
language,
&job.workspace_id,
job.runnable_path(),
conn.clone(),
)
.await?;
Box::pin(handle_go_job(
mem_peak,
canceled_by,
@@ -3817,12 +3879,12 @@ mount {{
parent_runnable_path,
&code,
job_dir,
lock.as_ref(),
&shared_mount,
base_internal_url,
worker_name,
envs,
occupancy_metrics,
maybe_lock,
))
.await
}
@@ -3873,7 +3935,7 @@ mount {{
))
.await
}
Some(ScriptLang::Php) => {
ScriptLang::Php => {
#[cfg(not(feature = "php"))]
return Err(Error::internal_err(
"PHP requires the php feature to be enabled".to_string(),
@@ -3886,8 +3948,17 @@ mount {{
"Inline execution is not yet supported for this language".to_string(),
));
}
let maybe_lock = resolve_maybe_lock(
&lock,
&code,
language,
&job.workspace_id,
job.runnable_path(),
conn.clone(),
)
.await?;
Box::pin(handle_php_job(
lock.as_ref(),
maybe_lock,
mem_peak,
canceled_by,
job,
@@ -3905,7 +3976,7 @@ mount {{
.await
}
}
Some(ScriptLang::Rust) => {
ScriptLang::Rust => {
#[cfg(not(feature = "rust"))]
return Err(Error::internal_err(
"Rust requires the rust feature to be enabled".to_string(),
@@ -3937,7 +4008,7 @@ mount {{
.await
}
}
Some(ScriptLang::Ansible) => {
ScriptLang::Ansible => {
#[cfg(not(feature = "python"))]
return Err(Error::internal_err(
"Ansible requires the python feature to be enabled".to_string(),
@@ -3994,7 +4065,7 @@ mount {{
))
.await
}
Some(ScriptLang::Nu) => {
ScriptLang::Nu => {
#[cfg(not(feature = "nu"))]
return Err(
anyhow::anyhow!("Nu is not available because the feature is not enabled").into(),
@@ -4026,7 +4097,7 @@ mount {{
.await
}
}
Some(ScriptLang::Java) => {
ScriptLang::Java => {
#[cfg(not(feature = "java"))]
return Err(anyhow::anyhow!(
"Java is not available because the feature is not enabled"
@@ -4059,7 +4130,7 @@ mount {{
.await
}
}
Some(ScriptLang::Ruby) => {
ScriptLang::Ruby => {
#[cfg(not(feature = "ruby"))]
return Err(anyhow::anyhow!(
"Ruby is not available because the feature is not enabled"
backend/windmill-worker/src/worker_lockfiles.rs
+288 -323
View File
File diff suppressed because it is too large Load Diff
backend/windmill-worker/src/workspace_dependencies.rs
+268
View File
@@ -0,0 +1,268 @@
use serde::{Deserialize, Serialize};
use windmill_common::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies};
use crate::{
scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies,
};
#[derive(sqlx::FromRow, Clone, Serialize, Deserialize, Hash, Debug)]
pub struct NewWorkspaceDependencies {
pub workspace_id: String,
pub language: ScriptLang,
pub name: Option<String>,
/// If None, will use description of previous version
/// If there is no older versions, will set to default
pub description: Option<String>,
// TODO: Make Option, or optimize it in any other way.
pub content: String,
}
impl NewWorkspaceDependencies {
/// Creates a new workspace dependencies entry in the database.
///
/// Archives any existing dependencies with the same name/language/workspace,
/// then inserts the new dependencies. Triggers recomputation of dependent scripts
/// and rebuilds the dependency map if this is the first unnamed dependency for the workspace.
pub async fn create<'c>(
self,
email: &str,
created_by: &str,
permissioned_as: &str,
db: &sqlx::Pool<sqlx::Postgres>,
) -> error::Result<i64> {
// Check if all workers support workspace dependencies feature
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
.await?;
let path = WorkspaceDependencies::to_path(&self.name, self.language)?;
// If it is unnamed then we want to rebuild dependency map. Otherwise trigger dependents to recompute locks will not work
// NOTE: We rebuild first, even before creating new w deps. We want to make sure that if rebuild failed, then no new default workspace dependencies were created.
if self.name.is_none() {
// Check if we already rebuilt the map for this workspace by checking if the setting exists
let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id);
let already_rebuilt =
windmill_common::global_settings::load_value_from_global_settings(
db,
&setting_name,
)
.await?
.is_some();
if !already_rebuilt {
tracing::info!(
workspace_id = %self.workspace_id,
"Rebuilding workspace dependencies map for first unnamed workspace dependencies"
);
ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, db).await?;
// Mark as rebuilt by creating the setting
windmill_common::global_settings::set_value_in_global_settings(
db,
&setting_name,
serde_json::json!({}),
)
.await?;
tracing::info!(
workspace_id = %self.workspace_id,
"Marked workspace dependencies map as rebuilt"
);
} else {
tracing::info!(
workspace_id = %self.workspace_id,
"Skipping workspace dependencies map rebuild - already rebuilt for this workspace"
);
}
};
let mut tx = db.begin().await?;
let prev_description = sqlx::query_scalar!(
"
UPDATE workspace_dependencies
SET archived = true
WHERE archived = false
AND name IS NOT DISTINCT FROM $1
AND workspace_id = $2
AND language = $3
RETURNING description
",
self.name,
self.workspace_id,
self.language as ScriptLang
)
.fetch_optional(&mut *tx)
.await?;
let new_id = sqlx::query_scalar!(
"
INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)
VALUES ($1, $2, $3, $4, $5)
RETURNING id
",
self.name.clone(),
self.workspace_id,
self.content,
self.language as ScriptLang,
self.description
.or(prev_description.clone())
.unwrap_or("Default Workspace Dependencies".to_owned())
)
.fetch_one(&mut *tx)
.await?;
tx.commit().await?;
// Make sure trigger dependents will have latest view.
// NOTE: Uncomment for tests
// #[cfg(test)]
// assert_eq!(
// sqlx::query_scalar!(
// "
// SELECT id FROM workspace_dependencies
// WHERE archived = false
// AND name IS NOT DISTINCT FROM $1
// AND workspace_id = $2
// AND language = $3
// ",
// self.name,
// self.workspace_id,
// self.language as ScriptLang,
// )
// .fetch_one(db) // Use db
// .await?,
// new_id
// );
// It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies
trigger_dependents_to_recompute_dependencies(
&self.workspace_id,
crate::scoped_dependency_map::ScopedDependencyMap::get_dependents(
path.as_str(),
&self.workspace_id,
db,
)
.await?,
None,
None,
email,
created_by,
permissioned_as,
db,
vec![],
)
.await?;
Ok(new_id)
}
}
// Type aliases for backward compatibility
pub type RawRequirements = WorkspaceDependencies;
pub type NewRawRequirements = NewWorkspaceDependencies;
#[cfg(test)]
mod workspace_dependencies_tests {
// // TODO: test all cases when it should reject.
// #[cfg(feature = "python")]
// mod new_workspace_dependencies {
// use windmill_common::scripts::ScriptLang;
// use crate::workspace_dependencies::NewWorkspaceDependencies;
// #[sqlx::test(
// fixtures("../../tests/fixtures/base.sql",),
// migrations = "../migrations"
// )]
// async fn test_create(db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
// assert_eq!(
// NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::Python3,
// name: None,
// description: None,
// content: "global:rev1".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .unwrap(),
// 1
// );
// assert_eq!(
// NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::Python3,
// name: Some("rrs1".to_owned()),
// description: None,
// content: "rrs1:rev1".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .unwrap(),
// 2
// );
// assert!(NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::DuckDb,
// description: None,
// name: None,
// content: "".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .is_err());
// // Will act as redeployment
// assert_eq!(
// NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::Python3,
// description: None,
// name: Some("rrs1".to_owned()),
// content: "rrs1:rev2".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .unwrap(),
// // It will just increment id
// 3
// );
// Ok(())
// }
// #[sqlx::test(
// fixtures("../../tests/fixtures/base.sql",),
// migrations = "../migrations"
// )]
// async fn violate_constraints(db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
// let db = &db;
// let create = |name| {
// sqlx::query_scalar!(
// "
// INSERT INTO workspace_dependencies(name, workspace_id, content, language)
// VALUES ($1, 'test-workspace', 'test', 'python3')
// RETURNING id
// ",
// name
// )
// .fetch_one(db)
// };
// assert_eq!(create(Some("test".to_owned())).await.unwrap(), 1);
// assert_eq!(create(None).await.unwrap(), 2);
// assert!(create(Some("test".to_owned())).await.is_err());
// assert!(create(None).await.is_err());
// assert_eq!(
// sqlx::query_scalar!("SELECT COUNT(*) FROM workspace_dependencies",)
// .fetch_one(db)
// .await
// .unwrap()
// .unwrap(),
// 2
// );
// Ok(())
// }
// }
}
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.585.1";
export const VERSION = "v1.587.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/deno.json
+4 -2
View File
@@ -13,6 +13,8 @@
"@std/path": "jsr:@std/path@^1.0.4",
"@std/streams": "jsr:@std/streams@^1.0.4",
"@std/yaml": "jsr:@std/yaml@^1.0.5",
"@types/diff": "npm:@types/diff@^5.2.2"
}
"@types/diff": "npm:@types/diff@^5.2.2",
"ws": "npm:ws@8.18.0"
},
"nodeModulesDir": "auto"
}
cli/deno.lock
Generated
+424 -83
View File
@@ -11,38 +11,44 @@
"jsr:@std/assert@1.0.0-rc.2": "1.0.0-rc.2",
"jsr:@std/bytes@0.223": "0.223.0",
"jsr:@std/bytes@^1.0.2": "1.0.2",
"jsr:@std/bytes@^1.0.5": "1.0.6",
"jsr:@std/cli@1.0.0-rc.2": "1.0.0-rc.2",
"jsr:@std/encoding@*": "1.0.4",
"jsr:@std/encoding@1.0.0-rc.2": "1.0.0-rc.2",
"jsr:@std/encoding@1.0.4": "1.0.4",
"jsr:@std/encoding@^1.0.4": "1.0.4",
"jsr:@std/fmt@0.223": "0.223.0",
"jsr:@std/fmt@1": "1.0.2",
"jsr:@std/fmt@1": "1.0.8",
"jsr:@std/fmt@^1.0.2": "1.0.2",
"jsr:@std/fmt@^1.0.5": "1.0.8",
"jsr:@std/fmt@~0.225.4": "0.225.6",
"jsr:@std/fs@*": "1.0.3",
"jsr:@std/fs@*": "1.0.20",
"jsr:@std/fs@0.223": "0.223.0",
"jsr:@std/fs@1": "1.0.3",
"jsr:@std/fs@1": "1.0.20",
"jsr:@std/fs@^1.0.11": "1.0.20",
"jsr:@std/fs@^1.0.3": "1.0.3",
"jsr:@std/fs@~0.229.3": "0.229.3",
"jsr:@std/io@*": "0.224.7",
"jsr:@std/internal@^1.0.12": "1.0.12",
"jsr:@std/io@*": "0.225.2",
"jsr:@std/io@0.223": "0.223.0",
"jsr:@std/io@~0.224.2": "0.224.7",
"jsr:@std/io@~0.224.2": "0.224.9",
"jsr:@std/io@~0.224.7": "0.224.7",
"jsr:@std/log@*": "0.224.7",
"jsr:@std/io@~0.225.2": "0.225.2",
"jsr:@std/log@*": "0.224.14",
"jsr:@std/log@~0.224.7": "0.224.7",
"jsr:@std/net@*": "1.0.2",
"jsr:@std/net@^1.0.2": "1.0.2",
"jsr:@std/path@*": "1.0.4",
"jsr:@std/path@*": "1.1.3",
"jsr:@std/path@0.223": "0.223.0",
"jsr:@std/path@1": "1.0.4",
"jsr:@std/path@1": "1.1.3",
"jsr:@std/path@1.0.0-rc.1": "1.0.0-rc.1",
"jsr:@std/path@1.0.0-rc.2": "1.0.0-rc.2",
"jsr:@std/path@^1.0.4": "1.0.4",
"jsr:@std/path@^1.1.3": "1.1.3",
"jsr:@std/path@~0.225.2": "0.225.2",
"jsr:@std/streams@^1.0.4": "1.0.4",
"jsr:@std/text@1.0.0-rc.1": "1.0.0-rc.1",
"jsr:@std/yaml@*": "1.0.5",
"jsr:@std/yaml@*": "1.0.10",
"jsr:@std/yaml@^1.0.5": "1.0.5",
"jsr:@ts-morph/bootstrap@0.24": "0.24.0",
"jsr:@ts-morph/common@0.24": "0.24.0",
@@ -58,21 +64,35 @@
"jsr:@windmill-labs/cliffy-prompt@^1.0.0-rc.5": "1.0.0-rc.6",
"jsr:@windmill-labs/cliffy-table@1.0.0-rc.5": "1.0.0-rc.5",
"jsr:@windmill-labs/cliffy-table@^1.0.0-rc.5": "1.0.0-rc.5",
"jsr:@windmill-labs/shared-utils@1.0.10": "1.0.10",
"jsr:@windmill-labs/shared-utils@1.0.3": "1.0.3",
"jsr:@windmill-labs/shared-utils@1.0.5": "1.0.5",
"jsr:@windmill-labs/shared-utils@1.0.6": "1.0.6",
"jsr:@windmill-labs/shared-utils@1.0.7": "1.0.7",
"npm:@ayonli/jsext@*": "1.8.0",
"npm:@types/node@*": "22.12.0",
"npm:@types/diff@^5.2.2": "5.2.3",
"npm:@types/node@*": "24.2.0",
"npm:@types/ws@*": "8.18.1",
"npm:@windmill-labs/shared-utils@1.0.1": "1.0.1",
"npm:@windmill-labs/shared-utils@1.0.2": "1.0.2",
"npm:centdix-utils@*": "1.0.15",
"npm:diff@*": "8.0.2",
"npm:es-main@*": "1.3.0",
"npm:esbuild@*": "0.25.8",
"npm:esbuild-plugin-vue3@0.5.1": "0.5.1_vue@3.5.25__typescript@4.9.5_typescript@4.9.5",
"npm:esbuild-svelte@0.9.3": "0.9.3_esbuild@0.24.2_svelte@5.45.2__acorn@8.14.1",
"npm:esbuild@*": "0.24.2",
"npm:esbuild@0.24.2": "0.24.2",
"npm:express@*": "5.1.0",
"npm:get-port@7.1.0": "7.1.0",
"npm:jszip@3.7.1": "3.7.1",
"npm:jszip@3.8.0": "3.8.0",
"npm:minimatch@*": "10.0.3",
"npm:open@*": "10.2.0",
"npm:ws@*": "8.18.3"
"npm:svelte-preprocess@6.0.3": "6.0.3_svelte@5.45.2__acorn@8.14.1",
"npm:svelte@5.45.2": "5.45.2_acorn@8.14.1",
"npm:ws@*": "8.18.3",
"npm:ws@8.18.0": "8.18.0",
"npm:ws@8.18.3": "8.18.3"
},
"jsr": {
"@david/code-block-writer@13.0.2": {
@@ -117,6 +137,9 @@
"@std/bytes@1.0.2": {
"integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57"
},
"@std/bytes@1.0.6": {
"integrity": "f6ac6adbd8ccd99314045f5703e23af0a68d7f7e58364b47d2c7f408aeb5820a"
},
"@std/cli@1.0.0-rc.2": {
"integrity": "97dfae82b9f0e189768ebfa7a5da53375955b94bad0a1804f8e3b73563b03787"
},
@@ -135,6 +158,9 @@
"@std/fmt@1.0.2": {
"integrity": "87e9dfcdd3ca7c066e0c3c657c1f987c82888eb8103a3a3baa62684ffeb0f7a7"
},
"@std/fmt@1.0.8": {
"integrity": "71e1fc498787e4434d213647a6e43e794af4fd393ef8f52062246e06f7e372b7"
},
"@std/fs@0.223.0": {
"integrity": "3b4b0550b2c524cbaaa5a9170c90e96cbb7354e837ad1bdaf15fc9df1ae9c31c"
},
@@ -150,6 +176,16 @@
"jsr:@std/path@^1.0.4"
]
},
"@std/fs@1.0.20": {
"integrity": "e953206aae48d46ee65e8783ded459f23bec7dd1f3879512911c35e5484ea187",
"dependencies": [
"jsr:@std/internal",
"jsr:@std/path@^1.1.3"
]
},
"@std/internal@1.0.12": {
"integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
},
"@std/io@0.223.0": {
"integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1",
"dependencies": [
@@ -163,6 +199,18 @@
"jsr:@std/bytes@^1.0.2"
]
},
"@std/io@0.224.9": {
"integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3",
"dependencies": [
"jsr:@std/bytes@^1.0.2"
]
},
"@std/io@0.225.2": {
"integrity": "3c740cd4ee4c082e6cfc86458f47e2ab7cb353dc6234d5e9b1f91a2de5f4d6c7",
"dependencies": [
"jsr:@std/bytes@^1.0.5"
]
},
"@std/log@0.224.7": {
"integrity": "021941e5cd16de60cb11599c9b36f892aea95987fe66c753922808da27909e18",
"dependencies": [
@@ -171,6 +219,14 @@
"jsr:@std/io@~0.224.7"
]
},
"@std/log@0.224.14": {
"integrity": "257f7adceee3b53bb2bc86c7242e7d1bc59729e57d4981c4a7e5b876c808f05e",
"dependencies": [
"jsr:@std/fmt@^1.0.5",
"jsr:@std/fs@^1.0.11",
"jsr:@std/io@~0.225.2"
]
},
"@std/net@1.0.2": {
"integrity": "520c18ddb7f67d3830a1adfef03a155d496fe9683a9cb63bb823b5afb86484dc"
},
@@ -195,6 +251,12 @@
"@std/path@1.0.4": {
"integrity": "48dd5d8389bcfcd619338a01bdf862cb7799933390146a54ae59356a0acc7105"
},
"@std/path@1.1.3": {
"integrity": "b015962d82a5e6daea980c32b82d2c40142149639968549c649031a230b1afb3",
"dependencies": [
"jsr:@std/internal"
]
},
"@std/streams@1.0.4": {
"integrity": "a1a5b01c74ca1d2dcaacfe1d4bbb91392e765946d82a3471bd95539adc6da83a",
"dependencies": [
@@ -207,6 +269,9 @@
"@std/yaml@1.0.5": {
"integrity": "71ba3d334305ee2149391931508b2c293a8490f94a337eef3a09cade1a2a2742"
},
"@std/yaml@1.0.10": {
"integrity": "245706ea3511cc50c8c6d00339c23ea2ffa27bd2c7ea5445338f8feff31fa58e"
},
"@ts-morph/bootstrap@0.24.0": {
"integrity": "a826a2ef7fa8a7c3f1042df2c034d20744d94da2ee32bf29275bcd4dffd3c060",
"dependencies": [
@@ -283,6 +348,21 @@
"jsr:@std/cli",
"jsr:@std/fmt@~0.225.4"
]
},
"@windmill-labs/shared-utils@1.0.3": {
"integrity": "35bafaf74092ebb63e96c75897337320378c04f93cf9b352fcc2137ffdb3e862"
},
"@windmill-labs/shared-utils@1.0.5": {
"integrity": "3709140dc40f89443dff5953ec2e7c35d964b71c5e1245fba4072cf513e0db91"
},
"@windmill-labs/shared-utils@1.0.6": {
"integrity": "34965cbc8e4fda69835fed37435468e8ca1123dabe4ea395d700ecdb2fa49738"
},
"@windmill-labs/shared-utils@1.0.7": {
"integrity": "528638c7c508910e7f51b1ad9a5f1ff394e3fefb28fd3f96ab958c258a26e978"
},
"@windmill-labs/shared-utils@1.0.10": {
"integrity": "bd1993eb8d693c8ba49da1618f82ff4601eeb59011b2cac13e664291f7a299d8"
}
},
"npm": {
@@ -291,137 +371,152 @@
"dependencies": [
"iconv-lite",
"sudo-prompt",
"ws",
"ws@8.18.3",
"zod"
]
},
"@esbuild/aix-ppc64@0.25.8": {
"integrity": "sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==",
"@babel/helper-string-parser@7.27.1": {
"integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="
},
"@babel/helper-validator-identifier@7.28.5": {
"integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="
},
"@babel/parser@7.28.5": {
"integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==",
"dependencies": [
"@babel/types"
],
"bin": true
},
"@babel/types@7.28.5": {
"integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==",
"dependencies": [
"@babel/helper-string-parser",
"@babel/helper-validator-identifier"
]
},
"@esbuild/aix-ppc64@0.24.2": {
"integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==",
"os": ["aix"],
"cpu": ["ppc64"]
},
"@esbuild/android-arm64@0.25.8": {
"integrity": "sha512-OD3p7LYzWpLhZEyATcTSJ67qB5D+20vbtr6vHlHWSQYhKtzUYrETuWThmzFpZtFsBIxRvhO07+UgVA9m0i/O1w==",
"@esbuild/android-arm64@0.24.2": {
"integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==",
"os": ["android"],
"cpu": ["arm64"]
},
"@esbuild/android-arm@0.25.8": {
"integrity": "sha512-RONsAvGCz5oWyePVnLdZY/HHwA++nxYWIX1atInlaW6SEkwq6XkP3+cb825EUcRs5Vss/lGh/2YxAb5xqc07Uw==",
"@esbuild/android-arm@0.24.2": {
"integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==",
"os": ["android"],
"cpu": ["arm"]
},
"@esbuild/android-x64@0.25.8": {
"integrity": "sha512-yJAVPklM5+4+9dTeKwHOaA+LQkmrKFX96BM0A/2zQrbS6ENCmxc4OVoBs5dPkCCak2roAD+jKCdnmOqKszPkjA==",
"@esbuild/android-x64@0.24.2": {
"integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==",
"os": ["android"],
"cpu": ["x64"]
},
"@esbuild/darwin-arm64@0.25.8": {
"integrity": "sha512-Jw0mxgIaYX6R8ODrdkLLPwBqHTtYHJSmzzd+QeytSugzQ0Vg4c5rDky5VgkoowbZQahCbsv1rT1KW72MPIkevw==",
"@esbuild/darwin-arm64@0.24.2": {
"integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==",
"os": ["darwin"],
"cpu": ["arm64"]
},
"@esbuild/darwin-x64@0.25.8": {
"integrity": "sha512-Vh2gLxxHnuoQ+GjPNvDSDRpoBCUzY4Pu0kBqMBDlK4fuWbKgGtmDIeEC081xi26PPjn+1tct+Bh8FjyLlw1Zlg==",
"@esbuild/darwin-x64@0.24.2": {
"integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==",
"os": ["darwin"],
"cpu": ["x64"]
},
"@esbuild/freebsd-arm64@0.25.8": {
"integrity": "sha512-YPJ7hDQ9DnNe5vxOm6jaie9QsTwcKedPvizTVlqWG9GBSq+BuyWEDazlGaDTC5NGU4QJd666V0yqCBL2oWKPfA==",
"@esbuild/freebsd-arm64@0.24.2": {
"integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==",
"os": ["freebsd"],
"cpu": ["arm64"]
},
"@esbuild/freebsd-x64@0.25.8": {
"integrity": "sha512-MmaEXxQRdXNFsRN/KcIimLnSJrk2r5H8v+WVafRWz5xdSVmWLoITZQXcgehI2ZE6gioE6HirAEToM/RvFBeuhw==",
"@esbuild/freebsd-x64@0.24.2": {
"integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==",
"os": ["freebsd"],
"cpu": ["x64"]
},
"@esbuild/linux-arm64@0.25.8": {
"integrity": "sha512-WIgg00ARWv/uYLU7lsuDK00d/hHSfES5BzdWAdAig1ioV5kaFNrtK8EqGcUBJhYqotlUByUKz5Qo6u8tt7iD/w==",
"@esbuild/linux-arm64@0.24.2": {
"integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==",
"os": ["linux"],
"cpu": ["arm64"]
},
"@esbuild/linux-arm@0.25.8": {
"integrity": "sha512-FuzEP9BixzZohl1kLf76KEVOsxtIBFwCaLupVuk4eFVnOZfU+Wsn+x5Ryam7nILV2pkq2TqQM9EZPsOBuMC+kg==",
"@esbuild/linux-arm@0.24.2": {
"integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==",
"os": ["linux"],
"cpu": ["arm"]
},
"@esbuild/linux-ia32@0.25.8": {
"integrity": "sha512-A1D9YzRX1i+1AJZuFFUMP1E9fMaYY+GnSQil9Tlw05utlE86EKTUA7RjwHDkEitmLYiFsRd9HwKBPEftNdBfjg==",
"@esbuild/linux-ia32@0.24.2": {
"integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==",
"os": ["linux"],
"cpu": ["ia32"]
},
"@esbuild/linux-loong64@0.25.8": {
"integrity": "sha512-O7k1J/dwHkY1RMVvglFHl1HzutGEFFZ3kNiDMSOyUrB7WcoHGf96Sh+64nTRT26l3GMbCW01Ekh/ThKM5iI7hQ==",
"@esbuild/linux-loong64@0.24.2": {
"integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==",
"os": ["linux"],
"cpu": ["loong64"]
},
"@esbuild/linux-mips64el@0.25.8": {
"integrity": "sha512-uv+dqfRazte3BzfMp8PAQXmdGHQt2oC/y2ovwpTteqrMx2lwaksiFZ/bdkXJC19ttTvNXBuWH53zy/aTj1FgGw==",
"@esbuild/linux-mips64el@0.24.2": {
"integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==",
"os": ["linux"],
"cpu": ["mips64el"]
},
"@esbuild/linux-ppc64@0.25.8": {
"integrity": "sha512-GyG0KcMi1GBavP5JgAkkstMGyMholMDybAf8wF5A70CALlDM2p/f7YFE7H92eDeH/VBtFJA5MT4nRPDGg4JuzQ==",
"@esbuild/linux-ppc64@0.24.2": {
"integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==",
"os": ["linux"],
"cpu": ["ppc64"]
},
"@esbuild/linux-riscv64@0.25.8": {
"integrity": "sha512-rAqDYFv3yzMrq7GIcen3XP7TUEG/4LK86LUPMIz6RT8A6pRIDn0sDcvjudVZBiiTcZCY9y2SgYX2lgK3AF+1eg==",
"@esbuild/linux-riscv64@0.24.2": {
"integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==",
"os": ["linux"],
"cpu": ["riscv64"]
},
"@esbuild/linux-s390x@0.25.8": {
"integrity": "sha512-Xutvh6VjlbcHpsIIbwY8GVRbwoviWT19tFhgdA7DlenLGC/mbc3lBoVb7jxj9Z+eyGqvcnSyIltYUrkKzWqSvg==",
"@esbuild/linux-s390x@0.24.2": {
"integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==",
"os": ["linux"],
"cpu": ["s390x"]
},
"@esbuild/linux-x64@0.25.8": {
"integrity": "sha512-ASFQhgY4ElXh3nDcOMTkQero4b1lgubskNlhIfJrsH5OKZXDpUAKBlNS0Kx81jwOBp+HCeZqmoJuihTv57/jvQ==",
"@esbuild/linux-x64@0.24.2": {
"integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==",
"os": ["linux"],
"cpu": ["x64"]
},
"@esbuild/netbsd-arm64@0.25.8": {
"integrity": "sha512-d1KfruIeohqAi6SA+gENMuObDbEjn22olAR7egqnkCD9DGBG0wsEARotkLgXDu6c4ncgWTZJtN5vcgxzWRMzcw==",
"@esbuild/netbsd-arm64@0.24.2": {
"integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==",
"os": ["netbsd"],
"cpu": ["arm64"]
},
"@esbuild/netbsd-x64@0.25.8": {
"integrity": "sha512-nVDCkrvx2ua+XQNyfrujIG38+YGyuy2Ru9kKVNyh5jAys6n+l44tTtToqHjino2My8VAY6Lw9H7RI73XFi66Cg==",
"@esbuild/netbsd-x64@0.24.2": {
"integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==",
"os": ["netbsd"],
"cpu": ["x64"]
},
"@esbuild/openbsd-arm64@0.25.8": {
"integrity": "sha512-j8HgrDuSJFAujkivSMSfPQSAa5Fxbvk4rgNAS5i3K+r8s1X0p1uOO2Hl2xNsGFppOeHOLAVgYwDVlmxhq5h+SQ==",
"@esbuild/openbsd-arm64@0.24.2": {
"integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==",
"os": ["openbsd"],
"cpu": ["arm64"]
},
"@esbuild/openbsd-x64@0.25.8": {
"integrity": "sha512-1h8MUAwa0VhNCDp6Af0HToI2TJFAn1uqT9Al6DJVzdIBAd21m/G0Yfc77KDM3uF3T/YaOgQq3qTJHPbTOInaIQ==",
"@esbuild/openbsd-x64@0.24.2": {
"integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==",
"os": ["openbsd"],
"cpu": ["x64"]
},
"@esbuild/openharmony-arm64@0.25.8": {
"integrity": "sha512-r2nVa5SIK9tSWd0kJd9HCffnDHKchTGikb//9c7HX+r+wHYCpQrSgxhlY6KWV1nFo1l4KFbsMlHk+L6fekLsUg==",
"os": ["openharmony"],
"cpu": ["arm64"]
},
"@esbuild/sunos-x64@0.25.8": {
"integrity": "sha512-zUlaP2S12YhQ2UzUfcCuMDHQFJyKABkAjvO5YSndMiIkMimPmxA+BYSBikWgsRpvyxuRnow4nS5NPnf9fpv41w==",
"@esbuild/sunos-x64@0.24.2": {
"integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==",
"os": ["sunos"],
"cpu": ["x64"]
},
"@esbuild/win32-arm64@0.25.8": {
"integrity": "sha512-YEGFFWESlPva8hGL+zvj2z/SaK+pH0SwOM0Nc/d+rVnW7GSTFlLBGzZkuSU9kFIGIo8q9X3ucpZhu8PDN5A2sQ==",
"@esbuild/win32-arm64@0.24.2": {
"integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==",
"os": ["win32"],
"cpu": ["arm64"]
},
"@esbuild/win32-ia32@0.25.8": {
"integrity": "sha512-hiGgGC6KZ5LZz58OL/+qVVoZiuZlUYlYHNAmczOm7bs2oE1XriPFi5ZHHrS8ACpV5EjySrnoCKmcbQMN+ojnHg==",
"@esbuild/win32-ia32@0.24.2": {
"integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==",
"os": ["win32"],
"cpu": ["ia32"]
},
"@esbuild/win32-x64@0.25.8": {
"integrity": "sha512-cn3Yr7+OaaZq1c+2pe+8yxC8E144SReCQjN6/2ynubzYjvyqZjTXfQJpAcQpsdJq3My7XADANiYGHoFC69pLQw==",
"@esbuild/win32-x64@0.24.2": {
"integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==",
"os": ["win32"],
"cpu": ["x64"]
},
@@ -434,12 +529,128 @@
"@isaacs/balanced-match"
]
},
"@types/node@22.12.0": {
"integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==",
"@jridgewell/gen-mapping@0.3.13": {
"integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==",
"dependencies": [
"@jridgewell/sourcemap-codec",
"@jridgewell/trace-mapping"
]
},
"@jridgewell/remapping@2.3.5": {
"integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==",
"dependencies": [
"@jridgewell/gen-mapping",
"@jridgewell/trace-mapping"
]
},
"@jridgewell/resolve-uri@3.1.2": {
"integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="
},
"@jridgewell/sourcemap-codec@1.5.5": {
"integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="
},
"@jridgewell/trace-mapping@0.3.31": {
"integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==",
"dependencies": [
"@jridgewell/resolve-uri",
"@jridgewell/sourcemap-codec"
]
},
"@sveltejs/acorn-typescript@1.0.7_acorn@8.14.1": {
"integrity": "sha512-znp1A/Y1Jj4l/Zy7PX5DZKBE0ZNY+5QBngiE21NJkfSTyzzC5iKNWOtwFXKtIrn7MXEFBck4jD95iBNkGjK92Q==",
"dependencies": [
"acorn"
]
},
"@types/diff@5.2.3": {
"integrity": "sha512-K0Oqlrq3kQMaO2RhfrNQX5trmt+XLyom88zS0u84nnIcLvFnRUMRRHmrGny5GSM+kNO9IZLARsdQHDzkhAgmrQ=="
},
"@types/estree@1.0.8": {
"integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="
},
"@types/node@24.2.0": {
"integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==",
"dependencies": [
"undici-types"
]
},
"@types/ws@8.18.1": {
"integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
"dependencies": [
"@types/node"
]
},
"@vue/compiler-core@3.5.25": {
"integrity": "sha512-vay5/oQJdsNHmliWoZfHPoVZZRmnSWhug0BYT34njkYTPqClh3DNWLkZNJBVSjsNMrg0CCrBfoKkjZQPM/QVUw==",
"dependencies": [
"@babel/parser",
"@vue/shared",
"entities",
"estree-walker",
"source-map-js"
]
},
"@vue/compiler-dom@3.5.25": {
"integrity": "sha512-4We0OAcMZsKgYoGlMjzYvaoErltdFI2/25wqanuTu+S4gismOTRTBPi4IASOjxWdzIwrYSjnqONfKvuqkXzE2Q==",
"dependencies": [
"@vue/compiler-core",
"@vue/shared"
]
},
"@vue/compiler-sfc@3.5.25": {
"integrity": "sha512-PUgKp2rn8fFsI++lF2sO7gwO2d9Yj57Utr5yEsDf3GNaQcowCLKL7sf+LvVFvtJDXUp/03+dC6f2+LCv5aK1ag==",
"dependencies": [
"@babel/parser",
"@vue/compiler-core",
"@vue/compiler-dom",
"@vue/compiler-ssr",
"@vue/shared",
"estree-walker",
"magic-string",
"postcss",
"source-map-js"
]
},
"@vue/compiler-ssr@3.5.25": {
"integrity": "sha512-ritPSKLBcParnsKYi+GNtbdbrIE1mtuFEJ4U1sWeuOMlIziK5GtOL85t5RhsNy4uWIXPgk+OUdpnXiTdzn8o3A==",
"dependencies": [
"@vue/compiler-dom",
"@vue/shared"
]
},
"@vue/reactivity@3.5.25": {
"integrity": "sha512-5xfAypCQepv4Jog1U4zn8cZIcbKKFka3AgWHEFQeK65OW+Ys4XybP6z2kKgws4YB43KGpqp5D/K3go2UPPunLA==",
"dependencies": [
"@vue/shared"
]
},
"@vue/runtime-core@3.5.25": {
"integrity": "sha512-Z751v203YWwYzy460bzsYQISDfPjHTl+6Zzwo/a3CsAf+0ccEjQ8c+0CdX1WsumRTHeywvyUFtW6KvNukT/smA==",
"dependencies": [
"@vue/reactivity",
"@vue/shared"
]
},
"@vue/runtime-dom@3.5.25": {
"integrity": "sha512-a4WrkYFbb19i9pjkz38zJBg8wa/rboNERq3+hRRb0dHiJh13c+6kAbgqCPfMaJ2gg4weWD3APZswASOfmKwamA==",
"dependencies": [
"@vue/reactivity",
"@vue/runtime-core",
"@vue/shared",
"csstype"
]
},
"@vue/server-renderer@3.5.25_vue@3.5.25__typescript@4.9.5_typescript@4.9.5": {
"integrity": "sha512-UJaXR54vMG61i8XNIzTSf2Q7MOqZHpp8+x3XLGtE3+fL+nQd+k7O5+X3D/uWrnQXOdMw5VPih+Uremcw+u1woQ==",
"dependencies": [
"@vue/compiler-ssr",
"@vue/shared",
"vue"
]
},
"@vue/shared@3.5.25": {
"integrity": "sha512-AbOPdQQnAnzs58H2FrrDxYj/TJfmeS2jdfEEhgiKINy+bnOANmVizIEgq1r+C5zsbs6l1CCQxtcj71rwNQ4jWg=="
},
"@windmill-labs/shared-utils@1.0.1": {
"integrity": "sha512-DUMzPIFCKImuGpbuHXXmGGUT3VXYlgrv/jIIEOW+Iig+9tZvYqOUxfgn32lDhm73k82xBg8MdAf+0qABzfqFeQ=="
},
@@ -453,6 +664,16 @@
"negotiator"
]
},
"acorn@8.14.1": {
"integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
"bin": true
},
"aria-query@5.3.2": {
"integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw=="
},
"axobject-query@4.1.0": {
"integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ=="
},
"body-parser@2.2.0": {
"integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==",
"dependencies": [
@@ -496,6 +717,9 @@
"windmill-client"
]
},
"clsx@2.1.1": {
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="
},
"content-disposition@1.0.0": {
"integrity": "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==",
"dependencies": [
@@ -514,6 +738,9 @@
"core-util-is@1.0.3": {
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
},
"csstype@3.2.3": {
"integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="
},
"debug@4.4.1": {
"integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
"dependencies": [
@@ -536,6 +763,9 @@
"depd@2.0.0": {
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="
},
"devalue@5.5.0": {
"integrity": "sha512-69sM5yrHfFLJt0AZ9QqZXGCPfJ7fQjvpln3Rq5+PS03LD32Ost1Q9N+eEnaQwGRIriKkMImXD56ocjQmfjbV3w=="
},
"diff@8.0.2": {
"integrity": "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg=="
},
@@ -553,6 +783,9 @@
"encodeurl@2.0.0": {
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="
},
"entities@4.5.0": {
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="
},
"es-define-property@1.0.1": {
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="
},
@@ -568,8 +801,23 @@
"es-errors"
]
},
"esbuild@0.25.8": {
"integrity": "sha512-vVC0USHGtMi8+R4Kz8rt6JhEWLxsv9Rnu/lGYbPR8u47B+DCBksq9JarW0zOO7bs37hyOK1l2/oqtbciutL5+Q==",
"esbuild-plugin-vue3@0.5.1_vue@3.5.25__typescript@4.9.5_typescript@4.9.5": {
"integrity": "sha512-rhTPImJ1Zi7FbVa4xWlu9dJdt+mqWxc9Z+AQd+ArbHHwtyQRe8FvER8gaTw0O6bNsBjAtU5rq0rpZEkP3QaThg==",
"dependencies": [
"typescript",
"vue"
]
},
"esbuild-svelte@0.9.3_esbuild@0.24.2_svelte@5.45.2__acorn@8.14.1": {
"integrity": "sha512-CgEcGY1r/d16+aggec3czoFBEBaYIrFOnMxpsO6fWNaNEqHregPN5DLAPZDqrL7rXDNplW+WMu8s3GMq9FqgJA==",
"dependencies": [
"@jridgewell/trace-mapping",
"esbuild",
"svelte"
]
},
"esbuild@0.24.2": {
"integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==",
"optionalDependencies": [
"@esbuild/aix-ppc64",
"@esbuild/android-arm",
@@ -592,7 +840,6 @@
"@esbuild/netbsd-x64",
"@esbuild/openbsd-arm64",
"@esbuild/openbsd-x64",
"@esbuild/openharmony-arm64",
"@esbuild/sunos-x64",
"@esbuild/win32-arm64",
"@esbuild/win32-ia32",
@@ -604,6 +851,18 @@
"escape-html@1.0.3": {
"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
},
"esm-env@1.2.2": {
"integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA=="
},
"esrap@2.2.0": {
"integrity": "sha512-WBmtxe7R9C5mvL4n2le8nMUe4mD5V9oiK2vJpQ9I3y20ENPUomPcphBXE8D1x/Bm84oN1V+lOfgXxtqmxTp3Xg==",
"dependencies": [
"@jridgewell/sourcemap-codec"
]
},
"estree-walker@2.0.2": {
"integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="
},
"etag@1.8.1": {
"integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="
},
@@ -634,7 +893,7 @@
"router",
"send",
"serve-static",
"statuses@2.0.2",
"statuses",
"type-is",
"vary"
]
@@ -647,7 +906,7 @@
"escape-html",
"on-finished",
"parseurl",
"statuses@2.0.2"
"statuses"
]
},
"forwarded@0.2.0": {
@@ -702,7 +961,7 @@
"depd",
"inherits",
"setprototypeof",
"statuses@2.0.1",
"statuses",
"toidentifier"
]
},
@@ -735,6 +994,12 @@
"is-promise@4.0.0": {
"integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="
},
"is-reference@3.0.3": {
"integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==",
"dependencies": [
"@types/estree"
]
},
"is-wsl@3.1.0": {
"integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==",
"dependencies": [
@@ -768,6 +1033,15 @@
"immediate"
]
},
"locate-character@3.0.0": {
"integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="
},
"magic-string@0.30.21": {
"integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==",
"dependencies": [
"@jridgewell/sourcemap-codec"
]
},
"math-intrinsics@1.1.0": {
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="
},
@@ -795,6 +1069,10 @@
"ms@2.1.3": {
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"nanoid@3.3.11": {
"integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==",
"bin": true
},
"negotiator@1.0.0": {
"integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="
},
@@ -831,6 +1109,17 @@
"path-to-regexp@8.2.0": {
"integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ=="
},
"picocolors@1.1.1": {
"integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="
},
"postcss@8.5.6": {
"integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==",
"dependencies": [
"nanoid",
"picocolors",
"source-map-js"
]
},
"process-nextick-args@2.0.1": {
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
@@ -906,7 +1195,7 @@
"ms",
"on-finished",
"range-parser",
"statuses@2.0.2"
"statuses"
]
},
"serve-static@2.2.0": {
@@ -960,12 +1249,12 @@
"side-channel-weakmap"
]
},
"source-map-js@1.2.1": {
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="
},
"statuses@2.0.1": {
"integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="
},
"statuses@2.0.2": {
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="
},
"string_decoder@1.1.1": {
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": [
@@ -976,6 +1265,33 @@
"integrity": "sha512-Mu7R0g4ig9TUuGSxJavny5Rv0egCEtpZRNMrZaYS1vxkiIxGiGUwoezU3LazIQ+KE04hTrTfNPgxU5gzi7F5Pw==",
"deprecated": true
},
"svelte-preprocess@6.0.3_svelte@5.45.2__acorn@8.14.1": {
"integrity": "sha512-PLG2k05qHdhmRG7zR/dyo5qKvakhm8IJ+hD2eFRQmMLHp7X3eJnjeupUtvuRpbNiF31RjVw45W+abDwHEmP5OA==",
"dependencies": [
"svelte"
],
"scripts": true
},
"svelte@5.45.2_acorn@8.14.1": {
"integrity": "sha512-yyXdW2u3H0H/zxxWoGwJoQlRgaSJLp+Vhktv12iRw2WRDlKqUPT54Fi0K/PkXqrdkcQ98aBazpy0AH4BCBVfoA==",
"dependencies": [
"@jridgewell/remapping",
"@jridgewell/sourcemap-codec",
"@sveltejs/acorn-typescript",
"@types/estree",
"acorn",
"aria-query",
"axobject-query",
"clsx",
"devalue",
"esm-env",
"esrap",
"is-reference",
"locate-character",
"magic-string",
"zimmerframe"
]
},
"toidentifier@1.0.1": {
"integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
},
@@ -987,8 +1303,12 @@
"mime-types"
]
},
"undici-types@6.20.0": {
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
"typescript@4.9.5": {
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"bin": true
},
"undici-types@7.10.0": {
"integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="
},
"unpipe@1.0.0": {
"integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="
@@ -999,12 +1319,29 @@
"vary@1.1.2": {
"integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="
},
"vue@3.5.25_typescript@4.9.5": {
"integrity": "sha512-YLVdgv2K13WJ6n+kD5owehKtEXwdwXuj2TTyJMsO7pSeKw2bfRNZGjhB7YzrpbMYj5b5QsUebHpOqR3R3ziy/g==",
"dependencies": [
"@vue/compiler-dom",
"@vue/compiler-sfc",
"@vue/runtime-dom",
"@vue/server-renderer",
"@vue/shared",
"typescript"
],
"optionalPeers": [
"typescript"
]
},
"windmill-client@1.515.1": {
"integrity": "sha512-o6qynOEbPubZTZUOLLs2Z9f+uBZQJUCw/+YWgvI6p8nu5BJ6J3N/wEfbY1X5TTnJNuqahQ0UgimYzhurT5XQFw=="
},
"wrappy@1.0.2": {
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
},
"ws@8.18.0": {
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw=="
},
"ws@8.18.3": {
"integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="
},
@@ -1014,6 +1351,9 @@
"is-wsl"
]
},
"zimmerframe@1.1.4": {
"integrity": "sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ=="
},
"zod@3.25.76": {
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="
}
@@ -1285,7 +1625,8 @@
"jsr:@windmill-labs/cliffy-command@^1.0.0-rc.5",
"jsr:@windmill-labs/cliffy-prompt@^1.0.0-rc.5",
"jsr:@windmill-labs/cliffy-table@^1.0.0-rc.5",
"npm:@types/diff@^5.2.2"
"npm:@types/diff@^5.2.2",
"npm:ws@8.18.0"
]
}
}
cli/deps.ts
+4 -1
View File
@@ -57,7 +57,10 @@ export { WebSocketServer, WebSocket } from "npm:ws";
export * as getPort from "npm:get-port@7.1.0";
export * as open from "npm:open";
export * as esMain from "npm:es-main";
export * as windmillUtils from "npm:@windmill-labs/shared-utils@1.0.2";
export * as windmillUtils from "jsr:@windmill-labs/shared-utils@1.0.10";
// needed for dnt transform
import * as wsTypes from "npm:@types/ws";
import { OpenAPI } from "./gen/index.ts";
cli/src/commands/app/app_metadata.ts
+463
View File
@@ -0,0 +1,463 @@
// deno-lint-ignore-file no-explicit-any
import path from "node:path";
import {
SEP,
colors,
log,
yamlParseFile,
yamlStringify,
} from "../../../deps.ts";
import { GlobalOptions } from "../../types.ts";
import {
checkifMetadataUptodate,
blueColor,
clearGlobalLock,
updateMetadataGlobalLock,
inferSchema,
getRawWorkspaceDependencies,
} from "../../utils/metadata.ts";
import { ScriptLanguage, workspaceDependenciesLanguages } from "../../utils/script_common.ts";
import {
inferContentTypeFromFilePath,
} from "../../utils/script_common.ts";
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
import { exts } from "../script/script.ts";
import { FSFSElement, yamlOptions } from "../sync/sync.ts";
import { Workspace } from "../workspace/workspace.ts";
import { AppFile } from "./raw_apps.ts";
import { replaceInlineScripts } from "./apps.ts";
import {
newPathAssigner,
SupportedLanguage,
} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
const TOP_HASH = "__app_hash";
/**
* Generates a hash for all inline scripts in an app directory
*/
async function generateAppHash(
rawReqs: Record<string, string> | undefined,
folder: string,
defaultTs: "bun" | "deno" | undefined
): Promise<Record<string, string>> {
const runnablesFolder = path.join(folder, "runnables");
const hashes: Record<string, string> = {};
try {
const elems = await FSFSElement(runnablesFolder, [], true);
for await (const f of elems.getChildren()) {
if (exts.some((e) => f.path.endsWith(e))) {
let reqs: string | undefined;
if (rawReqs) {
// Get language name from path
const lang = inferContentTypeFromFilePath(f.path, defaultTs);
// Get lock for that language
[, reqs] =
Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? [];
}
// Embed lock into hash
const relativePath = f.path.replace(runnablesFolder + SEP, "");
hashes[relativePath] = await generateHash(
(await f.getContentText()) + (reqs ?? "")
);
}
}
} catch (error: any) {
// If runnables folder doesn't exist, that's okay
if (error.name !== "NotFound") {
throw error;
}
}
return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
}
/**
* Updates locks for inline scripts in an app
*/
export async function generateAppLocksInternal(
appFolder: string,
dryRun: boolean,
workspace: Workspace,
opts: GlobalOptions & {
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean,
): Promise<string | void> {
if (appFolder.endsWith(SEP)) {
appFolder = appFolder.substring(0, appFolder.length - 1);
}
const remote_path = appFolder.replaceAll(SEP, "/");
if (!justUpdateMetadataLock && !noStaleMessage) {
log.info(`Generating locks for app ${appFolder} at ${remote_path}`);
}
const rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
let hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs);
const conf = await import("../../utils/metadata.ts").then((m) =>
m.readLockfile()
);
if (
await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)
) {
if (!noStaleMessage) {
log.info(
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
return remote_path;
}
if (Object.keys(rawWorkspaceDependencies).length > 0) {
log.info(
(await blueColor())(
`Found workspace dependencies (${workspaceDependenciesLanguages
.map((l) => l.filename)
.join("/")}) for ${appFolder}, using them`
)
);
}
// Read the app file
const appFilePath = path.join(appFolder, "raw_app.yaml");
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
if (!justUpdateMetadataLock) {
const changedScripts = [];
// Find hashes that do not correspond to previous hashes
for (const [scriptPath, hash] of Object.entries(hashes)) {
if (scriptPath == TOP_HASH) {
continue;
}
if (!(await checkifMetadataUptodate(appFolder, hash, conf, scriptPath))) {
changedScripts.push(scriptPath);
}
}
if (changedScripts.length > 0) {
log.info(
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
);
const runnablesPath = path.join(appFolder, "runnables") + SEP;
// Replace inline scripts for changed runnables
await replaceInlineScripts(appFile.runnables, runnablesPath);
// Update the app runnables with new locks
appFile.runnables = await updateAppRunnables(
workspace,
appFile.runnables,
remote_path,
appFolder,
rawWorkspaceDependencies,
opts.defaultTs
);
// Write the updated app file
writeIfChanged(
appFilePath,
yamlStringify(appFile as Record<string, any>, yamlOptions)
);
} else {
log.info(colors.gray(`No scripts changed in ${appFolder}`));
}
}
// Regenerate hashes after updates
hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs);
await clearGlobalLock(appFolder);
for (const [scriptPath, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(appFolder, hash, scriptPath);
}
log.info(colors.green(`App ${remote_path} lockfiles updated`));
}
/**
* Updates locks for all runnables in an app, generating locks inline script by inline script
* Also writes content and locks back to the runnables folder
*/
async function updateAppRunnables(
workspace: Workspace,
runnables: Record<string, any>,
remotePath: string,
appFolder: string,
rawDeps?: Record<string, string>,
defaultTs: "bun" | "deno" = "bun"
): Promise<Record<string, any>> {
const updatedRunnables = { ...runnables };
const runnablesFolder = path.join(appFolder, "runnables");
// Ensure runnables folder exists
try {
await Deno.mkdir(runnablesFolder, { recursive: true });
} catch {
// Folder may already exist
}
const pathAssigner = newPathAssigner(defaultTs);
for (const [runnableId, runnable] of Object.entries(runnables)) {
// Only process inline scripts (runnableByName with inlineScript)
if (runnable?.type !== "runnableByName" || !runnable?.inlineScript) {
continue;
}
const inlineScript = runnable.inlineScript;
const language = inlineScript.language as SupportedLanguage;
const content = inlineScript.content;
if (!content || !language) {
continue;
}
// Skip if content is still an !inline reference (should have been replaced by replaceInlineScripts)
if (typeof content === "string" && content.startsWith("!inline ")) {
log.warn(
colors.yellow(
`Runnable ${runnableId} content is still an !inline reference, skipping`
)
);
continue;
}
// Skip frontend scripts - they don't need locks
if (language === "frontend") {
continue;
}
// Find raw deps for this language if available
const langRawDeps = rawDeps?.[language];
log.info(
colors.gray(
`Generating lock for runnable ${runnableId} (${language})${
langRawDeps ? " with raw deps" : ""
}`
)
);
try {
const lock = await generateInlineScriptLock(
workspace,
content,
language,
`${remotePath}/${runnableId}`,
langRawDeps
);
// Determine file extension for this language
const [basePathO, ext] = pathAssigner.assignPath(runnable.name, language);
const basePath = basePathO.replaceAll(SEP, "/");
const contentPath = path.join(runnablesFolder, `${basePath}${ext}`);
const lockPath = path.join(runnablesFolder, `${basePath}lock`);
// Write content to file
writeIfChanged(contentPath, content);
// Write lock to file if it exists
if (lock && lock !== "") {
writeIfChanged(lockPath, lock);
}
// Update the runnable with !inline references (preserve existing schema)
const inlineContentRef = `!inline ${basePath}${ext}`;
const inlineLockRef =
lock && lock !== "" ? `!inline ${basePath}lock` : "";
updatedRunnables[runnableId] = {
...runnable,
inlineScript: {
...inlineScript,
content: inlineContentRef,
lock: inlineLockRef,
},
};
log.info(
colors.gray(
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
)
);
} catch (error: any) {
log.error(
colors.red(
`Failed to generate lock for runnable ${runnableId}: ${error.message}`
)
);
// Continue with other runnables even if one fails
}
}
return updatedRunnables;
}
/**
* Generates a lock for a single inline script using the dependencies endpoint
*/
async function generateInlineScriptLock(
workspace: Workspace,
content: string,
language: string,
scriptPath: string,
rawDeps?: string
): Promise<string> {
const extraHeaders = getHeaders();
const rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
...extraHeaders,
},
body: JSON.stringify({
raw_scripts: [
{
raw_code: content,
language: language,
script_path: scriptPath,
},
],
raw_deps: rawDeps,
entrypoint: scriptPath,
}),
}
);
if (!rawResponse.ok) {
const text = await rawResponse.text();
throw new Error(
`Dependency generation failed: ${rawResponse.status} ${rawResponse.statusText}\n${text}`
);
}
let responseText = "reading response failed";
try {
responseText = await rawResponse.text();
const response = JSON.parse(responseText);
const lock = response.lock;
if (lock === undefined) {
if (response?.["error"]?.["message"]) {
throw new Error(
`Failed to generate lockfile: ${response?.["error"]?.["message"]}`
);
}
throw new Error(
`Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`
);
}
return lock ?? "";
} catch (e: any) {
throw new Error(
`Failed to parse dependency response: ${rawResponse.statusText}, ${responseText}, ${e.message}`
);
}
}
/**
* Result of schema inference for a runnable
*/
export interface InferredSchemaResult {
runnableId: string;
schema: any;
}
/**
* Infers schema for a single runnable from its file content.
* Used by dev server to update schema in memory (for wmill.d.ts generation).
* Does NOT write to raw_app.yaml - schema is kept in memory only.
*
* @param appFolder - The folder containing the raw app
* @param runnableFilePath - The path to the changed runnable file (relative to runnables folder)
* @returns The runnable ID and inferred schema, or undefined if inference failed/not applicable
*/
export async function inferRunnableSchemaFromFile(
appFolder: string,
runnableFilePath: string,
): Promise<InferredSchemaResult | undefined> {
// Extract runnable ID from file path (e.g., "myRunnable.inline_script.ts" -> "myRunnable")
const fileName = path.basename(runnableFilePath);
// Skip lock files
if (fileName.endsWith(".lock")) {
return undefined;
}
// Match pattern: {runnableId}.inline_script.{ext}
const match = fileName.match(/^(.+)\.inline_script\.[^.]+$/);
if (!match) {
return undefined;
}
const runnableId = match[1];
// Read the app file to get the language
const appFilePath = path.join(appFolder, "raw_app.yaml");
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
if (!appFile.runnables?.[runnableId]) {
log.warn(colors.yellow(`Runnable ${runnableId} not found in raw_app.yaml`));
return undefined;
}
const runnable = appFile.runnables[runnableId];
// Only process inline scripts
if (!runnable?.inlineScript) {
return undefined;
}
const inlineScript = runnable.inlineScript;
const language = inlineScript.language as SupportedLanguage;
// Read the actual content from the file
const fullFilePath = path.join(appFolder, "runnables", runnableFilePath);
let content: string;
try {
content = await Deno.readTextFile(fullFilePath);
} catch {
log.warn(colors.yellow(`Could not read file: ${fullFilePath}`));
return undefined;
}
// Infer schema from script content
const currentSchema = inlineScript.schema;
const remotePath = appFolder.replaceAll(SEP, "/");
try {
const schemaResult = await inferSchema(
language as ScriptLanguage,
content,
currentSchema,
`${remotePath}/${runnableId}`
);
log.info(colors.green(` Inferred schema for ${runnableId}`));
return {
runnableId: runnableId,
schema: schemaResult.schema,
};
} catch (schemaError: any) {
log.warn(
colors.yellow(
`Failed to infer schema for ${runnableId}: ${schemaError.message}`
)
);
return undefined;
}
}
cli/src/commands/app/apps.ts
+75 -31
View File
@@ -15,6 +15,7 @@ import { ListableApp, Policy } from "../../../gen/types.gen.ts";
import { GlobalOptions, isSuperset } from "../../types.ts";
import { readInlinePathSync } from "../../utils/utils.ts";
import devCommand from "./dev.ts";
export interface AppFile {
value: any;
@@ -25,6 +26,55 @@ export interface AppFile {
const alreadySynced: string[] = [];
function respecializeFields(fields: Record<string, any>) {
Object.entries(fields).forEach(([k, v]) => {
if (typeof v == "object") {
if (v.value !== undefined) {
fields[k] = { value: v.value, type: "static" }
} else if (v.expr !== undefined) {
fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources, type: "javascript" }
}
}
})
}
export function repopulateFields(runnables: Record<string, any>) {
Object.values(runnables).forEach((v) => {
if (typeof v == "object") {
if (v.fields !== undefined) {
respecializeFields(v.fields)
}
}
})
}
export function replaceInlineScripts(rec: any, localPath: string) {
if (!rec) {
return;
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == 'runType') {
rec["type"] = 'path'
} else if (k == "inlineScript" && typeof v == "object") {
rec["type"] = 'inline'
const o: Record<string, any> = v as any;
if (o["content"] && o["content"].startsWith("!inline")) {
const basePath = localPath + o["content"].split(" ")[1];
o["content"] = readInlinePathSync(basePath);
}
if (o["lock"] && o["lock"].startsWith("!inline")) {
const basePath = localPath + o["lock"].split(" ")[1];
o["lock"] = readInlinePathSync(basePath);
}
} else {
replaceInlineScripts(v, localPath);
}
});
}
return [];
}
export function isExecutionModeAnonymous(app: any) {
return app?.["policy"]?.["execution_mode"] == "anonymous";
}
@@ -63,34 +113,8 @@ export async function pushApp(
const path = localPath + "app.yaml";
const localApp = (await yamlParseFile(path)) as AppFile;
function replaceInlineScripts(rec: any) {
if (!rec) {
return;
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
if (o["content"] && o["content"].startsWith("!inline")) {
const basePath = localPath + o["content"].split(" ")[1];
o["content"] = readInlinePathSync(basePath);
}
if (o["lock"] && o["lock"].startsWith("!inline")) {
const basePath = localPath + o["lock"].split(" ")[1];
o["lock"] = readInlinePathSync(basePath);
}
} else {
replaceInlineScripts(v);
}
});
}
return [];
}
replaceInlineScripts(localApp.value);
// console.log(localApp, localApp?.["policy"]);
await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? (localApp.policy ? isExecutionModeAnonymous(localApp) : false));
// console.log(localApp, localApp?.["policy"]);
replaceInlineScripts(localApp.value, localPath);
await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false);
if (app) {
if (isSuperset(localApp, app)) {
log.info(colors.green(`App ${remotePath} is up to date`));
@@ -119,7 +143,11 @@ export async function pushApp(
}
}
async function generatingPolicy(app: any, path: string, publicApp: boolean) {
export async function generatingPolicy(
app: any,
path: string,
publicApp: boolean
) {
log.info(colors.gray(`Generating fresh policy for app ${path}...`));
try {
app.policy = await windmillUtils.updatePolicy(app.value, undefined);
@@ -167,7 +195,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
await requireLogin(opts);
await pushApp(workspace.workspaceId, remotePath, filePath);
log.info(colors.bold.underline.green("Flow pushed"));
log.info(colors.bold.underline.green("App pushed"));
}
const command = new Command()
@@ -175,6 +203,22 @@ const command = new Command()
.action(list as any)
.command("push", "push a local app ")
.arguments("<file_path:string> <remote_path:string>")
.action(push as any);
.action(push as any)
.command("dev", devCommand)
.command(
"generate-locks",
"re-generate the lockfiles for app runnables inline scripts that have changed"
)
.arguments("[app_folder:string]")
.option("--yes", "Skip confirmation prompt")
.option("--dry-run", "Perform a dry run without making changes")
.option(
"--default-ts <runtime:string>",
"Default TypeScript runtime (bun or deno)"
)
.action(async (opts: any, appFolder: string | undefined) => {
const { generateLocksCommand } = await import("./raw_apps.ts");
await generateLocksCommand(opts, appFolder);
});
export default command;
cli/src/commands/app/bundle.ts
+299
View File
@@ -0,0 +1,299 @@
// deno-lint-ignore-file no-explicit-any
import * as fs from "node:fs";
import * as path from "node:path";
import process from "node:process";
import { spawn } from "node:child_process";
import { log, colors } from "../../../deps.ts";
import { windmillUtils } from "../../../deps.ts";
export interface BundleOptions {
entryPoint?: string;
outDir?: string;
sourcemap?: boolean;
minify?: boolean;
production?: boolean;
}
export interface BundleResult {
js: string;
css: string;
}
export const DEFAULT_BUILD_OPTIONS = {
bundle: true,
format: "iife" as const,
platform: "browser" as const,
target: "es2020",
jsx: "automatic" as const,
loader: {
".css": "css" as const,
},
logLevel: "info" as const,
write: true,
};
/**
* Detects which frontend frameworks are present in package.json
*/
export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolean } {
const packageJsonPath = path.join(appDir, "package.json");
if (!fs.existsSync(packageJsonPath)) {
return { svelte: false, vue: false };
}
try {
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
const allDeps = {
...packageJson.dependencies,
...packageJson.devDependencies,
};
return {
svelte: "svelte" in allDeps,
vue: "vue" in allDeps,
};
} catch {
return { svelte: false, vue: false };
}
}
/**
* Creates a Svelte esbuild plugin
* Uses the svelte compiler from the project's node_modules
*/
function createSveltePlugin(appDir: string): any {
return {
name: "svelte",
setup(build: any) {
build.onLoad({ filter: /\.svelte$/ }, async (args: any) => {
// Import svelte compiler from the project's node_modules
const svelte = await import("npm:svelte@5.45.2/compiler");
// Load the file from the file system
const source = await fs.promises.readFile(args.path, "utf8");
const filename = path.relative(process.cwd(), args.path);
// This converts a message in Svelte's format to esbuild's format
const convertMessage = ({ message, start, end }: any) => {
let location;
if (start && end) {
const lineText = source.split(/\r\n|\r|\n/g)[start.line - 1];
const lineEnd = start.line === end.line ? end.column : lineText.length;
location = {
file: filename,
line: start.line,
column: start.column,
length: lineEnd - start.column,
lineText,
};
}
return { text: message, location };
};
// Convert Svelte syntax to JavaScript
try {
const { js, warnings } = svelte.compile(source, { filename });
const contents = js.code + `//# sourceMappingURL=` + js.map.toUrl();
return { contents, warnings: warnings.map(convertMessage) };
} catch (e: any) {
return { errors: [convertMessage(e)] };
}
});
},
};
}
/**
* Creates framework-specific esbuild plugins based on detected dependencies
*/
export async function createFrameworkPlugins(appDir: string): Promise<any[]> {
const frameworks = detectFrameworks(appDir);
const plugins: any[] = [];
if (frameworks.svelte) {
log.info(colors.blue("🔧 Svelte detected, adding svelte plugin..."));
plugins.push(createSveltePlugin(appDir));
}
if (frameworks.vue) {
log.info(colors.blue("🔧 Vue detected, adding vue plugin..."));
throw new Error("Vue plugin not supported yet");
// try {
// const esbuildPluginVue = await import("npm:esbuild-plugin-vue3@0.5.1");
// plugins.push(esbuildPluginVue.default());
// } catch (error: any) {
// log.warn(colors.yellow(`Failed to load vue plugin: ${error.message}`));
// }
}
return plugins;
}
/**
* Ensures node_modules exists in the specified directory
* Runs npm install if node_modules is missing
* @param appDir Directory to check for node_modules (defaults to entry point directory)
*/
export async function ensureNodeModules(appDir?: string): Promise<void> {
const targetDir = appDir ?? process.cwd();
const nodeModulesPath = path.join(targetDir, "node_modules");
if (!fs.existsSync(nodeModulesPath)) {
log.info(colors.yellow("📦 node_modules not found, running npm install..."));
const code = await new Promise<number>((resolve, reject) => {
const npmInstall = spawn("npm", ["install"], {
cwd: targetDir,
stdio: "inherit",
shell: true,
});
npmInstall.on("close", (code) => resolve(code ?? 0));
npmInstall.on("error", reject);
});
if (code !== 0) {
throw new Error(`npm install failed with exit code ${code}`);
}
log.info(colors.green("✅ npm install completed"));
}
}
/**
* Creates an esbuild bundle for the app
* @param options Bundle configuration options
* @returns Bundle result containing JS and CSS blobs
*/
export async function createBundle(
options: BundleOptions = {}
): Promise<BundleResult> {
// Dynamically import esbuild
const esbuild = await import("npm:esbuild@0.24.2");
// Detect frameworks to determine default entry point
const frameworks = detectFrameworks(process.cwd());
const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx";
const entryPoint = options.entryPoint ?? defaultEntry;
const outDir = options.outDir ?? "dist";
const sourcemap = options.sourcemap ?? false;
const minify = options.minify ?? true;
const production = options.production ?? true;
// Verify entry point exists
if (!fs.existsSync(entryPoint)) {
throw new Error(
`Entry point "${entryPoint}" not found. Please ensure the file exists.`
);
}
// Ensure node_modules exists in the app directory
const appDir = path.dirname(entryPoint) || process.cwd();
await ensureNodeModules(appDir);
// Load framework-specific plugins (svelte, vue) based on package.json
const frameworkPlugins = await createFrameworkPlugins(appDir);
// Ensure output directory exists
const distDir = path.join(process.cwd(), outDir);
if (!fs.existsSync(distDir)) {
fs.mkdirSync(distDir, { recursive: true });
}
const outfile = path.join(outDir, "bundle.js");
// log.info("FOO")
// log.info("wmillTs" + JSON.stringify(wmillTs));
// Plugin to provide /wmill.ts as a virtual module
const wmillTs = (windmillUtils.wmillTsRaw as any).default ?? windmillUtils.wmillTsRaw;
const wmillPlugin = {
name: "wmill-virtual",
setup(build: any) {
// Intercept imports of /wmill.ts, /wmill, ./wmill.ts, or ./wmill
build.onResolve({ filter: /^(\.\/|\/)?wmill(\.ts)?$/ }, (args: any) => {
log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`));
return {
path: args.path,
namespace: "wmill-virtual",
};
});
// Provide the virtual module content
build.onLoad({ filter: /.*/, namespace: "wmill-virtual" }, (args: any) => {
log.info(colors.yellow(`[wmill-virtual] Loading virtual module: ${args.path}`));
return {
contents: wmillTs,
loader: "ts",
};
});
},
};
const buildOptions = {
...DEFAULT_BUILD_OPTIONS,
entryPoints: [entryPoint],
outfile,
sourcemap,
minify,
define: {
"process.env.NODE_ENV": production ? '"production"' : '"development"',
},
plugins: [...frameworkPlugins, wmillPlugin],
};
log.info(colors.blue("📦 Building bundle..."));
try {
const result = await esbuild.build(buildOptions);
if (result.errors.length > 0) {
log.error(colors.red("❌ Build failed:"));
result.errors.forEach((error: any) => {
log.error(colors.red(error.text));
});
throw new Error("Build failed with errors");
}
log.info(colors.green("✅ Bundle created successfully"));
// Read the generated files
const jsPath = path.join(process.cwd(), outfile);
const cssPath = path.join(process.cwd(), outDir, "bundle.css");
if (!fs.existsSync(jsPath)) {
throw new Error(`Expected JS bundle at ${jsPath} but file not found`);
}
const jsContent = fs.readFileSync(jsPath, "utf-8");
const cssContent = fs.existsSync(cssPath)
? fs.readFileSync(cssPath, "utf-8")
: "";
try {
fs.rmSync(distDir, { recursive: true });
} catch {
//ignore
}
return { js: jsContent, css: cssContent };
} finally {
// Stop esbuild
await esbuild.stop();
}
}
/**
* Gets the esbuild build options for use in watch mode (dev server)
* @param entryPoint Entry point file
* @returns esbuild build options
*/
export function getDevBuildOptions(entryPoint: string = "index.tsx") {
return {
...DEFAULT_BUILD_OPTIONS,
entryPoints: [entryPoint],
outfile: "dist/bundle.js",
sourcemap: true,
define: {
"process.env.NODE_ENV": '"development"',
},
};
}
cli/src/commands/app/dev.ts
+796
View File
@@ -0,0 +1,796 @@
// deno-lint-ignore-file no-explicit-any
import {
Command,
colors,
log,
getPort,
open,
windmillUtils,
yamlParseFile,
} from "../../../deps.ts";
import { GlobalOptions } from "../../types.ts";
import * as http from "node:http";
import * as fs from "node:fs";
import * as path from "node:path";
import process from "node:process";
import { Buffer } from "node:buffer";
import { writeFileSync } from "node:fs";
import { WebSocketServer, WebSocket } from "npm:ws";
import { getDevBuildOptions, ensureNodeModules, createFrameworkPlugins, detectFrameworks } from "./bundle.ts";
import { wmillTsDev as wmillTs } from "./wmillTsDev.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts";
import { replaceInlineScripts } from "./apps.ts";
import { Runnable } from "./metadata.ts";
import { inferRunnableSchemaFromFile } from "./app_metadata.ts";
const DEFAULT_PORT = 4000;
const DEFAULT_HOST = "localhost";
// HTML template with live reload
const createHTML = (jsPath: string, cssPath: string) => `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Windmill App Dev Preview</title>
<link rel="stylesheet" href="${cssPath}">
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
#root {
width: 100%;
height: 100vh;
}
</style>
</head>
<body>
<div id="root"></div>
<script src="${jsPath}"></script>
<script>
// Live reload via EventSource
const evtSource = new EventSource('/__events');
evtSource.addEventListener('change', () => {
console.log('🔄 Files changed, reloading...');
location.reload();
});
evtSource.addEventListener('error', () => {
console.log('⚠️ Lost connection to dev server, retrying...');
});
</script>
</body>
</html>
`;
interface DevOptions extends GlobalOptions {
port?: number;
host?: string;
entry?: string;
open?: boolean;
}
async function dev(opts: DevOptions) {
GLOBAL_CONFIG_OPT.noCdToRoot = true;
// Validate that we're in a .raw_app folder
const cwd = process.cwd();
const currentDirName = path.basename(cwd);
if (!currentDirName.endsWith(".raw_app")) {
log.error(
colors.red(
`Error: The dev command must be run inside a .raw_app folder.\n` +
`Current directory: ${currentDirName}\n` +
`Please navigate to a folder ending with '.raw_app' before running this command.`
)
);
Deno.exit(1);
}
// Check for raw_app.yaml
const rawAppPath = path.join(cwd, "raw_app.yaml");
if (!fs.existsSync(rawAppPath)) {
log.error(
colors.red(
`Error: raw_app.yaml not found in current directory.\n` +
`The dev command must be run in a .raw_app folder containing a raw_app.yaml file.`
)
);
Deno.exit(1);
}
// Resolve workspace and authenticate
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const workspaceId = workspace.workspaceId;
// Load app path from raw_app.yaml
const rawApp = (await yamlParseFile(rawAppPath)) as any;
const appPath = rawApp?.custom_path ?? "u/unknown/newapp";
// Dynamically import esbuild only when the dev command is called
const esbuild = await import("npm:esbuild@0.24.2");
const port =
opts.port ??
(await getPort.default({
port: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map((p) => p + DEFAULT_PORT),
}));
const host = opts.host ?? DEFAULT_HOST;
const shouldOpen = opts.open ?? true;
// Detect frameworks to determine default entry point
const frameworks = detectFrameworks(process.cwd());
const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx";
const entryPoint = opts.entry ?? defaultEntry;
// Verify entry point exists
if (!fs.existsSync(entryPoint)) {
log.error(
colors.red(
`Entry point "${entryPoint}" not found. Please specify a valid entry point with --entry.`
)
);
Deno.exit(1);
}
// Ensure node_modules exists
const appDir = path.dirname(entryPoint) || process.cwd();
await ensureNodeModules(appDir);
// In-memory cache of inferred schemas (runnableId -> schema)
// Used to generate wmill.d.ts without modifying raw_app.yaml
const inferredSchemas: Record<string, any> = {};
genRunnablesTs(inferredSchemas);
// Ensure dist directory exists
const distDir = path.join(process.cwd(), "dist");
if (!fs.existsSync(distDir)) {
fs.mkdirSync(distDir);
}
// SSE clients for live reload
const clients: http.ServerResponse[] = [];
function notifyClients() {
clients.forEach((client) => {
client.write(`event: change\ndata: reload\n\n`);
});
}
const buildOptions = getDevBuildOptions(entryPoint);
// Load framework-specific plugins (svelte, vue) based on package.json
const frameworkPlugins = await createFrameworkPlugins(appDir);
const wmillPlugin = {
name: "wmill-virtual",
setup(build: any) {
// Intercept imports of /wmill.ts, /wmill, ./wmill.ts, or ./wmill
build.onResolve({ filter: /^(\.\/|\/)?wmill(\.ts)?$/ }, (args: any) => {
log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`));
return {
path: args.path,
namespace: "wmill-virtual",
};
});
// Provide the virtual module content
build.onLoad(
{ filter: /.*/, namespace: "wmill-virtual" },
(args: any) => {
log.info(
colors.yellow(
`[wmill-virtual] Loading virtual module: ${args.path}`
)
);
return {
contents: wmillTs(port),
loader: "ts",
};
}
);
},
};
// Create esbuild context
const ctx = await esbuild.context({
...buildOptions,
plugins: [
...frameworkPlugins,
{
name: "notify-on-rebuild",
setup(build: any) {
build.onEnd((result: any) => {
if (result.errors.length === 0) {
log.info(
colors.green("✅ Build succeeded, notifying clients...")
);
notifyClients();
} else {
log.error(colors.red("❌ Build failed:"));
result.errors.forEach((error: any) => {
log.error(colors.red(error.text));
});
}
});
},
},
wmillPlugin,
],
});
// Start watching
await ctx.watch();
log.info(colors.blue("👀 Watching for file changes...\n"));
// Initial build
await ctx.rebuild();
// Watch runnables folder for changes
const runnablesPath = path.join(process.cwd(), "runnables");
let runnablesWatcher: Deno.FsWatcher | undefined;
if (fs.existsSync(runnablesPath)) {
log.info(
colors.blue(`👁️ Watching runnables folder at: ${runnablesPath}\n`)
);
runnablesWatcher = Deno.watchFs(runnablesPath);
// Per-file debounce timeouts for schema inference (longer debounce for typing)
const schemaInferenceTimeouts: Record<string, NodeJS.Timeout> = {};
const SCHEMA_DEBOUNCE_MS = 500; // Wait 500ms after last change before inferring schema
// Handle runnables file changes in the background
(async () => {
try {
for await (const event of runnablesWatcher!) {
// Process each changed path with individual debouncing
for (const changedPath of event.paths) {
const relativePath = path.relative(process.cwd(), changedPath);
const relativeToRunnables = path.relative(runnablesPath, changedPath);
// Skip non-modify events for schema inference
if (event.kind !== "modify" && event.kind !== "create") {
continue;
}
// Skip lock files
if (changedPath.endsWith(".lock")) {
continue;
}
// Log the change event
log.info(
colors.cyan(`📝 Runnable changed [${event.kind}]: ${relativePath}`)
);
// Debounce schema inference per file (wait for typing to finish)
if (schemaInferenceTimeouts[changedPath]) {
clearTimeout(schemaInferenceTimeouts[changedPath]);
}
schemaInferenceTimeouts[changedPath] = setTimeout(async () => {
delete schemaInferenceTimeouts[changedPath];
try {
log.info(colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`));
// Infer schema for this runnable (returns schema in memory, doesn't write to file)
const result = await inferRunnableSchemaFromFile(
process.cwd(),
relativeToRunnables
);
if (result) {
// log.info(colors.green(` Schema: ${JSON.stringify(result.schema, null, 2)}`));
// log.info(colors.green(` Runnable ID: ${result.runnableId}`));
// Store inferred schema in memory
inferredSchemas[result.runnableId] = result.schema;
log.info(colors.green(` Inferred Schemas: ${JSON.stringify(inferredSchemas, null, 2)}`));
// Regenerate wmill.d.ts with updated schema from memory
await genRunnablesTs(inferredSchemas);
}
} catch (error: any) {
log.error(
colors.red(`Error inferring schema: ${error.message}`)
);
}
}, SCHEMA_DEBOUNCE_MS);
}
}
} catch (error: any) {
if (error.name !== "Interrupted") {
log.error(colors.red(`Error watching runnables: ${error.message}`));
}
}
})();
} else {
log.info(
colors.gray(
"ℹ️ No runnables folder found (will not watch for runnable changes)\n"
)
);
}
// Create HTTP server
const server = http.createServer((req, res) => {
const url = req.url || "/";
// SSE endpoint for live reload
if (url === "/__events") {
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
Connection: "keep-alive",
});
res.write("data: connected\n\n");
clients.push(res);
req.on("close", () => {
const index = clients.indexOf(res);
if (index !== -1) clients.splice(index, 1);
});
return;
}
// Serve the bundled JS
if (url === "/dist/bundle.js" || url === "/bundle.js") {
const jsPath = path.join(process.cwd(), "dist/bundle.js");
if (fs.existsSync(jsPath)) {
res.writeHead(200, { "Content-Type": "application/javascript" });
res.end(fs.readFileSync(jsPath));
} else {
res.writeHead(404);
res.end("Bundle not found");
}
return;
}
// Serve the bundled CSS
if (url === "/dist/bundle.css" || url === "/bundle.css") {
const cssPath = path.join(process.cwd(), "dist/bundle.css");
if (fs.existsSync(cssPath)) {
res.writeHead(200, { "Content-Type": "text/css" });
res.end(fs.readFileSync(cssPath));
} else {
res.writeHead(404);
res.end("CSS not found");
}
return;
}
// Serve source maps
if (url === "/dist/bundle.js.map" || url === "/bundle.js.map") {
const mapPath = path.join(process.cwd(), "dist/bundle.js.map");
if (fs.existsSync(mapPath)) {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(fs.readFileSync(mapPath));
} else {
res.writeHead(404);
res.end("Source map not found");
}
return;
}
if (url === "/dist/bundle.css.map" || url === "/bundle.css.map") {
const mapPath = path.join(process.cwd(), "dist/bundle.css.map");
if (fs.existsSync(mapPath)) {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(fs.readFileSync(mapPath));
} else {
res.writeHead(404);
res.end("Source map not found");
}
return;
}
// Serve injected HTML for root and any other path
res.writeHead(200, { "Content-Type": "text/html" });
res.end(createHTML("/dist/bundle.js", "/dist/bundle.css"));
});
// Create WebSocket server on the same HTTP server
const wss = new WebSocketServer({ server });
wss.on("connection", (ws: WebSocket) => {
log.info(colors.cyan("[WebSocket] Client connected"));
ws.on("message", async (data: Buffer) => {
try {
const message = JSON.parse(data.toString());
log.info(
colors.cyan(`[WebSocket] Received: ${JSON.stringify(message)}`)
);
const { type, reqId, runnable_id, v, jobId } = message;
// Helper to send response
const respond = (responseType: string, result: any, error: boolean) => {
ws.send(JSON.stringify({ type: responseType, reqId, result, error }));
};
// Helper to execute and wait for result
const runAndWaitForResult = async (
runnableId: string,
args: any
): Promise<{ uuid: string; result: any }> => {
const runnables = await loadRunnables();
const runnable = runnables[runnableId];
if (!runnable) {
throw new Error(`Runnable not found: ${runnableId}`);
}
const uuid = await executeRunnable(
runnable,
workspaceId,
appPath,
runnableId,
args
);
log.info(colors.gray(`[backend] Job started: ${uuid}`));
const result = await waitForJob(workspaceId, uuid);
return { uuid, result };
};
switch (type) {
case "backend": {
// Run a runnable synchronously and wait for result
log.info(colors.blue(`[backend] Running runnable: ${runnable_id}`));
try {
const { result } = await runAndWaitForResult(runnable_id, v);
respond("backendRes", result, false);
} catch (error: any) {
log.error(colors.red(`[backend] Error: ${error.message}`));
respond(
"backendRes",
{ message: error.message, stack: error.stack },
true
);
}
break;
}
case "backendAsync": {
// Run a runnable asynchronously and return job ID immediately
log.info(
colors.blue(`[backendAsync] Running runnable async: ${runnable_id}`)
);
try {
const runnables = await loadRunnables();
const runnable = runnables[runnable_id];
if (!runnable) {
throw new Error(`Runnable not found: ${runnable_id}`);
}
const uuid = await executeRunnable(
runnable,
workspaceId,
appPath,
runnable_id,
v
);
log.info(colors.gray(`[backendAsync] Job started: ${uuid}`));
// Return job ID immediately
respond("backendAsyncRes", uuid, false);
// Wait for result in the background and send it when done
waitForJob(workspaceId, uuid)
.then((result) => {
respond("backendRes", result, false);
})
.catch((error: any) => {
respond(
"backendRes",
{ message: error.message, stack: error.stack },
true
);
});
} catch (error: any) {
log.error(colors.red(`[backendAsync] Error: ${error.message}`));
respond(
"backendAsyncRes",
{ message: error.message, stack: error.stack },
true
);
}
break;
}
case "waitJob": {
// Wait for a job to complete and return its result
log.info(colors.blue(`[waitJob] Waiting for job: ${jobId}`));
try {
const result = await waitForJob(workspaceId, jobId);
respond("backendRes", result, false);
} catch (error: any) {
log.error(colors.red(`[waitJob] Error: ${error.message}`));
respond(
"backendRes",
{ message: error.message, stack: error.stack },
true
);
}
break;
}
case "getJob": {
// Get the current status/result of a job
log.info(colors.blue(`[getJob] Getting job status: ${jobId}`));
try {
const result = await getJobStatus(workspaceId, jobId);
respond("backendRes", result, false);
} catch (error: any) {
log.error(colors.red(`[getJob] Error: ${error.message}`));
respond(
"backendRes",
{ message: error.message, stack: error.stack },
true
);
}
break;
}
default:
log.warn(
colors.yellow(`[WebSocket] Unknown message type: ${type}`)
);
respond(
"error",
{ message: `Unknown message type: ${type}` },
true
);
}
} catch (error: any) {
log.error(
colors.red(`[WebSocket] Failed to parse message: ${error.message}`)
);
}
});
ws.on("close", () => {
log.info(colors.cyan("[WebSocket] Client disconnected"));
});
ws.on("error", (error: Error) => {
log.error(colors.red(`[WebSocket] Error: ${error.message}`));
});
});
server.listen(port, host, () => {
const url = `http://${host}:${port}`;
log.info(colors.bold.green(`🚀 Dev server running at ${url}`));
log.info(
colors.cyan(`🔌 WebSocket server running at ws://${host}:${port}`)
);
log.info(colors.gray(`📦 Serving files from: ${process.cwd()}`));
log.info(colors.gray(`🔄 Live reload enabled\n`));
// Open browser if requested
if (shouldOpen) {
try {
open
.openApp(open.apps.browser, { arguments: [url] })
.catch((error: any) => {
log.error(
colors.yellow(
`Failed to open browser automatically: ${error.message}`
)
);
});
log.info(colors.gray("Opened browser for you"));
} catch (error: any) {
log.error(colors.yellow(`Failed to open browser: ${error.message}`));
}
}
});
// Graceful shutdown
process.on("SIGINT", async () => {
log.info(colors.yellow("\n\n🛑 Shutting down..."));
clients.forEach((client) => client.end());
server.close();
// Close runnables watcher if it exists
if (runnablesWatcher) {
runnablesWatcher.close();
}
await ctx.dispose();
process.exit(0);
});
}
const command = new Command()
.description(
"Start a development server for building apps with live reload and hot module replacement"
)
.option(
"--port <port:number>",
"Port to run the dev server on (will find next available port if occupied)"
)
.option("--host <host:string>", "Host to bind the dev server to", {
default: DEFAULT_HOST,
})
.option("--entry <entry:string>", "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)")
.option("--no-open", "Don't automatically open the browser")
.action(dev as any);
export default command;
/**
* Generates wmill.d.ts with type definitions for runnables.
* Merges in-memory inferred schemas with runnables from raw_app.yaml.
*
* @param schemaOverrides - In-memory schema overrides (runnableId -> schema)
*/
async function genRunnablesTs(schemaOverrides: Record<string, any> = {}) {
log.info(colors.blue("🔄 Generating wmill.d.ts..."));
const rawApp = (await yamlParseFile(
path.join(process.cwd(), "raw_app.yaml")
)) as any;
const runnables = rawApp?.["runnables"] as any;
// Apply schema overrides from in-memory cache
if (runnables && Object.keys(schemaOverrides).length > 0) {
for (const [runnableId, schema] of Object.entries(schemaOverrides)) {
if (runnables[runnableId]?.inlineScript) {
runnables[runnableId].inlineScript.schema = schema;
runnables[runnableId].type = "inline";
}
}
}
try {
const newWmillTs = windmillUtils.genWmillTs(runnables);
writeFileSync(path.join(process.cwd(), "wmill.d.ts"), newWmillTs);
} catch (error: any) {
log.error(colors.red(`Failed to generate wmill.d.ts: ${error.message}`));
}
}
async function loadRunnables(): Promise<Record<string, Runnable>> {
try {
const localPath = process.cwd();
const rawApp = (await yamlParseFile(
path.join(localPath, "raw_app.yaml")
)) as any;
replaceInlineScripts(rawApp.runnables, path.join(localPath, "runnables/"));
return rawApp?.runnables ?? {};
} catch (error: any) {
log.error(colors.red(`Failed to load runnables: ${error.message}`));
return {};
}
}
async function executeRunnable(
runnable: Runnable,
workspace: string,
appPath: string,
runnableId: string,
args: any
): Promise<string> {
const requestBody: any = {
component: runnableId,
args,
force_viewer_static_fields: {},
force_viewer_one_of_fields: {},
force_viewer_allow_user_resources: [],
};
// Handle static fields
if (runnable.fields) {
for (const [key, field] of Object.entries(runnable.fields)) {
if (field?.type === "static") {
requestBody.force_viewer_static_fields[key] = field.value;
}
if (field?.type === "user" && field?.allowUserResources) {
requestBody.force_viewer_allow_user_resources.push(key);
}
}
}
if ((runnable.type === "inline" || runnable.type === "runnableByName") && runnable.inlineScript) {
const inlineScript = runnable.inlineScript;
if (inlineScript.id !== undefined) {
requestBody.id = inlineScript.id;
}
requestBody.raw_code = {
content: inlineScript.id === undefined ? inlineScript.content : "",
language: inlineScript.language ?? "",
path: `${appPath}/${runnableId}`,
lock: inlineScript.id === undefined ? inlineScript.lock : undefined,
cache_ttl: inlineScript.cache_ttl,
};
} else if ((runnable.type === "path" || runnable.type === "runnableByPath") && runnable.path) {
const runType = runnable.runType ?? "script";
requestBody.path =
runType !== "hubscript"
? `${runType}/${runnable.path}`
: `script/${runnable.path}`;
}
const uuid = await wmill.executeComponent({
workspace,
path: appPath,
requestBody,
});
return uuid;
}
const ITERATIONS_BEFORE_SLOW_REFRESH = 10;
const ITERATIONS_BEFORE_SUPER_SLOW_REFRESH = 100;
async function waitForJob(workspace: string, jobId: string): Promise<any> {
if (!jobId) {
throw new Error("Job ID is required");
}
let syncIteration = 0;
return new Promise((resolve, reject) => {
async function checkJob() {
try {
const maybeJob = await wmill.getCompletedJobResultMaybe({
workspace,
id: jobId,
getStarted: false,
});
if (maybeJob.completed) {
if (
!maybeJob.success &&
typeof maybeJob.result === "object" &&
maybeJob.result !== null &&
"error" in maybeJob.result
) {
reject((maybeJob.result as any).error);
} else {
resolve(maybeJob.result);
}
return;
}
} catch (err: any) {
log.error(colors.red(`Error checking job ${jobId}: ${err.message}`));
}
syncIteration++;
let nextIteration = 50;
if (syncIteration > ITERATIONS_BEFORE_SUPER_SLOW_REFRESH) {
nextIteration = 2000;
} else if (syncIteration > ITERATIONS_BEFORE_SLOW_REFRESH) {
nextIteration = 500;
}
setTimeout(checkJob, nextIteration);
}
checkJob();
});
}
async function getJobStatus(workspace: string, jobId: string): Promise<any> {
return await wmill.getJob({
workspace,
id: jobId,
});
}
cli/src/commands/app/metadata.ts
+21
View File
@@ -0,0 +1,21 @@
export type Runnable =
| {
name: string;
type?: "runnableByName" | "inline";
path?: string;
inlineScript?: {
content: string;
language: string;
lock?: string;
cache_ttl?: number;
id?: number;
};
fields?: Record<string, any>;
}
| {
type: "runnableByPath" | "path";
path: string;
runType?: "script" | "flow" | "hubscript";
fields?: Record<string, any>;
schema?: any;
};
cli/src/commands/app/raw_apps.ts
+295
View File
@@ -0,0 +1,295 @@
// deno-lint-ignore-file no-explicit-any
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace, validatePath } from "../../core/context.ts";
import {
colors,
log,
SEP,
windmillUtils,
yamlParseFile,
} from "../../../deps.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { Policy } from "../../../gen/types.gen.ts";
import { GlobalOptions, isSuperset } from "../../types.ts";
import { replaceInlineScripts, repopulateFields } from "./apps.ts";
import { createBundle, detectFrameworks } from "./bundle.ts";
import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts";
export interface AppFile {
runnables: any;
custom_path: string;
public?: boolean;
summary: string;
policy: Policy;
}
const alreadySynced: string[] = [];
async function collectAppFiles(
localPath: string
): Promise<Record<string, string>> {
const files: Record<string, string> = {};
async function readDirRecursive(dir: string, basePath: string = "/") {
for await (const entry of Deno.readDir(dir)) {
const fullPath = dir + entry.name;
const relativePath = basePath + entry.name;
if (entry.isDirectory) {
// Skip the runnables and node_modules subfolders
if (entry.name === "runnables" || entry.name === "node_modules" || entry.name === "dist" || entry.name === ".claude") {
continue;
}
await readDirRecursive(fullPath + SEP, relativePath + SEP);
} else if (entry.isFile) {
// Skip raw_app.yaml as it's metadata, not an app file
// Skip node_modules and package-lock.json as they are generated
if (
relativePath === "raw_app.yaml" ||
relativePath === "package-lock.json"
) {
continue;
}
const content = await Deno.readTextFile(fullPath);
files[relativePath] = content;
}
}
}
await readDirRecursive(localPath);
return files;
}
export async function pushRawApp(
workspace: string,
remotePath: string,
localPath: string,
message?: string
): Promise<void> {
if (alreadySynced.includes(localPath)) {
return;
}
alreadySynced.push(localPath);
remotePath = remotePath.replaceAll(SEP, "/");
let app: any = undefined;
// deleting old app if it exists in raw mode
try {
app = await wmill.getAppByPath({
workspace,
path: remotePath,
});
} catch {
//ignore
}
if (app?.["policy"]?.["execution_mode"] == "anonymous") {
app.public = true;
}
// console.log(app);
if (app) {
app.policy = undefined;
}
if (!localPath.endsWith(SEP)) {
localPath += SEP;
}
const path = localPath + "raw_app.yaml";
const localApp = (await yamlParseFile(path)) as AppFile;
replaceInlineScripts(localApp.runnables, localPath + SEP + "runnables/");
repopulateFields(localApp.runnables)
await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false);
const files = await collectAppFiles(localPath);
async function createBundleRaw() {
log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`));
// Detect frameworks to determine entry point
const frameworks = detectFrameworks(localPath);
const entryFile = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx";
const entryPoint = localPath + entryFile;
return await createBundle({
entryPoint: entryPoint,
production: true,
minify: true,
});
}
if (app) {
if (isSuperset(localApp, app)) {
log.info(colors.green(`App ${remotePath} is up to date`));
return;
}
const { js, css } = await createBundleRaw();
log.info(colors.bold.yellow(`Updating app ${remotePath}...`));
await wmill.updateAppRaw({
workspace,
path: remotePath,
formData: {
app: {
value: { runnables: localApp.runnables, files },
path: remotePath,
summary: localApp.summary,
policy: localApp.policy,
deployment_message: message,
custom_path: localApp.custom_path,
},
js,
css,
},
});
} else {
const { js, css } = await createBundleRaw();
await wmill.createAppRaw({
workspace,
formData: {
app: {
value: { runnables: localApp.runnables, files },
path: remotePath,
summary: localApp.summary,
policy: localApp.policy,
deployment_message: message,
custom_path: localApp.custom_path,
},
js,
css,
},
});
// await wmill.createApp({
// workspace,
// requestBody: {
// path: remotePath,
// deployment_message: message,
// value: { runnables: localApp.runnables, files },
// summary: localApp.summary,
// policy: localApp.policy,
// },
// });
}
}
export async function generatingPolicy(
app: any,
path: string,
publicApp: boolean
) {
log.info(colors.gray(`Generating fresh policy for app ${path}...`));
try {
app.policy = await windmillUtils.updateRawAppPolicy(
app.runnables,
app.policy
);
app.policy.execution_mode = publicApp ? "anonymous" : "publisher";
} catch (e) {
log.error(colors.red(`Error generating policy for app ${path}: ${e}`));
throw e;
}
}
export async function generateLocksCommand(
opts: GlobalOptions & {
yes?: boolean;
dryRun?: boolean;
defaultTs?: "bun" | "deno";
} & SyncOptions,
appPath: string | undefined
) {
const { generateAppLocksInternal } = await import("./app_metadata.ts");
const { elementsToMap, FSFSElement } = await import("../sync/sync.ts");
const { ignoreF } = await import("../sync/sync.ts");
const { Confirm } = await import("../../../deps.ts");
if (appPath == "") {
appPath = undefined;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
if (appPath) {
// Generate metadata for a specific app
await generateAppLocksInternal(
appPath,
false,
workspace,
opts,
false,
false,
);
} else {
// Generate metadata for all apps
const ignore = await ignoreF(opts);
const elems = await elementsToMap(
await FSFSElement(Deno.cwd(), [], true),
(p, isD) => {
return ignore(p, isD) || (!isD && !p.endsWith(SEP + "raw_app.yaml"));
},
false,
{}
);
const appFolders = Object.keys(elems)
.filter((p) => p.endsWith(SEP + "raw_app.yaml"))
.map((p) => p.substring(0, p.length - (SEP + "raw_app.yaml").length));
let hasAny = false;
log.info("Checking metadata for all apps:");
for (const appFolder of appFolders) {
const candidate = await generateAppLocksInternal(
appFolder,
true,
workspace,
opts,
false,
true,
);
if (candidate) {
hasAny = true;
log.info(colors.green(`+ ${candidate}`));
}
}
if (hasAny) {
if (opts.dryRun) {
log.info(colors.gray(`Dry run complete.`));
return;
}
if (
!opts.yes &&
!(await Confirm.prompt({
message: "Update the metadata of the above apps?",
default: true,
}))
) {
return;
}
} else {
log.info(colors.green.bold("No metadata to update"));
return;
}
for (const appFolder of appFolders) {
await generateAppLocksInternal(
appFolder,
false,
workspace,
opts,
false,
true,
);
}
}
}
async function pushRawAppCommand(
opts: GlobalOptions,
filePath: string,
remotePath: string
) {
if (!validatePath(remotePath)) {
return;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
await pushRawApp(workspace.workspaceId, remotePath, filePath);
log.info(colors.bold.underline.green("Raw app pushed"));
}
cli/src/commands/app/wmillTsDev.ts
+87
View File
@@ -0,0 +1,87 @@
//comment this line and last to dev
export function wmillTsDev(port: number) { return `
let reqs: Record<string, any> = {}
let ws: WebSocket | null = null
let wsReady: Promise<void>
let wsReadyResolve: () => void
function initWebSocket() {
wsReady = new Promise((resolve) => {
wsReadyResolve = resolve
})
ws = new WebSocket('ws://localhost:${port}')
ws.onopen = () => {
console.log('[wmill] WebSocket connected')
wsReadyResolve()
}
ws.onmessage = (event) => {
const data = JSON.parse(event.data)
if (data.type === 'backendRes' || data.type === 'backendAsyncRes') {
console.log('Message from WebSocket backend', data)
const job = reqs[data.reqId]
if (job) {
const result = data.result
if (data.error) {
job.reject(new Error(result.stack ?? result.message))
} else {
job.resolve(result)
}
delete reqs[data.reqId]
} else {
console.error('No job found for', data.reqId)
}
}
}
ws.onerror = (error) => {
console.error('[wmill] WebSocket error:', error)
}
ws.onclose = () => {
console.log('[wmill] WebSocket closed, reconnecting...')
setTimeout(initWebSocket, 1000)
}
}
initWebSocket()
async function doRequest(type: string, o: object) {
await wsReady
return new Promise((resolve, reject) => {
const reqId = Math.random().toString(36)
reqs[reqId] = { resolve, reject }
ws?.send(JSON.stringify({ ...o, type, reqId }))
})
}
export const backend = new Proxy(
{},
{
get(_, runnable_id: string) {
return (v: any) => {
return doRequest('backend', { runnable_id, v })
}
}
})
export const backendAsync = new Proxy(
{},
{
get(_, runnable_id: string) {
return (v: any) => {
return doRequest('backendAsync', { runnable_id, v })
}
}
})
export function waitJob(jobId: string) {
return doRequest('waitJob', { jobId })
}
export function getJob(jobId: string) {
return doRequest('getJob', { jobId })
}
`}
cli/src/commands/dependencies/dependencies.ts
+105
View File
@@ -0,0 +1,105 @@
// deno-lint-ignore-file no-explicit-any
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { GlobalOptions } from "../../types.ts";
import { colors, Command, log } from "../../../deps.ts";
import * as wmill from "../../../gen/services.gen.ts";
import type { ScriptLang } from "../../../gen/types.gen.ts";
import fs from "node:fs";
import { generateHash } from "../../utils/utils.ts";
import { checkifMetadataUptodate, updateMetadataGlobalLock, workspaceDependenciesPathToLanguageAndFilename } from "../../utils/metadata.ts";
async function push(
opts: GlobalOptions,
filePath: string,
language?: ScriptLang,
name?: string
): Promise<void> {
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
if (!fs.existsSync(filePath)) {
throw new Error(`File not found: ${filePath}`);
}
const content = fs.readFileSync(filePath, "utf8");
// Use the existing pushWorkspaceDependencies function
await pushWorkspaceDependencies(workspace.workspaceId, filePath, null, content);
}
const command = new Command()
.alias("deps")
.description("workspace dependencies related commands")
.command(
"push",
"Push workspace dependencies from a local file"
)
.arguments("<file_path:string>")
.option(
"--language <language:string>",
"Programming language (python3, typescript, go, php). If not specified, will be inferred from file extension."
)
.option(
"--name <name:string>",
"Name for the dependencies. If not specified, creates workspace default dependencies."
)
.action(push as any);
export async function pushWorkspaceDependencies(
workspace: string,
path: string,
_befObj: any,
newDependenciesContent: string
): Promise<void> {
try {
let res = workspaceDependenciesPathToLanguageAndFilename(path);
if (!res) {
throw new Error(`Unknown workspace dependencies file format: ${path}`);
}
let {
language,
name
} = res;
// TODO: include workspace?
// Generate hash for workspace dependencies content and metadata
const contentHash = await generateHash(newDependenciesContent + path);
// Check if dependencies are up-to-date using wmill-lock.yaml tracking
const isUpToDate = await checkifMetadataUptodate(path, contentHash, undefined);
if (isUpToDate) {
const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`;
log.info(colors.green(`${displayName} for ${language} are up-to-date, skipping push`));
return;
}
log.info(colors.yellow(`Pushing ${name ? 'named' : 'workspace default'} dependencies for ${language}...`));
await wmill.createWorkspaceDependencies({
workspace,
requestBody: {
name,
content: newDependenciesContent,
language,
workspace_id: workspace,
// Description is not supported in cli, it will use old description
description: undefined
}
});
// Update wmill-lock.yaml with new hash after successful push
await updateMetadataGlobalLock(path, contentHash);
const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`;
log.info(colors.green(`Successfully pushed ${displayName} for ${language}`));
} catch (error: any) {
log.error(colors.red(`Failed to push workspace dependencies: ${error.message}`));
throw error;
}
}
export default command;
cli/src/commands/dev/dev.ts
+1 -1
View File
@@ -167,7 +167,7 @@ async function dev(opts: GlobalOptions & SyncOptions) {
ws.on("message", (message: WebSocket.RawData) => {
let data;
try {
data = JSON.parse(message);
data = JSON.parse(message.toString());
} catch (e) {
console.log("Received invalid JSON: " + message + " " + e);
return;
cli/src/commands/flow/flow.ts
+4 -16
View File
@@ -8,11 +8,11 @@ import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace, validatePath } from "../../core/context.ts";
import { resolve, track_job } from "../script/script.ts";
import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts";
import { generateFlowLockInternal } from "../../utils/metadata.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts";
import { Flow } from "../../../gen/types.gen.ts";
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { generateFlowLockInternal } from "./flow_metadata.ts";
export interface FlowFile {
summary: string;
@@ -200,9 +200,6 @@ async function generateLocks(
} & SyncOptions,
folder: string | undefined
) {
const useRawReqs =
opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true";
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
@@ -212,10 +209,7 @@ async function generateLocks(
folder,
false,
workspace,
opts,
undefined,
undefined,
useRawReqs
opts
);
} else {
const ignore = await ignoreF(opts);
@@ -241,10 +235,7 @@ async function generateLocks(
folder,
true,
workspace,
opts,
undefined,
undefined,
useRawReqs
opts
);
if (candidate) {
hasAny = true;
@@ -271,10 +262,7 @@ async function generateLocks(
folder,
false,
workspace,
opts,
undefined,
undefined,
useRawReqs
opts
);
}
}
cli/src/commands/flow/flow_metadata.ts
+238
View File
@@ -0,0 +1,238 @@
import {
SEP,
colors,
log,
path,
yamlParseFile,
yamlStringify,
} from "../../../deps.ts";
import { GlobalOptions } from "../../types.ts";
import {
readLockfile,
checkifMetadataUptodate,
blueColor,
clearGlobalLock,
updateMetadataGlobalLock,
LockfileGenerationError,
getRawWorkspaceDependencies,
} from "../../utils/metadata.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
import { exts, } from "../script/script.ts";
import { FSFSElement } from "../sync/sync.ts";
import { Workspace } from "../workspace/workspace.ts";
import { FlowFile } from "./flow.ts";
import { FlowValue } from "../../../gen/types.gen.ts";
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { workspaceDependenciesLanguages } from "../../utils/script_common.ts";
const TOP_HASH = "__flow_hash";
async function generateFlowHash(
rawWorkspaceDependencies: Record<string, string>,
folder: string,
defaultTs: "bun" | "deno" | undefined
) {
const elems = await FSFSElement(path.join(Deno.cwd(), folder), [], true);
const hashes: Record<string, string> = {};
for await (const f of elems.getChildren()) {
if (exts.some((e) => f.path.endsWith(e))) {
// Embed workspace dependencies into hash
hashes[f.path] = await generateHash(
(await f.getContentText()) + JSON.stringify(rawWorkspaceDependencies)
);
}
}
return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
}
export async function generateFlowLockInternal(
folder: string,
dryRun: boolean,
workspace: Workspace,
opts: GlobalOptions & {
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean
): Promise<string | void> {
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
}
const remote_path = folder
.replaceAll(SEP, "/")
.substring(0, folder.length - ".flow".length);
if (!justUpdateMetadataLock && !noStaleMessage) {
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
}
// Always get out-of-sync workspace dependencies
const rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
const conf = await readLockfile();
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
if (!noStaleMessage) {
log.info(
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
return remote_path;
}
if (Object.keys(rawWorkspaceDependencies).length > 0) {
log.info(
(await blueColor())(
`Found workspace dependencies (${workspaceDependenciesLanguages
.map((l) => l.filename)
.join("/")}) for ${folder}, using them`
)
);
}
const flowValue = (await yamlParseFile(
folder! + SEP + "flow.yaml"
)) as FlowFile;
if (!justUpdateMetadataLock) {
const changedScripts = [];
//find hashes that do not correspond to previous hashes
for (const [path, hash] of Object.entries(hashes)) {
if (path == TOP_HASH) {
continue;
}
if (!(await checkifMetadataUptodate(folder, hash, conf, path))) {
changedScripts.push(path);
}
}
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
await replaceInlineScripts(
flowValue.value.modules,
async (path: string) => await Deno.readTextFile(folder + SEP + path),
log,
folder + SEP!,
SEP,
changedScripts,
// (path: string, newPath: string) => Deno.renameSync(path, newPath),
// (path: string) => Deno.removeSync(path)
);
//removeChangedLocks
flowValue.value = await updateFlow(
workspace,
flowValue.value,
remote_path,
rawWorkspaceDependencies
);
const inlineScripts = extractInlineScriptsForFlows(
flowValue.value.modules,
{},
SEP,
opts.defaultTs
);
inlineScripts.forEach((s) => {
writeIfChanged(Deno.cwd() + SEP + folder + SEP + s.path, s.content);
});
// Overwrite `flow.yaml` with the new lockfile references
writeIfChanged(
Deno.cwd() + SEP + folder + SEP + "flow.yaml",
yamlStringify(flowValue as Record<string, any>)
);
}
hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
await clearGlobalLock(folder);
for (const [path, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(folder, hash, path);
}
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
}
export async function updateFlow(
workspace: Workspace,
flow_value: FlowValue,
remotePath: string,
rawWorkspaceDependencies: Record<string, string>
): Promise<FlowValue | undefined> {
let rawResponse;
if (Object.keys(rawWorkspaceDependencies).length > 0) {
log.info(colors.blue("Using raw workspace dependencies for flow dependencies"));
// generate the script lock running a dependency job in Windmill and update it inplace
const extraHeaders = getHeaders();
rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
...extraHeaders,
},
body: JSON.stringify({
flow_value,
path: remotePath,
use_local_lockfiles: true,
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies
: null,
}),
}
);
} else {
// Standard dependency resolution on the server
const extraHeaders = getHeaders();
rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
...extraHeaders,
},
body: JSON.stringify({
flow_value,
path: remotePath,
}),
}
);
}
let responseText = "reading response failed";
try {
const res = (await rawResponse.json()) as
| { updated_flow_value: any }
| { error: { message: string } }
| undefined;
if (rawResponse.status != 200) {
const msg = (res as any)?.["error"]?.["message"];
if (msg) {
throw new LockfileGenerationError(
`Failed to generate lockfile: ${msg}`
);
}
throw new LockfileGenerationError(
`Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}`
);
}
return (res as any).updated_flow_value;
} catch (e) {
try {
responseText = await rawResponse.text();
} catch {
responseText = "";
}
throw new Error(
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
);
}
}
cli/src/commands/init/init.ts
+17 -33
View File
@@ -1,14 +1,9 @@
import {
colors,
Command,
log,
yamlStringify,
Confirm,
} from "../../../deps.ts";
import { colors, Command, log, yamlStringify, Confirm } from "../../../deps.ts";
import { GlobalOptions } from "../../types.ts";
import { readLockfile } from "../../utils/metadata.ts";
import { SCRIPT_GUIDANCE } from "../../guidance/script_guidance.ts";
import { FLOW_GUIDANCE } from "../../guidance/flow_guidance.ts";
import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts";
export interface InitOptions {
useDefault?: boolean;
@@ -61,12 +56,10 @@ async function initAction(opts: InitOptions) {
// Offer to bind workspace profile to current branch
if (isGitRepository()) {
const { getActiveWorkspace } = await import("../workspace/workspace.ts");
const activeWorkspace = await getActiveWorkspace(
const activeWorkspace = await getActiveWorkspaceOrFallback(
opts as GlobalOptions
);
const currentBranch = getCurrentGitBranch();
if (activeWorkspace && currentBranch) {
// Determine binding behavior based on flags
const shouldBind = opts.bindProfile === true;
@@ -74,10 +67,10 @@ async function initAction(opts: InitOptions) {
opts.bindProfile === undefined &&
Deno.stdin.isTerminal() &&
!opts.useDefault;
const shouldSkip =
opts.bindProfile === false ||
opts.useDefault ||
(!Deno.stdin.isTerminal() && opts.bindProfile === undefined);
opts.bindProfile != true &&
(opts.useDefault || !Deno.stdin.isTerminal());
if (shouldSkip) {
return;
@@ -86,15 +79,11 @@ async function initAction(opts: InitOptions) {
// Show workspace info if we're binding or prompting
if (shouldBind || shouldPrompt) {
log.info(
colors.yellow(
`\nCurrent Git branch: ${colors.bold(currentBranch)}`
)
colors.yellow(`\nCurrent Git branch: ${colors.bold(currentBranch)}`)
);
log.info(
colors.yellow(
`Active workspace profile: ${colors.bold(
activeWorkspace.name
)}`
`Active workspace profile: ${colors.bold(activeWorkspace.name)}`
)
);
log.info(
@@ -123,15 +112,15 @@ async function initAction(opts: InitOptions) {
currentConfig.gitBranches[currentBranch] = { overrides: {} };
}
log.info(
`binding branch ${currentBranch} to workspace ${activeWorkspace.name} on ${activeWorkspace.remote}`
);
currentConfig.gitBranches[currentBranch].baseUrl =
activeWorkspace.remote;
currentConfig.gitBranches[currentBranch].workspaceId =
activeWorkspace.workspaceId;
await Deno.writeTextFile(
"wmill.yaml",
yamlStringify(currentConfig)
);
await Deno.writeTextFile("wmill.yaml", yamlStringify(currentConfig));
log.info(
colors.green(
@@ -149,10 +138,10 @@ async function initAction(opts: InitOptions) {
const { resolveWorkspace } = await import("../../core/context.ts");
// Check if user has workspace configured
const { getActiveWorkspace } = await import("../workspace/workspace.ts");
const activeWorkspace = await getActiveWorkspace(
opts as GlobalOptions
const { getActiveWorkspace } = await import(
"../workspace/workspace.ts"
);
const activeWorkspace = await getActiveWorkspace(opts as GlobalOptions);
if (!activeWorkspace) {
log.info("No workspace configured. Using default settings.");
@@ -233,9 +222,7 @@ async function initAction(opts: InitOptions) {
replace: true, // Auto-replace when using backend settings during init
});
log.info(
colors.green("Git-sync settings applied from backend")
);
log.info(colors.green("Git-sync settings applied from backend"));
}
}
} catch (error) {
@@ -266,10 +253,7 @@ async function initAction(opts: InitOptions) {
}
if (!(await Deno.stat(".cursor/rules/flow.mdc").catch(() => null))) {
await Deno.writeTextFile(
".cursor/rules/flow.mdc",
flowGuidanceContent
);
await Deno.writeTextFile(".cursor/rules/flow.mdc", flowGuidanceContent);
log.info(colors.green("Created .cursor/rules/flow.mdc"));
}
cli/src/commands/script/script.ts
+63 -82
View File
@@ -26,13 +26,14 @@ import {
import { Workspace } from "../workspace/workspace.ts";
import {
generateScriptMetadataInternal,
getRawWorkspaceDependencies,
parseMetadataFile,
} from "../../utils/metadata.ts";
import {
LanguageWithRawReqsSupport,
WorkspaceDependenciesLanguage,
ScriptLanguage,
inferContentTypeFromFilePath,
languagesWithRawReqsSupport,
workspaceDependenciesLanguages,
} from "../../utils/script_common.ts";
import {
elementsToMap,
@@ -88,14 +89,13 @@ async function push(opts: PushOptions, filePath: string) {
await requireLogin(opts);
const codebases = await listSyncCodebases(opts as SyncOptions);
const globalDeps = await findGlobalDeps();
await handleFile(
filePath,
workspace,
[],
undefined,
opts,
globalDeps,
await getRawWorkspaceDependencies(),
codebases
);
log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
@@ -142,7 +142,7 @@ export async function findResourceFile(path: string) {
if (validCandidates.length > 1) {
throw new Error(
"Found two resource files for the same resource" +
validCandidates.join(", ")
validCandidates.join(", ")
);
}
if (validCandidates.length < 1) {
@@ -156,7 +156,7 @@ export async function handleScriptMetadata(
workspace: Workspace,
alreadySynced: string[],
message: string | undefined,
globalDeps: GlobalDeps,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[],
opts: GlobalOptions
): Promise<boolean> {
@@ -172,7 +172,7 @@ export async function handleScriptMetadata(
alreadySynced,
message,
opts,
globalDeps,
rawWorkspaceDependencies,
codebases
);
} else {
@@ -181,11 +181,11 @@ export async function handleScriptMetadata(
}
export interface OutputFile {
path: string
contents: Uint8Array
hash: string
path: string;
contents: Uint8Array;
hash: string;
/** "contents" as text (changes automatically with "contents") */
readonly text: string
readonly text: string;
}
export async function handleFile(
@@ -194,7 +194,7 @@ export async function handleFile(
alreadySynced: string[],
message: string | undefined,
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined,
globalDeps: GlobalDeps,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[]
): Promise<boolean> {
if (
@@ -223,13 +223,12 @@ export async function handleFile(
let outputFiles: OutputFile[] = [];
if (codebase.customBundler) {
log.info(`Using custom bundler ${codebase.customBundler} for ${path}`);
bundleContent = execSync(
codebase.customBundler + " " + path,
{ maxBuffer: 1024 * 1024 * 50 }
).toString();
bundleContent = execSync(codebase.customBundler + " " + path, {
maxBuffer: 1024 * 1024 * 50,
}).toString();
log.info("Custom bundler executed for " + path);
} else {
const esbuild = await import("npm:esbuild");
const esbuild = await import("npm:esbuild@0.24.2");
log.info(`Started bundling ${path} ...`);
const startTime = performance.now();
@@ -243,14 +242,19 @@ export async function handleFile(
inject: codebase.inject,
define: codebase.define,
loader: codebase.loader ?? { ".node": "file" },
outdir: '/',
outdir: "/",
platform: "node",
packages: "bundle",
target: format == "cjs" ? "node20.15.1" : "esnext",
banner: codebase.banner,
// ...(codebase.banner != null && { banner: codebase.banner }),
});
const endTime = performance.now();
bundleContent = out.outputFiles[0].text;
outputFiles = out.outputFiles;
outputFiles = out.outputFiles ?? [];
if (outputFiles.length == 0) {
throw new Error(`No output files found for ${path}`);
}
log.info(
`Finished bundling ${path}: ${(bundleContent.length / 1024).toFixed(
0
@@ -260,17 +264,18 @@ export async function handleFile(
if (outputFiles.length > 1) {
const archiveNpm = await import("npm:@ayonli/jsext/archive");
log.info(
`Found multiple output files for ${path}, creating a tarball... ${outputFiles.map((file) => file.path).join(", ")}`
`Found multiple output files for ${path}, creating a tarball... ${outputFiles
.map((file) => file.path)
.join(", ")}`
);
forceTar = true;
const startTime = performance.now();
const tarball = new archiveNpm.Tarball();
const mainPath = path.split(SEP).pop()?.split(".")[0] + ".js";
const content = outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? '';
const content =
outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? "";
log.info(`Main content: ${content.length}chars`);
tarball.append(
new File([content], "main.js", { type: "text/plain" })
);
tarball.append(new File([content], "main.js", { type: "text/plain" }));
for (const file of outputFiles) {
if (file.path == "/" + mainPath) {
continue;
@@ -318,20 +323,20 @@ export async function handleFile(
let typed = opts?.skipScriptsMetadata
? undefined
: (
await parseMetadataFile(
remotePath,
opts
? {
...opts,
path,
workspaceRemote: workspace,
schemaOnly: codebase ? true : undefined,
globalDeps,
codebases
}
: undefined,
)
)?.payload;
await parseMetadataFile(
remotePath,
opts
? {
...opts,
path,
workspaceRemote: workspace,
schemaOnly: codebase ? true : undefined,
rawWorkspaceDependencies,
codebases,
}
: undefined
)
)?.payload;
const workspaceId = workspace.workspaceId;
@@ -401,6 +406,7 @@ export async function handleFile(
on_behalf_of_email: typed?.on_behalf_of_email,
};
// console.log(requestBodyCommon.codebase);
// log.info(JSON.stringify(requestBodyCommon, null, 2))
// log.info(JSON.stringify(opts, null, 2))
if (remote) {
@@ -418,19 +424,19 @@ export async function handleFile(
deepEqual(typed.schema, remote.schema) &&
typed.tag == remote.tag &&
(typed.ws_error_handler_muted ?? false) ==
remote.ws_error_handler_muted &&
remote.ws_error_handler_muted &&
typed.dedicated_worker == remote.dedicated_worker &&
typed.cache_ttl == remote.cache_ttl &&
typed.concurrency_time_window_s ==
remote.concurrency_time_window_s &&
remote.concurrency_time_window_s &&
typed.concurrent_limit == remote.concurrent_limit &&
Boolean(typed.restart_unless_cancelled) ==
Boolean(remote.restart_unless_cancelled) &&
Boolean(remote.restart_unless_cancelled) &&
Boolean(typed.visible_to_runner_only) ==
Boolean(remote.visible_to_runner_only) &&
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
Boolean(typed.has_preprocessor) ==
Boolean(remote.has_preprocessor) &&
Boolean(remote.has_preprocessor) &&
typed.priority == Boolean(remote.priority) &&
typed.timeout == remote.timeout &&
//@ts-ignore
@@ -523,7 +529,8 @@ async function createScript(
});
} catch (e: any) {
throw Error(
`Script creation for ${body.path} with parent ${body.parent_hash
`Script creation for ${body.path} with parent ${
body.parent_hash
} was not successful: ${e.body ?? e.message} `
);
}
@@ -549,7 +556,8 @@ async function createScript(
});
if (req.status != 201) {
throw Error(
`Script snapshot creation was not successful: ${req.status} - ${req.statusText
`Script snapshot creation was not successful: ${req.status} - ${
req.statusText
} - ${await req.text()} `
);
}
@@ -561,8 +569,8 @@ export async function findContentFile(filePath: string) {
const candidates = filePath.endsWith("script.json")
? exts.map((x) => filePath.replace(".script.json", x))
: filePath.endsWith("script.lock")
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
const validCandidates = (
await Promise.all(
@@ -581,7 +589,7 @@ export async function findContentFile(filePath: string) {
if (validCandidates.length > 1) {
throw new Error(
"No content path given and more than one candidate found: " +
validCandidates.join(", ")
validCandidates.join(", ")
);
}
if (validCandidates.length < 1) {
@@ -940,39 +948,10 @@ async function bootstrap(
}
export type GlobalDeps = Map<
LanguageWithRawReqsSupport,
WorkspaceDependenciesLanguage,
Record<string, string>
>;
export async function findGlobalDeps(): Promise<GlobalDeps> {
var globalDeps: GlobalDeps = new Map();
const els = await FSFSElement(Deno.cwd(), [], false);
for await (const entry of readDirRecursiveWithIgnore((p, isDir) => {
p = SEP + p;
return (
!isDir &&
// Skip if the filename is not one of lockfile names
!languagesWithRawReqsSupport.some((lockfile) =>
p.endsWith(SEP + lockfile.rrFilename)
)
);
}, els)) {
if (entry.isDirectory || entry.ignored) continue;
const content = await entry.getContentText();
// Iterate over available languages to find which lockfile
languagesWithRawReqsSupport.map((lock) => {
if (entry.path.endsWith(lock.rrFilename)) {
const current = globalDeps.get(lock) ?? {};
current[
entry.path.substring(0, entry.path.length - lock.rrFilename.length)
] = content;
globalDeps.set(lock, current);
}
});
}
return globalDeps;
}
async function generateMetadata(
opts: GlobalOptions & {
lockOnly?: boolean;
@@ -996,7 +975,7 @@ async function generateMetadata(
opts = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(opts);
const globalDeps = await findGlobalDeps();
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
if (scriptPath) {
// read script metadata file
await generateScriptMetadataInternal(
@@ -1005,11 +984,12 @@ async function generateMetadata(
opts,
false,
false,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
} else {
// TODO: test this as well.
const ignore = await ignoreF(opts);
const elems = await elementsToMap(
await FSFSElement(Deno.cwd(), codebases, false),
@@ -1033,7 +1013,7 @@ async function generateMetadata(
opts,
true,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
@@ -1060,6 +1040,7 @@ async function generateMetadata(
log.info(colors.green.bold("No metadata to update"));
return;
}
// TODO: test this
for (const e of Object.keys(elems)) {
await generateScriptMetadataInternal(
e,
@@ -1067,7 +1048,7 @@ async function generateMetadata(
opts,
false,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
cli/src/commands/sync/pull.ts
+6 -4
View File
@@ -17,6 +17,7 @@ export async function downloadZip(
includeGroups?: boolean,
includeSettings?: boolean,
includeKey?: boolean,
skipWorkspaceDependencies?: boolean,
defaultTs?: "bun" | "deno"
): Promise<JSZip | undefined> {
const requestHeaders = new Headers();
@@ -30,8 +31,8 @@ export async function downloadZip(
}
}
const zipResponse = await fetch(
workspace.remote +
const includeWorkspaceDependenciesValue = !(skipWorkspaceDependencies ?? false);
const url = workspace.remote +
"api/w/" +
workspace.workspaceId +
`/workspaces/tarball?archive_type=zip&plain_secret=${plainSecrets ?? false
@@ -39,8 +40,9 @@ export async function downloadZip(
}&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false
}&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false
}&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false
}&include_key=${includeKey ?? false}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`,
{
}&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`;
const zipResponse = await fetch(url, {
headers: requestHeaders,
method: "GET",
}
cli/src/commands/sync/sync.ts
+314 -98
View File
@@ -5,13 +5,13 @@ import {
Command,
Confirm,
ensureDir,
minimatch,
JSZip,
path,
log,
yamlStringify,
yamlParseContent,
minimatch,
path,
SEP,
yamlParseContent,
yamlStringify,
} from "../../../deps.ts";
import * as wmill from "../../../gen/services.gen.ts";
@@ -28,37 +28,37 @@ import { downloadZip } from "./pull.ts";
import {
exts,
findContentFile,
findGlobalDeps,
findResourceFile,
handleScriptMetadata,
removeExtensionToPath,
} from "../script/script.ts";
import { handleFile } from "../script/script.ts";
import { deepEqual, isFileResource } from "../../utils/utils.ts";
import { deepEqual, isFileResource, isRawAppFile, isWorkspaceDependencies } from "../../utils/utils.ts";
import {
SyncOptions,
getEffectiveSettings,
validateBranchConfiguration,
mergeConfigWithConfigFile,
SyncOptions,
validateBranchConfiguration,
} from "../../core/conf.ts";
import {
SpecificItemsConfig,
getSpecificItemsForCurrentBranch,
isSpecificItem,
getBranchSpecificPath,
fromBranchSpecificPath,
isCurrentBranchFile,
getBranchSpecificPath,
getSpecificItemsForCurrentBranch,
isBranchSpecificFile,
isCurrentBranchFile,
isSpecificItem,
SpecificItemsConfig,
} from "../../core/specific_items.ts";
import { getCurrentGitBranch } from "../../utils/git.ts";
import { Workspace } from "../workspace/workspace.ts";
import { removePathPrefix } from "../../types.ts";
import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts";
import { listSyncCodebases, SyncCodebase } from "../../utils/codebase.ts";
import {
generateFlowLockInternal,
generateScriptMetadataInternal,
getRawWorkspaceDependencies,
readLockfile,
workspaceDependenciesPathToLanguageAndFilename,
} from "../../utils/metadata.ts";
import { OpenFlow } from "../../../gen/types.gen.ts";
import { pushResource } from "../resource/resource.ts";
@@ -67,7 +67,9 @@ import {
PathAssigner,
} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
import { isExecutionModeAnonymous } from "../app/apps.ts";
import { generateAppLocksInternal } from "../app/app_metadata.ts";
// Merge CLI options with effective settings, preserving CLI flags as overrides
function mergeCliWithEffectiveOptions<
@@ -89,6 +91,7 @@ async function resolveEffectiveSyncOptions(
type DynFSElement = {
isDirectory: boolean;
path: string;
whitelistedExt?: boolean;
// getContentBytes(): Promise<Uint8Array>;
getContentText(): Promise<string>;
getChildren(): AsyncIterable<DynFSElement>;
@@ -261,19 +264,55 @@ export interface InlineScript {
content: string;
}
function extractFields(fields: Record<string, any>) {
Object.entries(fields).forEach(([k, v]) => {
if (typeof v == "object") {
if (v.type == "static") {
fields[k] = { value: v.value }
} else if (v.type == "javascript") {
fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources }
} else if (v.type == "user") {
fields[k] = undefined
}
}
// if (k == 'runType') {
// fields["type"] = undefined
// fields["schema"] = undefined
// }
})
}
export function extractFieldsForRawApps(runnables: Record<string, any>) {
Object.values(runnables).forEach((v) => {
if (typeof v == "object") {
if (v.fields !== undefined) {
extractFields(v.fields)
}
}
})
}
export function extractInlineScriptsForApps(
key: string | undefined,
rec: any,
pathAssigner: PathAssigner
pathAssigner: PathAssigner,
toId: (key: string, val: any) => string
): InlineScript[] {
if (!rec) {
return [];
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
if (k == 'runType') {
rec["type"] = undefined
rec["schema"] = undefined
return []
} else if (k == "inlineScript" && typeof v == "object") {
rec["type"] = undefined
const o: Record<string, any> = v as any;
const name = rec["name"];
const [basePath, ext] = pathAssigner.assignPath(name, o["language"]);
const name = toId(key ?? "", rec);
const [basePathO, ext] = pathAssigner.assignPath(name, o["language"]);
const basePath = basePathO.replaceAll(SEP, "/");
const r = [];
if (o["content"]) {
const content = o["content"];
@@ -291,9 +330,10 @@ export function extractInlineScriptsForApps(
content: lock,
});
}
o.schema = undefined;
return r;
} else {
return extractInlineScriptsForApps(v, pathAssigner);
return extractInlineScriptsForApps(k, v, pathAssigner, toId);
}
});
}
@@ -311,17 +351,20 @@ function ZipFSElement(
p: string,
f: JSZip.JSZipObject
): Promise<DynFSElement[]> {
const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith(
"flow.json"
)
? "flow"
: p.endsWith("app.json")
? "app"
: p.endsWith("script.json")
? "script"
: p.endsWith("resource.json")
? "resource"
: "other";
const kind: "flow" | "app" | "script" | "resource" | "other" | "raw_app" | "dependencies" =
p.endsWith(".flow.json")
? "flow"
: p.endsWith(".app.json")
? "app"
: p.endsWith(".raw_app.json")
? "raw_app"
: p.endsWith(".script.json")
? "script"
: p.endsWith(".resource.json")
? "resource"
: p.startsWith("dependencies/")
? "dependencies"
: "other";
const isJson = p.endsWith(".json");
@@ -330,15 +373,20 @@ function ZipFSElement(
return p.replace("flow.json", "flow");
} else if (kind == "app") {
return p.replace("app.json", "app");
} else if (kind == "raw_app") {
return p.replace("raw_app.json", "raw_app");
} else if (kind == "dependencies") {
return p;
} else {
return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p;
}
}
const finalPath = transformPath();
const r = [
{
isDirectory: kind == "flow" || kind == "app",
isDirectory: kind == "flow" || kind == "app" || kind == "raw_app",
path: finalPath,
async *getChildren(): AsyncIterable<DynFSElement> {
if (kind == "flow") {
@@ -395,8 +443,10 @@ function ZipFSElement(
let inlineScripts;
try {
inlineScripts = extractInlineScriptsForApps(
undefined,
app?.["value"],
newPathAssigner(defaultTs)
newPathAssigner(defaultTs),
(_, val) => val["name"]
);
} catch (error) {
log.error(
@@ -404,6 +454,7 @@ function ZipFSElement(
);
throw error;
}
for (const s of inlineScripts) {
yield {
isDirectory: false,
@@ -429,6 +480,88 @@ function ZipFSElement(
return yamlStringify(app, yamlOptions);
},
};
} else if (kind == "raw_app") {
let rawApp;
try {
rawApp = JSON.parse(await f.async("text"));
} catch (error) {
log.error(`Failed to parse app.yaml at path: ${p}`);
throw error;
}
if (rawApp?.["policy"]?.["execution_mode"] == "anonymous") {
rawApp.public = true;
}
// console.log("rawApp", rawApp);
rawApp.policy = undefined;
let inlineScripts;
const value = rawApp?.["value"];
// console.log("FOOB", value?.["runnables"])
extractFieldsForRawApps(value?.["runnables"]);
try {
inlineScripts = extractInlineScriptsForApps(
undefined,
value,
newPathAssigner(defaultTs),
(key, val_) => key
);
} catch (error) {
log.error(
`Failed to extract inline scripts for raw app at path: ${p}`
);
throw error;
}
try {
for (const [filePath, content] of Object.entries(
value?.["files"] ?? []
)) {
yield {
isDirectory: false,
path: path.join(finalPath, filePath.substring(1)),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
if (typeof content !== "string") {
throw new Error(
`Content of raw app file ${filePath} is not a string`
);
}
return content as string;
},
};
}
} catch (error) {
log.error(`Failed to extract files for raw app at path: ${p}`);
throw error;
}
for (const s of inlineScripts) {
yield {
isDirectory: false,
path: path.join(finalPath, "runnables", s.path),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return s.content;
},
};
}
const runnables = value?.["runnables"];
if (runnables) {
rawApp.runnables = runnables;
delete rawApp?.["value"];
}
yield {
isDirectory: false,
path: path.join(finalPath, "raw_app.yaml"),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(rawApp, yamlOptions);
},
};
}
},
@@ -489,7 +622,7 @@ function ZipFSElement(
: JSON.stringify(parsed, null, 2);
}
return useYaml && isJson
return useYaml && isJson && kind != "dependencies"
? (() => {
try {
return yamlStringify(JSON.parse(content), yamlOptions);
@@ -622,16 +755,14 @@ export async function* readDirRecursiveWithIgnore(
while (stack.length > 0) {
const e = stack.pop()!;
// console.log(e.path);
yield e;
for await (const e2 of e.c()) {
if (e2.isDirectory) {
const dirName = e2.path.split(SEP).pop();
if (dirName == "node_modules" || dirName?.startsWith(".")) {
if (dirName == "node_modules" || dirName == ".claude" || dirName?.startsWith(".")) {
continue;
}
}
// console.log(e2.path);
stack.push({
path: e2.path,
ignored: e.ignored || ignore(e2.path, e2.isDirectory),
@@ -661,16 +792,56 @@ export async function elementsToMap(
ignore: (path: string, isDirectory: boolean) => boolean,
json: boolean,
skips: Skips,
specificItems?: SpecificItemsConfig
specificItems?: SpecificItemsConfig,
): Promise<{ [key: string]: string }> {
const map: { [key: string]: string } = {};
const processedBasePaths = new Set<string>();
for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
if (entry.isDirectory || entry.ignored) continue;
// console.log("FOO", entry.path, entry.ignored, entry.isDirectory)
if (entry.isDirectory || entry.ignored) {
continue;
}
const path = entry.path;
if (json && path.endsWith(".yaml") && !isFileResource(path)) continue;
if (!json && path.endsWith(".json") && !isFileResource(path)) continue;
if (!isFileResource(path) && !isRawAppFile(path) && !isWorkspaceDependencies(path)) {
if (json && path.endsWith(".yaml")) continue;
if (!json && path.endsWith(".json")) continue;
if (
![
"json",
"yaml",
"go",
"sh",
"ts",
"py",
"sql",
"gql",
"ps1",
"php",
"js",
"lock",
"rs",
"cs",
"yml",
"nu",
"java",
"rb",
// for related places search: ADD_NEW_LANG
].includes(path.split(".").pop() ?? "")
) {
continue;
}
}
if (isRawAppFile(path)) {
const suffix = path.split(".raw_app" + SEP).pop();
if (suffix?.startsWith("dist/") || suffix == "wmill.d.ts" || suffix == "package-lock.json") {
continue;
}
}
if (skips.skipResources && isFileResource(path)) continue;
const ext = json ? ".json" : ".yaml";
if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue;
if (
@@ -684,15 +855,18 @@ export async function elementsToMap(
path.endsWith(".sqs_trigger" + ext) ||
path.endsWith(".gcp_trigger" + ext) ||
path.endsWith(".email_trigger" + ext))
)
) {
continue;
}
if (!skips.includeUsers && path.endsWith(".user" + ext)) continue;
if (!skips.includeGroups && path.endsWith(".group" + ext)) continue;
if (!skips.includeSettings && path === "settings" + ext) continue;
if (!skips.includeKey && path === "encryption_key") continue;
if (skips.skipResources && path.endsWith(".resource" + ext)) continue;
if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext))
if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext)) {
continue;
}
// Use getTypeStrFromPath for consistent type detection
try {
@@ -702,37 +876,11 @@ export async function elementsToMap(
if (skips.skipFlows && fileType === "flow") continue;
if (skips.skipApps && fileType === "app") continue;
if (skips.skipFolders && fileType === "folder") continue;
if (skips.skipWorkspaceDependencies && fileType === "workspace_dependencies") continue;
} catch {
// If getTypeStrFromPath can't determine the type, continue processing the file
}
if (skips.skipResources && isFileResource(path)) continue;
if (
![
"json",
"yaml",
"go",
"sh",
"ts",
"py",
"sql",
"gql",
"ps1",
"php",
"js",
"lock",
"rs",
"cs",
"yml",
"nu",
"java",
"rb",
// for related places search: ADD_NEW_LANG
].includes(path.split(".").pop() ?? "") &&
!isFileResource(path)
)
continue;
// Handle branch-specific files - skip files for other branches
if (specificItems && isBranchSpecificFile(path)) {
@@ -797,7 +945,7 @@ export async function elementsToMap(
// No specific items configuration, use regular path
map[entry.path] = content;
}
}
}
return map;
}
@@ -810,6 +958,7 @@ export interface Skips {
skipFlows?: boolean | undefined;
skipApps?: boolean | undefined;
skipFolders?: boolean | undefined;
skipWorkspaceDependencies?: boolean | undefined;
skipScriptsMetadata?: boolean | undefined;
includeSchedules?: boolean | undefined;
includeTriggers?: boolean | undefined;
@@ -878,11 +1027,14 @@ async function compareDynFSElement(
if (skipMetadata) {
continue;
}
if (k.startsWith("dependencies/")) {
log.info(`Adding workspace dependencies file: ${k}`);
}
changes.push({ name: "added", path: k, content: v });
} else {
if (m2[k] == v) {
continue;
} else if (k.endsWith(".json")) {
} else if (k.endsWith(".json") && !isWorkspaceDependencies(k)) {
let parsedV, parsedM2;
try {
parsedV = JSON.parse(v);
@@ -938,6 +1090,7 @@ async function compareDynFSElement(
const remoteCodebase: Record<string, string> = {};
for (const [k] of Object.entries(m2)) {
if (m1[k] === undefined) {
if (
!ignoreMetadataDeletion ||
@@ -945,7 +1098,7 @@ async function compareDynFSElement(
) {
changes.push({ name: "deleted", path: k });
} else if (k?.endsWith(".script.yaml")) {
let o = parseYaml(k, m2[k]);
const o = parseYaml(k, m2[k]);
if (o.codebase != undefined) {
remoteCodebase[k] = o.codebase;
}
@@ -1050,7 +1203,8 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
!p.startsWith("f" + SEP) &&
!p.startsWith("g" + SEP) &&
!p.startsWith("users" + SEP) &&
!p.startsWith("groups" + SEP)
!p.startsWith("groups" + SEP) &&
!p.startsWith("dependencies" + SEP)
);
}
@@ -1068,7 +1222,8 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
!p.startsWith("f" + SEP) &&
!p.startsWith("g" + SEP) &&
!p.startsWith("users" + SEP) &&
!p.startsWith("groups" + SEP)
!p.startsWith("groups" + SEP) &&
!p.startsWith("dependencies" + SEP)
);
}
} catch {
@@ -1084,7 +1239,8 @@ export const isWhitelisted = (p: string) => {
p == "f" ||
p == "g" ||
p == "users" ||
p == "groups"
p == "groups" ||
p == "dependencies"
);
};
@@ -1093,6 +1249,7 @@ export async function ignoreF(wmillconf: {
excludes?: string[];
extraIncludes?: string[];
skipResourceTypes?: boolean;
skipWorkspaceDependencies?: boolean;
json?: boolean;
includeUsers?: boolean;
includeGroups?: boolean;
@@ -1151,6 +1308,9 @@ export async function ignoreF(wmillconf: {
if (wmillconf.includeKey && fileType === "encryption_key") {
return false; // Don't ignore, always include
}
if (!wmillconf.skipWorkspaceDependencies && fileType === "workspace_dependencies") {
return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped)
}
} catch {
// If getTypeStrFromPath can't determine the type, fall through to normal logic
}
@@ -1168,22 +1328,31 @@ interface ChangeTracker {
scripts: string[];
flows: string[];
apps: string[];
rawApps: string[];
}
const FLOW_EXT = ".flow" + SEP;
const APP_EXT = ".app" + SEP;
const RAW_APP_EXT = ".raw_app" + SEP;
// deno-lint-ignore no-inner-declarations
async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
const isScript = exts.some((e) => p.endsWith(e));
if (isScript) {
if (p.includes(".flow" + SEP)) {
const folder = p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP;
if (p.includes(FLOW_EXT)) {
const folder = p.substring(0, p.indexOf(FLOW_EXT)) + FLOW_EXT;
if (!tracker.flows.includes(folder)) {
tracker.flows.push(folder);
}
} else if (p.includes(".app" + SEP)) {
const folder = p.substring(0, p.indexOf(".app" + SEP)) + ".app" + SEP;
} else if (p.includes(APP_EXT)) {
const folder = p.substring(0, p.indexOf(APP_EXT)) + APP_EXT;
if (!tracker.apps.includes(folder)) {
tracker.apps.push(folder);
}
} else if (p.includes(RAW_APP_EXT)) {
const folder = p.substring(0, p.indexOf(RAW_APP_EXT)) + RAW_APP_EXT;
if (!tracker.rawApps.includes(folder)) {
tracker.rawApps.push(folder);
}
} else {
if (!tracker.scripts.includes(p)) {
tracker.scripts.push(p);
@@ -1206,6 +1375,7 @@ async function buildTracker(changes: Change[]) {
scripts: [],
flows: [],
apps: [],
rawApps: [],
};
for (const change of changes) {
if (change.name == "added" || change.name == "edited") {
@@ -1269,8 +1439,7 @@ export async function pull(
} catch {
// ignore
}
const remote = ZipFSElement(
(await downloadZip(
const zipFile = await downloadZip(
workspace,
opts.plainSecrets,
opts.skipVariables,
@@ -1283,8 +1452,12 @@ export async function pull(
opts.includeGroups,
opts.includeSettings,
opts.includeKey,
opts.skipWorkspaceDependencies,
opts.defaultTs
))!,
);
const remote = ZipFSElement(
zipFile!,
!opts.json,
opts.defaultTs ?? "bun",
resourceTypeToFormatExtension,
@@ -1309,6 +1482,7 @@ export async function pull(
`remote (${workspace.name}) -> local: ${changes.length} changes to apply`
);
// Handle JSON output for dry-run
if (opts.dryRun && opts.jsonOutput) {
const result = {
@@ -1495,9 +1669,9 @@ export async function pull(
}
log.info("All local changes pulled, now updating wmill-lock.yaml");
await readLockfile(); // ensure wmill-lock.yaml exists
const globalDeps = await findGlobalDeps();
const tracker: ChangeTracker = await buildTracker(changes);
const rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
for (const change of tracker.scripts) {
await generateScriptMetadataInternal(
@@ -1506,22 +1680,28 @@ export async function pull(
opts,
false,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
true
);
}
for (const change of tracker.flows) {
log.info(`Updating lock for flow ${change}`);
await generateFlowLockInternal(change, false, workspace, opts, true);
await generateFlowLockInternal(change, false, workspace, opts, true, false);
}
if (tracker.apps.length > 0) {
log.info(
`Apps ${tracker.apps.join(
", "
)} scripts were changed but ignoring for now`
colors.gray(
`Apps ${tracker.apps.join(
", "
)} inline scripts were changed but ignoring metadata regeneration for now`
)
);
}
for (const change of tracker.rawApps) {
log.info(`Updating lock metadata for raw app ${change}`);
await generateAppLocksInternal(change, false, workspace, opts, true, true);
}
if (opts.jsonOutput) {
const result = {
success: true,
@@ -1733,6 +1913,7 @@ export async function push(
opts.includeGroups,
opts.includeSettings,
opts.includeKey,
opts.skipWorkspaceDependencies,
opts.defaultTs
))!,
!opts.json,
@@ -1754,7 +1935,8 @@ export async function push(
specificItems
);
const globalDeps = await findGlobalDeps();
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
const tracker: ChangeTracker = await buildTracker(changes);
@@ -1767,7 +1949,7 @@ export async function push(
opts,
true,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
@@ -1910,10 +2092,19 @@ export async function push(
while (queue.length > 0 || pool.size > 0) {
// Fill the pool until we reach parallelizationFactor
while (pool.size < parallelizationFactor && queue.length > 0) {
const [_basePath, changes] = queue.shift()!;
let [_basePath, changes] = queue.shift()!;
const promise = (async () => {
const alreadySynced: string[] = [];
const isRawApp = isRawAppFile(changes[0].path);
if (isRawApp) {
const deleteRawApp = changes.find(change => change.name === "deleted" && change.path.endsWith(".raw_app/raw_app.yaml"))
if (deleteRawApp) {
changes = [deleteRawApp];
} else {
changes.splice(1, changes.length - 1);
}
}
for await (const change of changes) {
let stateTarget = undefined;
if (stateful) {
@@ -1932,7 +2123,7 @@ export async function push(
workspace,
alreadySynced,
opts.message,
globalDeps,
rawWorkspaceDependencies,
codebases,
opts
)
@@ -1948,7 +2139,7 @@ export async function push(
alreadySynced,
opts.message,
opts,
globalDeps,
rawWorkspaceDependencies,
codebases
)
) {
@@ -2040,7 +2231,7 @@ export async function push(
alreadySynced,
opts.message,
opts,
globalDeps,
rawWorkspaceDependencies,
codebases
)
) {
@@ -2132,6 +2323,14 @@ export async function push(
path: removeSuffix(target, ".app/app.json"),
});
break;
case "raw_app":
if (target.endsWith(".raw_app/raw_app.yaml") || target.endsWith(".raw_app/raw_app.json")) {
await wmill.deleteApp({
workspace: workspaceId,
path: removeSuffix(target, ".raw_app/raw_app.json"),
});
}
break;
case "schedule":
await wmill.deleteSchedule({
workspace: workspaceId,
@@ -2225,6 +2424,22 @@ export async function push(
".group.json"
),
});
break;
case "workspace_dependencies":
const relativePath = removePathPrefix(change.path, "dependencies");
const res = workspaceDependenciesPathToLanguageAndFilename(change.path);
if (!res) {
throw new Error(`Unknown workspace dependencies file format: ${change.path}`);
}
const { name, language } = res;
await wmill.deleteWorkspaceDependencies({
workspace: workspaceId,
language,
name
});
break;
default:
break;
@@ -2302,7 +2517,6 @@ const command = new Command()
.description(
"sync local with a remote workspaces or the opposite (push or pull)"
)
.action(() =>
log.info("2 actions available, pull and push. Use -h to display help.")
)
@@ -2323,6 +2537,7 @@ const command = new Command()
.option("--skip-flows", "Skip syncing flows")
.option("--skip-apps", "Skip syncing apps")
.option("--skip-folders", "Skip syncing folders")
.option("--skip-workspace-dependencies", "Skip syncing workspace dependencies")
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
.option("--include-schedules", "Include syncing schedules")
.option("--include-triggers", "Include syncing triggers")
@@ -2371,6 +2586,7 @@ const command = new Command()
.option("--skip-flows", "Skip syncing flows")
.option("--skip-apps", "Skip syncing apps")
.option("--skip-folders", "Skip syncing folders")
.option("--skip-workspace-dependencies", "Skip syncing workspace dependencies")
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
.option("--include-schedules", "Include syncing schedules")
.option("--include-triggers", "Include syncing triggers")
cli/src/commands/workspace/fork.ts
+6 -11
View File
@@ -1,7 +1,7 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions } from "../../types.ts";
import { colors, Input, log, setClient } from "../../../deps.ts";
import { addWorkspace, allWorkspaces, list, removeWorkspace } from "./workspace.ts";
import { allWorkspaces, list, removeWorkspace } from "./workspace.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, isGitRepository } from "../../utils/git.ts";
import { WM_FORK_PREFIX } from "../../main.ts";
@@ -123,19 +123,14 @@ async function createWorkspaceFork(
throw error;
}
await addWorkspace(
{
name: workspaceName,
remote: remote,
workspaceId: trueWorkspaceId,
token: token,
},
opts
);
const newBranchName = `${WM_FORK_PREFIX}/${clonedBranchName}/${workspaceId}`
log.info(`Created forked workspace ${trueWorkspaceId}. To start contributing to your fork, create and push edits to the branch \`${newBranchName}\` by using the command:\n\n\t`+colors.white(`git checkout -b ${newBranchName}`) + `\n\nThe changes will then be reflected in your fork if you've setup the git sync workflows correctly.`);
log.info(`Created forked workspace ${trueWorkspaceId}. To start contributing to your fork, create and push edits to the branch \`${newBranchName}\` by using the command:
\t`+colors.white(`git checkout -b ${newBranchName}`) + `
When doing operations on the forked workspace, it will use the remote setup in gitBranches for the branch it was forked from.`);
}
async function deleteWorkspaceFork(

Some files were not shown because too many files have changed in this diff Show More