starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-24 16:01:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: workspace dependencies (#7124)

Browse Source 
* commit raw requirements

Signed-off-by: pyranota <pyra@duck.com>

* raw requirements

Signed-off-by: pyranota <pyra@duck.com>

* implement `parse_annotation`

Signed-off-by: pyranota <pyra@duck.com>

* more progress on wdeps

Signed-off-by: pyranota <pyra@duck.com>

* more progress

Signed-off-by: pyranota <pyra@duck.com>

* fixes

Signed-off-by: pyranota <pyra@duck.com>

* more progress

Signed-off-by: pyranota <pyra@duck.com>

* fixes

Signed-off-by: pyranota <pyra@duck.com>

* cli improvements + raw deps

Signed-off-by: pyranota <pyra@duck.com>

* cleanup

Signed-off-by: pyranota <pyra@duck.com>

* fix python versions

Signed-off-by: pyranota <pyra@duck.com>

* progress

Signed-off-by: pyranota <pyra@duck.com>

* update :)

Signed-off-by: pyranota <pyra@duck.com>

* add MaybeLock

Signed-off-by: pyranota <pyra@duck.com>

* go WIP

Signed-off-by: pyranota <pyra@duck.com>

* fix python ignoring py version from requirements

Signed-off-by: pyranota <pyra@duck.com>

* optimize php

Signed-off-by: pyranota <pyra@duck.com>

* require admin to alter

Signed-off-by: pyranota <pyra@duck.com>

* fix(cli): flow generateLocks raw deps

Signed-off-by: pyranota <pyra@duck.com>

* progress in checklist

Signed-off-by: pyranota <pyra@duck.com>

* fix agent workers

Signed-off-by: pyranota <pyra@duck.com>

* nits

Signed-off-by: pyranota <pyra@duck.com>

* nits

Signed-off-by: pyranota <pyra@duck.com>

* nit: remove default features

Signed-off-by: pyranota <pyra@duck.com>

* oh-wow

Signed-off-by: pyranota <pyra@duck.com>

* remove dbg!

Signed-off-by: pyranota <pyra@duck.com>

* nits

Signed-off-by: pyranota <pyra@duck.com>

* add indexes

Signed-off-by: pyranota <pyra@duck.com>

* cleanup

Signed-off-by: pyranota <pyra@duck.com>

* nits

Signed-off-by: pyranota <pyra@duck.com>

* remove todos

Signed-off-by: pyranota <pyra@duck.com>

* fix cli

Signed-off-by: pyranota <pyra@duck.com>

* add debug flag

Signed-off-by: pyranota <pyra@duck.com>

* cli: remove noise

Signed-off-by: pyranota <pyra@duck.com>

* fix cli

Signed-off-by: pyranota <pyra@duck.com>

* remove todos

Signed-off-by: pyranota <pyra@duck.com>

* trigger deps correctly

Signed-off-by: pyranota <pyra@duck.com>

* fix frontend

Signed-off-by: pyranota <pyra@duck.com>

* fix frontend again

Signed-off-by: pyranota <pyra@duck.com>

* finally fix frontend

Signed-off-by: pyranota <pyra@duck.com>

* ee repo ref

Signed-off-by: pyranota <pyra@duck.com>

* fix all

Signed-off-by: pyranota <pyra@duck.com>

* more fixes...

Signed-off-by: pyranota <pyra@duck.com>

* remove test

Signed-off-by: pyranota <pyra@duck.com>

* Update backend-test.yml

* comment out legacy test

Signed-off-by: pyranota <pyra@duck.com>

* fix ci

Signed-off-by: pyranota <pyra@duck.com>

* fix ci?

Signed-off-by: pyranota <pyra@duck.com>

* comment out thing

Signed-off-by: pyranota <pyra@duck.com>

* ignore test

Signed-off-by: pyranota <pyra@duck.com>

* ci

Signed-off-by: pyranota <pyra@duck.com>

* base fixture

Signed-off-by: pyranota <pyra@duck.com>

* fix regression

Signed-off-by: pyranota <pyra@duck.com>

* fix docs links

Signed-off-by: pyranota <pyra@duck.com>

* update min version

Signed-off-by: pyranota <pyra@duck.com>

* simplify

* implement cache for get_latest

Signed-off-by: pyranota <pyra@duck.com>

* move to workspace settings

Signed-off-by: pyranota <pyra@duck.com>

* sqlx + migration

Signed-off-by: pyranota <pyra@duck.com>

* more migrations

Signed-off-by: pyranota <pyra@duck.com>

* use box pin

Signed-off-by: pyranota <pyra@duck.com>

* nit

Signed-off-by: pyranota <pyra@duck.com>

---------

Signed-off-by: pyranota <pyra@duck.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
Pyra
2025-11-28 17:00:11 +01:00
committed by GitHub
parent e26b5c94a3
commit d38c96db36
94 changed files with 6919 additions and 1944 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-test.yml
+1
View File
@@ -83,6 +83,7 @@ jobs:
RUST_LOG: info
RUST_LOG_STYLE: never
CARGO_NET_GIT_FETCH_WITH_CLI: true
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
run: |
deno --version && bun -v && go version && python3 --version
cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd ..
Dockerfile
+1
View File
@@ -100,6 +100,7 @@ ARG POWERSHELL_VERSION=7.5.0
ARG POWERSHELL_DEB_VERSION=7.5.0-1
ARG KUBECTL_VERSION=1.28.7
ARG HELM_VERSION=3.14.3
# NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte
ARG GO_VERSION=1.25.0
ARG APP=/usr/src/app
ARG WITH_POWERSHELL=true
backend/.sqlx/query-04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM v2_job",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493"
}
backend/.sqlx/query-05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at)\n SELECT $1, language, name, description, content, archived, created_at\n FROM workspace_dependencies \n WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c"
}
backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "hash",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd"
}
backend/.sqlx/query-0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d.json
Generated
-24
View File
@@ -1,24 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false
]
},
"hash": "0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d"
}
backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json
Generated
+47
View File
@@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_dependencies\n SET archived = true\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": []
},
"hash": "0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0"
}
backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n importer_path,\n importer_kind::text as \"importer_kind!\", -- sqlx thinks this is nullable somehow, so enfore with !\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "importer_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "importer_kind!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "importer_node_ids",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null,
null
]
},
"hash": "1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708"
}
backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json
Generated
+96
View File
@@ -0,0 +1,96 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, archived, description, workspace_id, created_at\n FROM workspace_dependencies\n WHERE id = $1 AND workspace_id = $2\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041"
}
backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_dependencies\n SET archived = true \n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "description",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false
]
},
"hash": "37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b"
}
backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json
Generated
+47
View File
@@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": []
},
"hash": "5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab"
}
backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json
Generated
+1 -1
View File
@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json
Generated
+57
View File
@@ -0,0 +1,57 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5) \n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
},
"Text"
]
},
"nullable": [
false
]
},
"hash": "5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0"
}
backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json
Generated
+65
View File
@@ -0,0 +1,65 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, hash, language AS \"language: ScriptLang\" FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "hash",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false
]
},
"hash": "5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea"
}
backend/.sqlx/query-9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0.json
Generated
-66
View File
@@ -1,66 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "display_name",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owners",
"type_info": "VarcharArray"
},
{
"ordinal": 4,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "summary",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "edited_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
true,
true
]
},
"hash": "9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0"
}
backend/.sqlx/query-a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6"
}
backend/.sqlx/query-abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d.json
Generated
-42
View File
@@ -1,42 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "extra_perms",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
false,
false,
true,
false
]
},
"hash": "abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d"
}
backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false
]
},
"hash": "bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d"
}
backend/.sqlx/query-c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n imported_path,\n COUNT(DISTINCT importer_path) as \"count!\"\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = ANY($2)\n GROUP BY imported_path\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "imported_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"TextArray"
]
},
"nullable": [
false,
null
]
},
"hash": "c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c"
}
backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json
Generated
+95
View File
@@ -0,0 +1,95 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, created_at, archived, name, description, workspace_id, content, language AS \"language: ScriptLang\"\n FROM workspace_dependencies\n WHERE archived = false AND workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 2,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d"
}
backend/.sqlx/query-d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name desc",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce"
}
backend/.sqlx/query-e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM v2_job_queue",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39"
}
backend/.sqlx/query-ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4.json
Generated
-35
View File
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT importer_path, importer_kind::text, array_agg(importer_node_id) as importer_node_ids FROM dependency_map\n WHERE imported_path = $1\n AND workspace_id = $2\n GROUP BY importer_path, importer_kind",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "importer_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "importer_kind",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "importer_node_ids",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null,
null
]
},
"hash": "ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4"
}
backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json
Generated
+128
View File
@@ -0,0 +1,128 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, description, archived, workspace_id, created_at\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc"
}
backend/.sqlx/query-f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f.json
Generated
+51
View File
@@ -0,0 +1,51 @@
{
"db_name": "PostgreSQL",
"query": "SELECT language AS \"language: ScriptLang\" FROM script WHERE path = 'u/admin/hub_sync'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f"
}
backend/Cargo.lock
Generated
+9
View File
@@ -15193,6 +15193,7 @@ dependencies = [
"reqwest 0.12.24",
"rustls 0.23.35",
"serde",
"serde_derive",
"serde_json",
"serde_yml",
"sha1",
@@ -15438,6 +15439,8 @@ dependencies = [
"opentelemetry-otlp",
"opentelemetry-semantic-conventions",
"opentelemetry_sdk",
"pep440_rs",
"phf 0.11.3",
"pin-project-lite",
"prometheus",
"quick_cache",
@@ -15450,6 +15453,7 @@ dependencies = [
"semver 1.0.27",
"serde",
"serde_json",
"serde_yml",
"sha2 0.10.9",
"size",
"sqlx",
@@ -15525,9 +15529,13 @@ version = "1.586.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
"pep440_rs",
"proc-macro2",
"quote",
"regex",
"serde",
"serde_derive",
"serde_yml",
"syn 2.0.111",
]
@@ -15653,6 +15661,7 @@ dependencies = [
"serde_json",
"sqlx",
"toml",
"tracing",
"windmill-common",
"windmill-parser",
]
backend/Cargo.toml
+1
View File
@@ -137,6 +137,7 @@ prometheus = { workspace = true, optional = true }
uuid.workspace = true
gethostname.workspace = true
serde_json.workspace = true
serde_derive.workspace = true
serde_yml.workspace = true
serde.workspace = true
deno_core = { workspace = true, optional = true }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
99828e4a3a27af428a45d1e6e65092ca15082b1e
88d5023df7f41bb86e948b35889f962b741c860e
backend/migrations/20251106152104_raw_requirements.down.sql
+10
View File
@@ -0,0 +1,10 @@
-- Drop indexes first (though dropping table will cascade)
DROP INDEX IF EXISTS workspace_dependencies_id_workspace ;
DROP INDEX IF EXISTS workspace_dependencies_workspace_lang_name_archived_idx ;
DROP INDEX IF EXISTS workspace_dependencies_workspace_archived_idx;
DROP INDEX IF EXISTS one_non_archived_per_null_name_language_constraint;
DROP INDEX IF EXISTS one_non_archived_per_name_language_constraint;
-- Drop table and sequence
DROP TABLE IF EXISTS workspace_dependencies;
DROP SEQUENCE IF EXISTS workspace_dependencies_id_seq CASCADE;
backend/migrations/20251106152104_raw_requirements.up.sql
+23
View File
@@ -0,0 +1,23 @@
CREATE SEQUENCE IF NOT EXISTS workspace_dependencies_id_seq;
CREATE TABLE IF NOT EXISTS workspace_dependencies(
id BIGINT DEFAULT nextval('workspace_dependencies_id_seq') PRIMARY KEY,
name VARCHAR(255), -- If NULL - it's global
content TEXT NOT NULL,
language SCRIPT_LANG NOT NULL,
description text NOT NULL DEFAULT '',
archived BOOLEAN NOT NULL DEFAULT false,
workspace_id character varying(50) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
);
-- Make any query that tries to create non-linear history fail
CREATE UNIQUE INDEX IF NOT EXISTS one_non_archived_per_name_language_constraint ON workspace_dependencies(name, language, workspace_id) WHERE archived = false AND name IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS one_non_archived_per_null_name_language_constraint ON workspace_dependencies(language, workspace_id) WHERE archived = false AND name IS NULL;
-- Performance indexes for common query patterns
-- For the list query (filtering by workspace_id and archived)
CREATE INDEX IF NOT EXISTS workspace_dependencies_workspace_archived_idx ON workspace_dependencies(workspace_id, archived) WHERE archived = false;
CREATE INDEX IF NOT EXISTS workspace_dependencies_workspace_lang_name_archived_idx ON workspace_dependencies(workspace_id, language, name, archived);
CREATE INDEX IF NOT EXISTS workspace_dependencies_id_workspace ON workspace_dependencies(id, workspace_id);
backend/parsers/windmill-parser-py-imports/Cargo.toml
+1
View File
@@ -30,3 +30,4 @@ async-recursion.workspace = true
toml.workspace = true
serde.workspace = true
pep440_rs.workspace = true
tracing.workspace = true
backend/parsers/windmill-parser-py-imports/src/lib.rs
+240 -206
View File
@@ -27,7 +27,11 @@ use rustpython_parser::{
use sqlx::{Pool, Postgres};
use windmill_common::{
error::{self, to_anyhow},
worker::PythonAnnotations,
worker::{
split_python_requirements, try_parse_locked_python_version_from_requirements,
PythonAnnotations,
},
workspace_dependencies::{RawWorkspaceDependencies, WorkspaceDependenciesPrefetched},
};
fn replace_import(x: String) -> String {
@@ -170,7 +174,6 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<NImport>>
// This is needed because we've split off the real main function above
let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string()))
})?;
@@ -259,6 +262,8 @@ pub async fn parse_python_imports(
path: &str,
db: &Pool<Postgres>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
locked_v: &mut Option<pep440_rs::Version>,
raw_workspace_dependencies_o: &Option<RawWorkspaceDependencies>,
) -> error::Result<(Vec<String>, Option<String>)> {
let mut compile_error_hint: Option<String> = None;
let mut imports = parse_python_imports_inner(
@@ -268,8 +273,9 @@ pub async fn parse_python_imports(
db,
&mut vec![],
version_specifiers,
// &mut version_specifier.and_then(|_| Some(path.to_owned())),
&mut None
&mut None,
locked_v,
raw_workspace_dependencies_o,
)
.await?
.into_values()
@@ -323,11 +329,22 @@ async fn parse_python_imports_inner(
already_visited: &mut Vec<String>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
path_where_annotated_pyv: &mut Option<String>,
locked_v: &mut Option<pep440_rs::Version>,
raw_workspace_dependencies_o: &Option<RawWorkspaceDependencies>,
) -> error::Result<HashMap<String, NImportResolved>> {
tracing::debug!("Parsing python imports for path: {}", path);
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
tracing::debug!(
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}",
py310,
py311,
py312,
py313
);
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
if perform {
tracing::debug!("Adding version specifier: {}", unparsed);
pep440_rs::VersionSpecifiers::from_str(unparsed.as_str())
.ok()
.map(|vs| version_specifiers.extend(vs.to_vec()));
@@ -341,10 +358,9 @@ async fn parse_python_imports_inner(
for x in code.lines() {
if x.starts_with("# py:") || x.starts_with("#py:") {
push_version_specifiers(
true,
x.replace('#', "").replace("py:", "").trim().to_owned(),
)?;
let version_spec = x.replace('#', "").replace("py:", "").trim().to_owned();
tracing::debug!("Found inline python version specifier: {}", version_spec);
push_version_specifiers(true, version_spec)?;
} else if !x.starts_with('#') {
break;
}
@@ -369,12 +385,40 @@ async fn parse_python_imports_inner(
// dependencies: Vec<String>,
// }
let find_requirements = code.lines().find_position(|x| {
x.starts_with("#requirements:")
|| x.starts_with("# requirements:")
|| x.starts_with("# /// script")
});
let mut final_imports = HashMap::new();
tracing::debug!("Extracting workspace dependencies for workspace: {}", w_id);
let wdp = WorkspaceDependenciesPrefetched::extract(
code,
windmill_common::scripts::ScriptLang::Python3,
w_id,
raw_workspace_dependencies_o,
path,
db.into(),
)
.await?;
if let Some(c) = wdp.get_python()? {
*locked_v = extract_nimports_from_content(&c, &mut final_imports);
}
if wdp.is_manual() {
tracing::debug!(
"Workspace dependencies mode is Manual, returning {} imports",
final_imports.len()
);
return Ok(final_imports);
}
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("# /// script"));
tracing::debug!(
"Looking for script metadata block, found: {}",
find_requirements.is_some()
);
if let Some((pos, item)) = find_requirements {
tracing::debug!("Found script metadata block at position {}: {}", pos, item);
let mut requirements = HashMap::new();
if item.starts_with("# /// script") {
let mut incorrect = false;
@@ -384,7 +428,7 @@ async fn parse_python_imports_inner(
.map_while(|x| {
incorrect = !x.starts_with('#');
if incorrect || x.starts_with("# ///") {
None
Option::None
} else {
x.get(1..)
}
@@ -392,9 +436,11 @@ async fn parse_python_imports_inner(
.join("\n")
.parse::<toml::Table>()
.map_err(to_anyhow)?;
tracing::debug!("Parsed script metadata: {:?}", metadata);
{
if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) {
tracing::debug!("Found requires-python in metadata: {}", v);
push_version_specifiers(true, v.to_owned())?;
}
};
@@ -403,9 +449,15 @@ async fn parse_python_imports_inner(
.get("dependencies")
.and_then(|dependencies| dependencies.as_array())
.inspect(|list| {
tracing::debug!("Found {} dependencies in script metadata", list.len());
for dependency_v in list.into_iter() {
let requirement = dependency_v.as_str().unwrap_or("ERROR").to_owned();
let key = extract_pkg_name(&requirement);
tracing::debug!(
"Adding dependency from metadata: {} (key: {})",
requirement,
key
);
requirements.insert(
key.clone(),
NImportResolved::Pin {
@@ -418,217 +470,199 @@ async fn parse_python_imports_inner(
);
}
});
} else {
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
let key = extract_pkg_name(&requirement);
requirements.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin {
pkg: requirement.clone(),
path: Default::default(),
}],
key,
},
);
})
})
})
.collect_vec();
}
Ok(requirements)
} else {
let find_extra_requirements = code.lines().find_position(|x| {
x.starts_with("#extra_requirements:") || x.starts_with("# extra_requirements:")
});
let mut imports: HashMap<String, NImportResolved> = HashMap::new();
if let Some((pos, _)) = find_extra_requirements {
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
let key = extract_pkg_name(&requirement);
imports.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin {
pkg: requirement,
path: Default::default(),
}],
key,
},
);
})
})
})
.collect_vec();
}
tracing::debug!(
"Returning {} requirements from script metadata",
requirements.len()
);
return Ok(requirements);
}
// Will get unsorted vector of imports found in current script
let mut nimports = parse_code_for_imports(code, path)?;
tracing::debug!("Found {} imports in code", nimports.len());
// Will get unsorted vector of imports found in current script
let mut nimports = parse_code_for_imports(code, path)?;
// It is important to note, that sorting is important and will always result in this pattern:
// 1. All Repins go first
// 2. All Pins go second
// 3. All Auto go third
// 4. All relative imports go the last
//
// This way we make sure all repins are resolved before (re)pins inside imported relative scripts.
nimports.sort();
tracing::debug!("Processing imports in sorted order");
// It is important to note, that sorting is important and will always result in this pattern:
// 1. All Repins go first
// 2. All Pins go second
// 3. All Auto go third
// 4. All relative imports go the last
//
// This way we make sure all repins are resolved before (re)pins inside imported relative scripts.
nimports.sort();
for n in nimports.into_iter() {
let mut nested = match n {
NImport::Relative(rpath) => {
let code = sqlx::query_scalar!(
r#"
for n in nimports.into_iter() {
let mut nested = match n {
NImport::Relative(rpath) => {
let code = sqlx::query_scalar!(
r#"
SELECT content FROM script WHERE path = $1 AND workspace_id = $2
AND archived = false ORDER BY created_at DESC LIMIT 1
"#,
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string());
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string());
if already_visited.contains(&rpath) {
vec![]
if already_visited.contains(&rpath) {
vec![]
} else {
already_visited.push(rpath.clone());
// Because the algo goes depth first, this function will never return relative import
// This why we can safely assume later, that there is no relative imports
parse_python_imports_inner(
&code,
w_id,
&rpath,
db,
already_visited,
version_specifiers,
path_where_annotated_pyv,
locked_v,
raw_workspace_dependencies_o,
)
.await?
.into_values()
.collect_vec()
}
}
NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }],
NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }],
NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }],
};
// Nested should also be sorted for the same reason
nested.sort();
tracing::debug!("Processing {} nested imports", nested.len());
// At this point there should be no NImport::Relative in `nested`
for imp in nested {
let key = match imp.clone() {
NImportResolved::Pin { key, .. } => key,
NImportResolved::Repin { key, .. } => key,
NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg),
};
tracing::debug!("Resolving import with key: {}", key);
// Handled cases:
//
// 1.
// Error: Imported windmill scripts have different pins
//
// auto
// ├── pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// ├── pin:1
// └── pin:1
//
// Fix 2:
//
// repin:1
// ├── pin:2
// └── pin:1
//
// 2.
// Error: Imported windmill scripts have different pins
//
// pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// └── pin:1
//
// Fix 2:
//
// repin:2
// └── pin:1
//
// 3. repins allowed to be repinned again
//
// repin:2
// └── repin:1
//
match imp.clone() {
NImportResolved::Repin { .. } => {
if let Some(existing_import) = final_imports.get(&key) {
match existing_import {
// replace
p if matches!(
p,
NImportResolved::Pin { .. } | NImportResolved::Auto { .. }
) =>
{
final_imports.insert(key, imp);
}
// do nothing (older repins have greater precedence)
NImportResolved::Repin { .. } => {}
// Should not be possible
_ => {
return Err(anyhow::anyhow!(
"Internal error: cannot resolve requirement pins",
)
.into());
}
}
} else {
already_visited.push(rpath.clone());
// Because the algo goes depth first, this function will never return relative import
// This why we can safely assume later, that there is no relative imports
parse_python_imports_inner(
&code,
w_id,
&rpath,
db,
already_visited,
version_specifiers,
path_where_annotated_pyv,
)
.await?
.into_values()
.collect_vec()
final_imports.insert(key, imp.clone());
}
}
NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }],
NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }],
NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }],
};
// Nested should also be sorted for the same reason
nested.sort();
// At this point there should be no NImport::Relative in `nested`
for imp in nested {
let key = match imp.clone() {
NImportResolved::Pin { key, .. } => key,
NImportResolved::Repin { key, .. } => key,
NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg),
};
// Handled cases:
//
// 1.
// Error: Imported windmill scripts have different pins
//
// auto
// ├── pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// ├── pin:1
// └── pin:1
//
// Fix 2:
//
// repin:1
// ├── pin:2
// └── pin:1
//
// 2.
// Error: Imported windmill scripts have different pins
//
// pin:2
// └── pin:1
//
// Fix 1:
//
// auto
// └── pin:1
//
// Fix 2:
//
// repin:2
// └── pin:1
//
// 3. repins allowed to be repinned again
//
// repin:2
// └── repin:1
//
match imp.clone() {
NImportResolved::Repin { .. } => {
if let Some(existing_import) = imports.get(&key) {
match existing_import {
// replace
p if matches!(
p,
NImportResolved::Pin { .. } | NImportResolved::Auto { .. }
) =>
{
imports.insert(key, imp);
}
// do nothing (older repins have greater precedence)
NImportResolved::Repin { .. } => {}
// Should not be possible
_ => {
return Err(anyhow::anyhow!(
"Internal error: cannot resolve requirement pins",
)
.into());
}
NImportResolved::Pin { pins: new_pins, .. } => {
if let Some(existing_import) = final_imports.get_mut(&key) {
match existing_import {
// Check if pin is the same version, if same, do nothing, if not error
NImportResolved::Pin { pins: existing_pins, .. } => {
existing_pins.extend(new_pins)
}
} else {
imports.insert(key, imp.clone());
}
}
NImportResolved::Pin { pins: new_pins, .. } => {
if let Some(existing_import) = imports.get_mut(&key) {
match existing_import {
// Check if pin is the same version, if same, do nothing, if not error
NImportResolved::Pin { pins: existing_pins, .. } => {
existing_pins.extend(new_pins)
}
// do nothing
NImportResolved::Repin { .. } => {}
// Replace with new pin
NImportResolved::Auto { .. } => {
imports.insert(key, imp);
}
// do nothing
NImportResolved::Repin { .. } => {}
// Replace with new pin
NImportResolved::Auto { .. } => {
final_imports.insert(key, imp);
}
} else {
imports.insert(key, imp.clone());
}
} else {
final_imports.insert(key, imp.clone());
}
NImportResolved::Auto { .. } => {
if !imports.contains_key(&key) {
imports.insert(key, imp);
}
}
NImportResolved::Auto { .. } => {
if !final_imports.contains_key(&key) {
final_imports.insert(key, imp);
}
}
}
}
Ok(imports)
}
tracing::debug!(
"Finished processing imports, returning {} final imports",
final_imports.len()
);
Ok(final_imports)
}
fn extract_nimports_from_content(
content: &str,
hm: &mut HashMap<String, NImportResolved>,
) -> Option<pep440_rs::Version> {
let lines = split_python_requirements(content);
let locked_version = try_parse_locked_python_version_from_requirements(&lines);
for requirement in lines {
let key = extract_pkg_name(&requirement);
hm.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin { pkg: requirement, path: Default::default() }],
key,
},
);
}
locked_version
}
const STDIMPORTS: [&str; 303] = [
backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql
Vendored
+99
View File
@@ -2581,3 +2581,102 @@ import innerdifffolder
'',
'',
'f/foobar/bar', -28028598712388159, 'python3', '');
--
-- Name: workspace_dependencies_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres
--
CREATE SEQUENCE public.workspace_dependencies_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.workspace_dependencies_id_seq OWNER TO postgres;
--
-- Name: workspace_dependencies_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres
--
SELECT pg_catalog.setval('public.workspace_dependencies_id_seq', 7, true);
--
-- Name: SEQUENCE workspace_dependencies_id_seq; Type: ACL; Schema: public; Owner: postgres
--
GRANT ALL ON SEQUENCE public.workspace_dependencies_id_seq TO windmill_user;
GRANT ALL ON SEQUENCE public.workspace_dependencies_id_seq TO windmill_admin;
--
-- Name: workspace_dependencies; Type: TABLE; Schema: public; Owner: postgres
--
CREATE TABLE public.workspace_dependencies (
id bigint DEFAULT nextval('public.workspace_dependencies_id_seq'::regclass) NOT NULL,
name character varying(255),
content text NOT NULL,
language public.script_lang NOT NULL,
description text DEFAULT ''::text NOT NULL,
archived boolean DEFAULT false NOT NULL,
workspace_id character varying(50) NOT NULL,
created_at timestamp with time zone DEFAULT now() NOT NULL
);
ALTER TABLE public.workspace_dependencies OWNER TO postgres;
--
-- Name: workspace_dependencies workspace_dependencies_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
--
ALTER TABLE ONLY public.workspace_dependencies
ADD CONSTRAINT workspace_dependencies_pkey PRIMARY KEY (id);
--
-- Name: one_non_archived_per_name_language_constraint; Type: INDEX; Schema: public; Owner: postgres
--
CREATE UNIQUE INDEX one_non_archived_per_name_language_constraint ON public.workspace_dependencies USING btree (name, language, workspace_id) WHERE ((archived = false) AND (name IS NOT NULL));
--
-- Name: one_non_archived_per_null_name_language_constraint; Type: INDEX; Schema: public; Owner: postgres
--
CREATE UNIQUE INDEX one_non_archived_per_null_name_language_constraint ON public.workspace_dependencies USING btree (language, workspace_id) WHERE ((archived = false) AND (name IS NULL));
--
-- Name: workspace_dependencies_workspace_archived_idx; Type: INDEX; Schema: public; Owner: postgres
--
CREATE INDEX workspace_dependencies_workspace_archived_idx ON public.workspace_dependencies USING btree (workspace_id, archived) WHERE (archived = false);
--
-- Name: workspace_dependencies_workspace_lang_name_idx; Type: INDEX; Schema: public; Owner: postgres
--
CREATE INDEX workspace_dependencies_workspace_lang_name_idx ON public.workspace_dependencies USING btree (workspace_id, language, name);
--
-- Name: TABLE workspace_dependencies; Type: ACL; Schema: public; Owner: postgres
--
GRANT SELECT,INSERT,REFERENCES,DELETE,TRIGGER,TRUNCATE,UPDATE ON TABLE public.workspace_dependencies TO windmill_user;
GRANT SELECT,INSERT,REFERENCES,DELETE,TRIGGER,TRUNCATE,UPDATE ON TABLE public.workspace_dependencies TO windmill_admin;
--
-- PostgreSQL database dump complete
--
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+30 -6
View File
@@ -18,8 +18,16 @@ def main():
pass
";
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut vec![],
&mut None,
&None,
)
.await?;
// println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
@@ -51,8 +59,16 @@ def main():
pass
";
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut vec![],
&mut None,
&None,
)
.await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
@@ -74,8 +90,16 @@ def main():
";
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut vec![],
&mut None,
&None,
)
.await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
backend/src/main.rs
+3 -3
View File
@@ -271,8 +271,8 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
let job_id = Uuid::new_v4();
let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id);
create_dir_all(&job_dir)?;
if let Some(lockfile) = res.lockfile {
let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?;
if let Some(lock) = res.lockfile {
let _ = windmill_worker::prepare_job_dir(&lock, &job_dir).await?;
let envs = windmill_worker::get_common_bun_proc_envs(None).await;
let _ = windmill_worker::install_bun_lockfile(
&mut 0,
@@ -292,7 +292,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
if let Err(e) = windmill_worker::prebundle_bun_script(
&res.content,
Some(&lockfile),
&lock,
&path,
&job_id,
"admins",
backend/tests/relative_imports.rs → backend/tests/debouncing.rs
+3 -485
View File
@@ -1,7 +1,9 @@
// TODO: move all related logic here (if anything left anywhere in codebase)
mod common;
#[cfg(feature = "test_job_debouncing")]
use windmill_api_client::types::NewScript;
#[cfg(feature = "test_job_debouncing")]
fn quick_ns(
content: &str,
language: windmill_api_client::types::ScriptLang,
@@ -43,490 +45,6 @@ fn quick_ns(
}
}
mod dependency_map {
use super::quick_ns;
use sqlx::{Pool, Postgres};
use tokio_stream::StreamExt;
use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer};
async fn init(db: Pool<Postgres>) -> (windmill_api_client::Client, u16, ApiServer) {
init_client(db).await
}
async fn _clear_dmap(db: &Pool<Postgres>) {
sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'")
.execute(db)
.await
.unwrap();
}
/// Corrects map according to provided replacements.
/// Only changes importer_path and/or id
/// Does not affect imported_path nor kind!
fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> {
CORRECT_DMAP
.clone()
.into_iter()
.map(|e| {
let mut r = (
e.0.to_owned(),
e.1.to_owned(),
e.2.to_owned(),
e.3.to_owned(),
);
for (from, to) in &replacements {
r = (
r.0.replace(from, to),
r.1, // Kind should be immutable
r.2, // Imported path should be immutable
// We do not modify script contents in test, so we can assume scripts always import the same path
// Modification of kind or imported path considered to be incorrect.
r.3.replace(from, to),
);
}
r
})
.collect()
}
async fn assert_dmap(
db: &Pool<Postgres>,
importer: Option<String>,
expected: Vec<(
impl Into<String>,
impl Into<String>,
impl Into<String>,
impl Into<String>,
)>,
) {
let dmap = sqlx::query_as::<_, (String, String, String, String)>(
"SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)",
)
.bind(importer)
.fetch_all(db)
.await
.unwrap();
assert_eq!(
dmap,
expected
.into_iter()
.map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into()))
.collect::<Vec<(String, String, String, String)>>()
);
}
lazy_static::lazy_static! {
pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![
("f/rel/branch", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/branch", ""),
("f/rel/root_script", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/leaf_2", ""),
("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"),
("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"),
("f/rel/root_app", "app", "f/rel/branch", "youcanpressme")];
}
// TODO:
// Test that checks that we can run rebuild_dmap multiple times in tests.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_correctness(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
// rebuild map
assert!(super::common::rebuild_dmap(&client).await);
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_lock(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
// Spawn first rebuild
let handle = {
let client = client.clone();
tokio::spawn(async move { super::common::rebuild_dmap(&client).await })
};
// Immidiately spawn another
let res = client
.client()
.post(format!(
"{}/w/test-workspace/workspaces/rebuild_dependency_map",
client.baseurl()
))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
// Should tell us there is already rebuilt in progress
// Or if it is too fast we will be able to trigger it second time
assert!(&res == "There is already one task pending, try again later." || &res == "Success");
assert!(handle.await.unwrap());
Ok(())
}
// If you deploy from cli and you use raw requirements you don't want the script be included in dmap
// Otherwise script will be overwritten once any relative import is updated
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_with_requirements_txt(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script",
Some("# from requirements.txt".to_string()),
Some("000000000005165B".into()),
),
)
.await
.unwrap();
assert_dmap(
&db,
Some("f/rel/root_script".into()),
vec![
("f/rel/root_script", "script", "f/rel/branch", ""),
("f/rel/root_script", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/leaf_2", ""),
],
)
.await;
tokio::time::sleep(std::time::Duration::from_secs(13)).await;
assert_dmap(
&db,
Some("f/rel/root_script".into()),
Vec::<(String, String, String, String)>::new(),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_without_requirements_txt(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script",
// We still want to pass lock to it.
Some("# py311".to_string()),
Some("000000000005165B".into()),
),
)
.await
.unwrap();
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
// tokio::time::sleep(std::time::Duration::from_secs(13)).await;
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
// Consider simple one. Only referenced directly. No deep connections
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_2(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf3';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_2_renamed",
None,
Some("0000000000051659".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
// Consider hard one. Referenced deeply and exists in double references.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_1(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf1';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_1_renamed",
None,
Some("0000000000051658".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_branch(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.leaf_1 import main as lf_1;
def main():
return lf_1();
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/branch_renamed",
None,
Some("000000000005165A".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing branches SHOULD change dependency map
// Though it should only change branch item in dmap when it is importer.
// All entries when branch is imported should not change.
let mut corrected_dmap = CORRECT_DMAP.clone();
// Corresponds to importer path of branch entry
corrected_dmap[0].0 = "f/rel/branch_renamed";
assert_dmap(&db, None, corrected_dmap).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_script(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script_renamed",
None,
Some("000000000005165B".into()),
),
)
.await
.unwrap();
let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(&db, None, corrected_dmap.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value};
let (client, port, _s) = init(db.clone()).await;
let flow = fetch_version(&db, 1443253234253454).await.unwrap();
let res = client
.client()
.post(format!(
"{}/w/test-workspace/flows/update/{}",
client.baseurl(),
"f/rel/root_flow" // encode_path()
))
.json(&NewFlow {
path: "f/rel/root_flow_renamed".into(),
summary: "".into(),
description: None,
value: to_raw_value(
&serde_json::from_str::<serde_json::Value>(
&serde_json::to_string(flow.value())
.unwrap()
.replace("nstep1", "Foxes")
.replace("nstep2_2", "like")
.replace("nstep_4_1", "Emeralds"),
)
.unwrap(),
),
schema: None,
draft_only: None,
tag: None,
dedicated_worker: None,
timeout: None,
deployment_message: None,
visible_to_runner_only: None,
on_behalf_of_email: None,
ws_error_handler_muted: None,
})
.send()
.await
.unwrap();
assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed");
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_flow", "f/rel/root_flow_renamed"),
("nstep1", "Foxes"),
("nstep2_2", "like"),
("nstep_4_1", "Emeralds"),
]),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_app(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
let app_value: String =
sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2")
.fetch_one(&db)
.await
.unwrap()
.unwrap();
// TODO: There is:
// 1. update app
// 2. create app
// 3. update app raw
// Ideally all of them should be handled
let res = client
.client()
.post(format!(
"{}/w/test-workspace/apps/update/{}",
client.baseurl(),
"f/rel/root_app" // encode_path()
))
.json(&windmill_api::EditApp {
path: Some("f/rel/root_app_renamed".into()),
summary: None,
value: serde_json::from_str(
&app_value
.replace("dontpressmeplz", "Apps")
.replace("youcanpressme", "Work"),
)
.unwrap(),
policy: None,
deployment_message: None,
custom_path: None,
})
.send()
.await
.unwrap();
assert_eq!(
res.text().await.unwrap(),
"app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")"
);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_app", "f/rel/root_app_renamed"),
("dontpressmeplz", "Apps"),
("youcanpressme", "Work"),
]),
)
.await;
Ok(())
}
}
#[cfg(feature = "test_job_debouncing")]
mod dependency_job_debouncing {
async fn trigger_djob_for(
backend/tests/dependency_map.rs
+535
View File
@@ -0,0 +1,535 @@
use sqlx::{Pool, Postgres};
use tokio_stream::StreamExt;
use windmill_api_client::types::NewScript;
mod common;
use common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer};
mod dependency_map {
use super::*;
fn quick_ns(
content: &str,
language: windmill_api_client::types::ScriptLang,
path: &str,
lock: Option<String>,
parent_hash: Option<String>,
) -> NewScript {
NewScript {
content: content.into(),
language,
lock,
parent_hash,
path: path.into(),
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: None,
description: "".to_string(),
draft_only: None,
envs: vec![],
is_template: None,
kind: None,
summary: "".to_string(),
tag: None,
schema: std::collections::HashMap::new(),
ws_error_handler_muted: Some(false),
priority: None,
delete_after_use: None,
timeout: None,
restart_unless_cancelled: None,
deployment_message: None,
concurrency_key: None,
visible_to_runner_only: None,
no_main_func: None,
codebase: None,
has_preprocessor: None,
on_behalf_of_email: None,
assets: vec![],
}
}
async fn init(db: Pool<Postgres>) -> (windmill_api_client::Client, u16, ApiServer) {
init_client(db).await
}
async fn _clear_dmap(db: &Pool<Postgres>) {
sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'")
.execute(db)
.await
.unwrap();
}
/// Corrects map according to provided replacements.
/// Only changes importer_path and/or id
/// Does not affect imported_path nor kind!
fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> {
CORRECT_DMAP
.clone()
.into_iter()
.map(|e| {
let mut r = (
e.0.to_owned(),
e.1.to_owned(),
e.2.to_owned(),
e.3.to_owned(),
);
for (from, to) in &replacements {
r = (
r.0.replace(from, to),
r.1, // Kind should be immutable
r.2, // Imported path should be immutable
// We do not modify script contents in test, so we can assume scripts always import the same path
// Modification of kind or imported path considered to be incorrect.
r.3.replace(from, to),
);
}
r
})
.collect()
}
async fn assert_dmap(
db: &Pool<Postgres>,
importer: Option<String>,
expected: Vec<(
impl Into<String>,
impl Into<String>,
impl Into<String>,
impl Into<String>,
)>,
) {
let mut dmap = sqlx::query_as::<_, (String, String, String, String)>(
"SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)",
)
.bind(importer)
.fetch_all(db)
.await
.unwrap();
let mut expected = expected
.into_iter()
.map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into()))
.collect::<Vec<(String, String, String, String)>>();
dmap.sort();
expected.sort();
assert_eq!(dmap, expected);
}
lazy_static::lazy_static! {
pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![
("f/rel/root_script", "script", "dependencies/test.requirements.in", ""),
("f/rel/root_flow", "flow", "dependencies/test.requirements.in", "nstep1"),
("f/rel/root_app", "app", "dependencies/test.requirements.in", "dontpressmeplz"),
("f/rel/branch", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/branch", ""),
("f/rel/root_script", "script", "f/rel/leaf_1", ""),
("f/rel/root_script", "script", "f/rel/leaf_2", ""),
("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"),
("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"),
("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"),
("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"),
("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"),
("f/rel/root_app", "app", "f/rel/branch", "youcanpressme"),
// Default
("f/rel/leaf_1", "script", "dependencies/requirements.in", ""),
("f/rel/leaf_2", "script", "dependencies/requirements.in", ""),
("f/rel/branch", "script", "dependencies/requirements.in", ""),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "failure"),
("f/rel/root_flow", "flow", "dependencies/package.json", "nstep2_1"),
("f/rel/root_flow", "flow", "dependencies/package.json", "nstep3_2"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep2_2"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep3_1"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep4_1"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep5_1"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "preprocessor"),
("f/rel/root_app", "app", "dependencies/requirements.in", "pressmeplz"),
("f/rel/root_flow", "flow", "dependencies/requirements.in", "qtool1"),
("f/rel/root_app", "app", "dependencies/requirements.in", "youcanpressme")
];
}
// TODO:
// Test that checks that we can run rebuild_dmap multiple times in tests.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_correctness(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
// rebuild map
assert!(common::rebuild_dmap(&client).await);
assert_dmap(&db, None, CORRECT_DMAP.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rebuild_lock(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, _port, _s) = init(db.clone()).await;
// Spawn first rebuild
let handle = {
let client = client.clone();
tokio::spawn(async move { common::rebuild_dmap(&client).await })
};
// Immidiately spawn another
let res = client
.client()
.post(format!(
"{}/w/test-workspace/workspaces/rebuild_dependency_map",
client.baseurl()
))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
// Should tell us there is already rebuilt in progress
// Or if it is too fast we will be able to trigger it second time
assert!(&res == "There is already one task pending, try again later." || &res == "Success");
assert!(handle.await.unwrap());
Ok(())
}
// If you deploy from cli and you use raw requirements you don't want the script be included in dmap
// Otherwise script will be overwritten once any relative import is updated
// #[cfg(feature = "python")]
// #[sqlx::test(fixtures("base", "dependency_map"))]
// async fn relative_imports_test_with_legacy(db: Pool<Postgres>) -> anyhow::Result<()> {
// let (client, _port, _s) = init(db.clone()).await;
// client
// .create_script(
// "test-workspace",
// &quick_ns(
// "
// from f.rel.branch import main as br;
// from f.rel.leaf_1 import main as lf_1;
// from f.rel.leaf_2 import main as lf_2;
// def main():
// return [br(), lf_1(), lf_2];
// ",
// windmill_api_client::types::ScriptLang::Python3,
// "f/rel/root_script",
// Some("# from requirements.txt".to_string()),
// Some("000000000005165B".into()),
// ),
// )
// .await
// .unwrap();
// assert_dmap(
// &db,
// Some("f/rel/root_script".into()),
// vec![
// ("f/rel/root_script", "script", "f/rel/branch", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_1", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_2", ""),
// ],
// )
// .await;
// tokio::time::sleep(std::time::Duration::from_secs(13)).await;
// assert_dmap(
// &db,
// Some("f/rel/root_script".into()),
// vec![
// ("f/rel/root_script", "script", "f/rel/branch", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_1", ""),
// ("f/rel/root_script", "script", "f/rel/leaf_2", ""),
// ],
// )
// .await;
// Ok(())
// }
// Consider simple one. Only referenced directly. No deep connections
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_2(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf3';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_2_renamed",
None,
Some("0000000000051659".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(
&db,
None,
corrected_dmap(vec![("f/rel/leaf_2", "f/rel/leaf_2_renamed")]),
)
.await;
Ok(())
}
// Consider hard one. Referenced deeply and exists in double references.
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_leaf_1(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
def main():
return 'leaf1';
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/leaf_1_renamed",
None,
Some("0000000000051658".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing leafs should not change dependency map
assert_dmap(
&db,
None,
corrected_dmap(vec![("f/rel/leaf_1", "f/rel/leaf_1_renamed")]),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_branch(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
from f.rel.leaf_1 import main as lf_1;
def main():
return lf_1();
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/branch_renamed",
None,
Some("000000000005165A".into()),
),
)
.await
.unwrap();
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
// Changing branches SHOULD change dependency map
// Though it should only change branch item in dmap when it is importer.
// All entries when branch is imported should not change.
let corrected_dmap = CORRECT_DMAP
.clone()
.iter_mut()
.map(|el| {
if el.0 == "f/rel/branch" {
el.0 = "f/rel/branch_renamed";
}
*el
})
.collect::<Vec<_>>();
assert_dmap(&db, None, corrected_dmap).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_script(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"
# requirements: test
from f.rel.branch import main as br;
from f.rel.leaf_1 import main as lf_1;
from f.rel.leaf_2 import main as lf_2;
def main():
return [br(), lf_1(), lf_2];
",
windmill_api_client::types::ScriptLang::Python3,
"f/rel/root_script_renamed",
None,
Some("000000000005165B".into()),
),
)
.await
.unwrap();
let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(&db, None, corrected_dmap.clone()).await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value};
let (client, port, _s) = init(db.clone()).await;
let flow = fetch_version(&db, 1443253234253454).await.unwrap();
let res = client
.client()
.post(format!(
"{}/w/test-workspace/flows/update/{}",
client.baseurl(),
"f/rel/root_flow" // encode_path()
))
.json(&NewFlow {
path: "f/rel/root_flow_renamed".into(),
summary: "".into(),
description: None,
value: to_raw_value(
&serde_json::from_str::<serde_json::Value>(
&serde_json::to_string(flow.value())
.unwrap()
.replace("nstep1", "Foxes")
.replace("nstep2_2", "like")
.replace("nstep_4_1", "Emeralds"),
)
.unwrap(),
),
schema: None,
draft_only: None,
tag: None,
dedicated_worker: None,
timeout: None,
deployment_message: None,
visible_to_runner_only: None,
on_behalf_of_email: None,
ws_error_handler_muted: None,
})
.send()
.await
.unwrap();
assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed");
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_flow", "f/rel/root_flow_renamed"),
("nstep1", "Foxes"),
("nstep2_2", "like"),
("nstep_4_1", "Emeralds"),
]),
)
.await;
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "dependency_map"))]
async fn relative_imports_test_rename_primary_app(db: Pool<Postgres>) -> anyhow::Result<()> {
let (client, port, _s) = init(db.clone()).await;
let app_value: String =
sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2")
.fetch_one(&db)
.await
.unwrap()
.unwrap();
// TODO: There is:
// 1. update app
// 2. create app
// 3. update app raw
// Ideally all of them should be handled
let res = client
.client()
.post(format!(
"{}/w/test-workspace/apps/update/{}",
client.baseurl(),
"f/rel/root_app" // encode_path()
))
.json(&windmill_api::EditApp {
path: Some("f/rel/root_app_renamed".into()),
summary: None,
value: serde_json::from_str(
&app_value
.replace("dontpressmeplz", "Apps")
.replace("youcanpressme", "Work"),
)
.unwrap(),
policy: None,
deployment_message: None,
custom_path: None,
})
.send()
.await
.unwrap();
assert_eq!(
res.text().await.unwrap(),
"app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")"
);
let mut completed = listen_for_completed_jobs(&db).await;
in_test_worker(&db, completed.next(), port).await;
assert_dmap(
&db,
None,
corrected_dmap(vec![
("f/rel/root_app", "f/rel/root_app_renamed"),
("dontpressmeplz", "Apps"),
("youcanpressme", "Work"),
]),
)
.await;
Ok(())
}
}
backend/tests/fixtures/dependency_map.sql
Vendored
+35 -3
View File
File diff suppressed because one or more lines are too long
backend/tests/fixtures/hub_sync_blacklist.sql
Vendored
+15
View File
@@ -0,0 +1,15 @@
-- Fixture for testing hub_sync blacklist from workspace dependencies
-- hub_sync and apps are already built-in via migrations
-- Insert a simple Bun script that should be affected by workspace dependencies
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'admins',
'test-user',
'
export async function main() {
return "Simple bun script";
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'Simple bun script',
'',
'u/admin/simple_bun', 700001, 'bun', '');
backend/tests/fixtures/workspace_dependencies.sql
Vendored
+33
View File
@@ -0,0 +1,33 @@
-- NOTE: Applied after workspace_dependencies_leafs.sql
-- Python script that uses relative imports
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
from f.leafs.python import main as python_leaf
def main():
return {"python_import": python_leaf()}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/python_importer', 500005, 'python3', '');
-- TypeScript script that uses relative imports
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
import { main as tsLeaf } from "./leafs/ts";
export async function main() {
return { ts_import: await tsLeaf() };
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/ts_importer', 500006, 'nativets', '');
-- Dependency map entries
INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/python_importer', 'script', 'f/leafs/python', '');
INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/ts_importer', 'script', 'f/leafs/ts', '');
backend/tests/fixtures/workspace_dependencies_leafs.sql
Vendored
+54
View File
@@ -0,0 +1,54 @@
-- Basic leaf scripts in different languages
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
export async function main() {
return "TypeScript leaf";
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/ts', 500001, 'nativets', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
package main
import "fmt"
func main() {
fmt.Println("Go leaf")
}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/go', 500002, 'go', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
def main():
return "Python leaf"',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/python', 500003, 'python3', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'
<?php
function main() {
return "PHP leaf";
}
?>',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/leafs/php', 500004, 'php', '');
backend/tests/python_jobs.rs
+12 -4
View File
@@ -25,7 +25,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.11.11", "tiny==0.1.3"],
vec!["# workspace-dependencies-mode: manual\n# py: 3.11.11","tiny==0.1.3"],
)
.await?;
Ok(())
@@ -55,7 +55,11 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.11.11", "bottle==0.13.2", "tiny==0.1.2"],
vec![
"# workspace-dependencies-mode: extra\n# py: 3.11.11",
"bottle==0.13.2",
"tiny==0.1.2",
],
)
.await?;
}
@@ -79,7 +83,11 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.11.11", "simplejson==3.20.1", "tiny==0.1.3"],
vec![
"# workspace-dependencies-mode: extra\n# py: 3.11.11",
"simplejson==3.20.1",
"tiny==0.1.3",
],
)
.await?;
Ok(())
@@ -108,7 +116,7 @@ def main():
content,
ScriptLang::Python3,
vec![
"# py: 3.11.11",
"# workspace-dependencies-mode: extra\n# py: 3.11.11",
"bottle==0.13.2",
"microdot==2.2.0",
"simplejson==3.19.3",
backend/tests/workspace_dependencies.rs
+194
View File
@@ -0,0 +1,194 @@
mod common;
mod workspace_dependencies {
use crate::common::in_test_worker;
use crate::common::init_client;
use crate::common::listen_for_completed_jobs;
use sqlx::{Pool, Postgres};
use tokio_stream::StreamExt;
use windmill_common::scripts::ScriptLang;
use windmill_worker::workspace_dependencies::NewWorkspaceDependencies;
mod deps {
pub const REQUIREMENTS_IN: &'static str = "tiny==0.1.3";
// pub const GO_MOD: &'static str = r##"
// module example.com/project
// go 1.20
// require github.com/gin-gonic/gin v1.8.1
// "##;
pub const PACKAGE_JSON: &'static str = r##"
{
"name": "example-project",
"version": "1.0.0",
"dependencies": {
"express": "^4.17.1"
}
}
"##;
pub const COMPOSER_JSON: &'static str = r##"
{
"name": "example/project",
"require": {
"monolog/monolog": "^2.3"
}
}
"##;
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "workspace_dependencies_leafs"))]
#[ignore]
async fn basic_manual_named(db: Pool<Postgres>) -> anyhow::Result<()> {
let ((_client, port, _s), db, mut completed) = (
init_client(db.clone()).await,
&db,
listen_for_completed_jobs(&db).await,
);
for (idx, (l, c)) in [
(ScriptLang::Python3, deps::REQUIREMENTS_IN),
(ScriptLang::Bun, deps::PACKAGE_JSON),
(ScriptLang::Php, deps::COMPOSER_JSON),
// (ScriptLang::Go, deps::GO_MOD),
]
.iter()
.enumerate()
{
let id = NewWorkspaceDependencies {
workspace_id: "test-workspace".into(),
language: *l,
content: (*c).into(),
name: Some("test".to_owned()),
description: None,
}
.create("", "", "", db)
.await
.unwrap();
assert_eq!(idx + 1, id as usize);
}
// Wait for 4 jobs.
// Creating those dependencies will trigger redeployment of all scripts in workspace_dependencies_leafs.sql
in_test_worker(
db,
async {
completed.next().await;
completed.next().await;
completed.next().await;
// completed.next().await;
},
port,
)
.await;
// Verify all scripts have correct locks
// let mut langs = vec![];
// for r in sqlx::query!(
// r#"SELECT language AS "language: ScriptLang",lock FROM script WHERE archived = false"#
// )
// .fetch_all(db)
// .await
// .unwrap()
// {
// match r.language {
// ScriptLang::Python3 => assert_eq!("", &r.lock.unwrap()),
// ScriptLang::Go => todo!(),
// ScriptLang::Bun => todo!(),
// ScriptLang::Bunnative => todo!(),
// ScriptLang::Php => todo!(),
// _ => panic!("Unsupported language"),
// }
// langs.push(r.language);
// }
// langs.sort();
// // Just tiny additional verification for peace of mind.
// assert_eq!(langs.as_slice(), &[]);
Ok(())
}
#[sqlx::test(fixtures("base", "hub_sync_blacklist"))]
async fn hub_sync_blacklist_from_workspace_deps(db: Pool<Postgres>) -> anyhow::Result<()> {
let ((_client, port, _s), db, mut completed) = (
init_client(db.clone()).await,
&db,
listen_for_completed_jobs(&db).await,
);
// Verify built-in fixtures exist
// Check that the setup_app exists
let app_exists =
sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')")
.fetch_one(db)
.await
.unwrap();
assert!(app_exists.unwrap(), "Expected g/all/setup_app to exist");
// Check that hub_sync script exists and is a Bun script
let hub_sync_lang = sqlx::query_scalar!(
r#"SELECT language AS "language: ScriptLang" FROM script WHERE path = 'u/admin/hub_sync'"#
)
.fetch_one(db)
.await
.unwrap();
assert_eq!(
hub_sync_lang,
ScriptLang::Bun,
"Expected hub_sync to be a Bun script"
);
// Create unnamed (default) workspace dependencies for Bun
let _id = NewWorkspaceDependencies {
workspace_id: "admins".into(),
language: ScriptLang::Bun,
content: deps::PACKAGE_JSON.into(),
name: None, // No name = default workspace dependencies
description: None,
}
.create("", "", "", db)
.await
.unwrap();
// Wait for exactly 1 job (only u/admin/simple_bun, not hub_sync)
let job_id = in_test_worker(db, async { completed.next().await }, port)
.await
.expect("Expected one job to complete");
// Query the job's runnable_path
let runnable_path =
sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", job_id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(
runnable_path,
Some("u/admin/simple_bun".to_string()),
"Expected job runnable_path to be 'u/admin/simple_bun' (hub_sync should be blacklisted)"
);
// Assert total job count is 1
let job_count = sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job")
.fetch_one(db)
.await
.unwrap();
assert_eq!(job_count, Some(1), "Expected exactly one job total");
// Assert v2_job_queue is empty
let queue_count = sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue")
.fetch_one(db)
.await
.unwrap();
assert_eq!(queue_count, Some(0), "Expected job queue to be empty");
Ok(())
}
}
backend/windmill-api/openapi.yaml
+248
View File
@@ -2249,6 +2249,56 @@ paths:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/get_dependents/{imported_path}:
get:
summary: get dependents of an imported path
operationId: getDependents
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: imported_path
in: path
required: true
schema:
type: string
description: The imported path to get dependents for
responses:
"200":
description: list of dependents
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/DependencyDependent"
/w/{workspace}/workspaces/get_dependents_amounts:
post:
summary: get dependents amounts for multiple imported paths
operationId: getDependentsAmounts
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: List of imported paths to get dependents counts for
required: true
content:
application/json:
schema:
type: array
items:
type: string
responses:
"200":
description: list of dependents amounts
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/DependentsAmount"
/w/{workspace}/workspaces/get_dependency_map:
get:
@@ -5400,6 +5450,126 @@ paths:
schema:
type: boolean
/w/{workspace}/workspace_dependencies/create:
post:
summary: create workspace dependencies
operationId: createWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: New workspace dependencies
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/NewWorkspaceDependencies"
responses:
"201":
description: workspace dependencies created
content:
text/plain:
schema:
type: string
/w/{workspace}/workspace_dependencies/archive/{language}:
post:
summary: archive workspace dependencies (require admin)
operationId: archiveWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: language
in: path
required: true
schema:
$ref: "#/components/schemas/ScriptLang"
- name: name
in: query
required: false
schema:
type: string
responses:
"200":
description: result
content:
application/json:
schema: {}
/w/{workspace}/workspace_dependencies/delete/{language}:
post:
summary: delete workspace dependencies (require admin)
operationId: deleteWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: language
in: path
required: true
schema:
$ref: "#/components/schemas/ScriptLang"
- name: name
in: query
required: false
schema:
type: string
responses:
"200":
description: result
content:
application/json:
schema: {}
/w/{workspace}/workspace_dependencies/list:
get:
summary: list all workspace dependencies
operationId: listWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: All workspace dependencies
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/WorkspaceDependencies"
/w/{workspace}/workspace_dependencies/get_latest/{language}:
get:
summary: get latest workspace dependencies by language and name
operationId: getLatestWorkspaceDependencies
tags:
- workspace_dependencies
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: language
in: path
required: true
schema:
$ref: "#/components/schemas/ScriptLang"
- name: name
in: query
required: false
schema:
type: string
responses:
"200":
description: Latest workspace dependencies
content:
application/json:
schema:
$ref: "#/components/schemas/WorkspaceDependencies"
/w/{workspace}/scripts/archive/p/{path}:
post:
summary: archive script by path
@@ -15759,6 +15929,52 @@ components:
items:
$ref: "#/components/schemas/Alert"
WorkspaceDependencies:
type: object
properties:
id:
type: integer
archived:
type: boolean
name:
type: string
description:
type: string
content:
type: string
language:
$ref: "#/components/schemas/ScriptLang"
workspace_id:
type: string
created_at:
type: string
format: date-time
required:
- workspace_id
- language
- created_at
- content
- id
- archived
NewWorkspaceDependencies:
type: object
properties:
workspace_id:
type: string
language:
$ref: "#/components/schemas/ScriptLang"
name:
type: string
description:
type: string
content:
type: string
required:
- workspace_id
- language
- content
Script:
type: object
properties:
@@ -18957,6 +19173,38 @@ components:
type: string
nullable: true
DependencyDependent:
type: object
properties:
importer_path:
type: string
importer_kind:
type: string
enum:
- script
- flow
- app
importer_node_ids:
type: array
items:
type: string
nullable: true
required:
- importer_path
- importer_kind
DependentsAmount:
type: object
properties:
imported_path:
type: string
count:
type: integer
format: int64
required:
- imported_path
- count
WorkspaceInvite:
type: object
properties:
backend/windmill-api/src/jobs.rs
+64 -31
View File
@@ -25,6 +25,7 @@ use std::str::FromStr;
use std::time::Instant;
use tokio::io::AsyncReadExt;
use tower::ServiceBuilder;
use url::Url;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
use windmill_common::auth::is_super_admin_email;
use windmill_common::auth::TOKEN_PREFIX_LEN;
@@ -39,6 +40,9 @@ use windmill_common::jobs::{
use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat};
use windmill_common::utils::{RunnableKind, WarnAfterExt};
use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR};
use windmill_common::workspace_dependencies::{
RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES,
};
use windmill_common::DYNAMIC_INPUT_CACHE;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
use windmill_common::{email_oss::send_email_html, server::load_smtp_config};
@@ -6116,14 +6120,17 @@ async fn run_bundle_preview_script(
Ok((StatusCode::CREATED, job_id.unwrap().to_string()))
}
#[derive(Deserialize)]
#[derive(Deserialize, Debug)]
pub struct RunDependenciesRequest {
pub raw_scripts: Vec<RawScriptForDependencies>,
pub entrypoint: String,
#[serde(default)]
pub raw_workspace_dependencies: Option<RawWorkspaceDependencies>,
#[serde(default)]
pub raw_deps: Option<String>,
}
#[derive(Deserialize, Clone)]
#[derive(Deserialize, Clone, Debug)]
pub struct RawScriptForDependencies {
pub script_path: String,
pub raw_code: Option<String>,
@@ -6148,41 +6155,49 @@ async fn run_dependencies_job(
));
}
if req.raw_deps.is_some() {
return Err(error::Error::MigrationNeeded {
feature: "cli is outdated".into(),
version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(),
guide_url: Url::from_str(
"https://www.windmill.dev/docs/core_concepts/workspace_dependencies/migration",
)?,
});
}
// Check if workers support workspace dependencies feature
if req.raw_workspace_dependencies.is_some() {
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
.await?;
}
if req.raw_scripts.len() != 1 || req.raw_scripts[0].script_path != req.entrypoint {
return Err(error::Error::internal_err(
"For now only a single raw script can be passed to this endpoint, and the entrypoint should be set to the script path".to_string(),
));
}
let raw_script = req.raw_scripts[0].clone();
let script_path = raw_script.script_path;
let ehm = HashMap::new();
let raw_code = raw_script.raw_code.unwrap_or_else(|| "".to_string());
let language = raw_script.language;
let (args, raw_code) = if let Some(deps) = req.raw_deps {
let mut hm = HashMap::new();
hm.insert(
"raw_deps".to_string(),
JsonRawValue::from_string("true".to_string()).unwrap(),
);
if language == ScriptLang::Bun {
let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code);
hm.insert(
"npm_mode".to_string(),
JsonRawValue::from_string(annotation.npm.to_string()).unwrap(),
);
}
(PushArgs { extra: Some(hm), args: &ehm }, deps)
} else {
(PushArgs::from(&ehm), raw_code)
};
let RawScriptForDependencies {
// unwrap
script_path,
raw_code,
language,
} = req.raw_scripts[0].clone();
let mut hm = HashMap::new();
req.raw_workspace_dependencies
.map(|v| hm.insert("raw_workspace_dependencies".to_owned(), to_raw_value(&v)));
let (uuid, tx) = push(
&db,
PushIsolationLevel::IsolatedRoot(db.clone()),
&w_id,
JobPayload::RawScriptDependencies { script_path, content: raw_code, language },
args,
JobPayload::RawScriptDependencies {
script_path,
content: raw_code.unwrap_or_default(),
language,
},
PushArgs { extra: Some(hm), args: &HashMap::new() },
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -6218,6 +6233,9 @@ async fn run_dependencies_job(
pub struct RunFlowDependenciesRequest {
pub path: String,
pub flow_value: FlowValue,
#[serde(default)]
pub raw_workspace_dependencies: Option<RawWorkspaceDependencies>,
#[serde(default)]
pub raw_deps: Option<HashMap<String, String>>,
}
@@ -6239,13 +6257,28 @@ async fn run_flow_dependencies_job(
));
}
// Create args HashMap with skip_flow_update and raw_deps if present
if req.raw_deps.is_some() {
return Err(error::Error::MigrationNeeded {
feature: "cli is outdated".into(),
version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(),
guide_url: Url::from_str(
"https://www.windmill.dev/docs/core_concepts/workspace_dependencies/migration",
)?,
});
}
// Check if workers support workspace dependencies feature
if req.raw_workspace_dependencies.is_some() {
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
.await?;
}
// Create args HashMap with skip_flow_update and raw_workspace_dependencies if present
let mut args_map = HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]);
// Add raw_deps to args if present
if let Some(ref raw_deps) = req.raw_deps {
args_map.insert("raw_deps".to_string(), to_raw_value(raw_deps));
}
// Add raw_workspace_dependencies to args if present
req.raw_workspace_dependencies
.map(|v| args_map.insert("raw_workspace_dependencies".to_string(), to_raw_value(&v)));
let (uuid, tx) = push(
&db,
backend/windmill-api/src/lib.rs
+5
View File
@@ -110,6 +110,7 @@ mod openapi;
#[cfg(all(feature = "private", feature = "parquet"))]
pub mod s3_proxy_ee;
mod s3_proxy_oss;
mod workspace_dependencies;
mod approvals;
#[cfg(all(feature = "enterprise", feature = "private"))]
@@ -455,6 +456,10 @@ pub async fn run_server(
.nest("/drafts", drafts::workspaced_service())
.nest("/favorites", favorite::workspaced_service())
.nest("/flows", flows::workspaced_service())
.nest(
"/workspace_dependencies",
workspace_dependencies::workspaced_service(),
)
.nest(
"/flow_conversations",
flow_conversations::workspaced_service(),
backend/windmill-api/src/scripts.rs
+3 -10
View File
@@ -41,11 +41,11 @@ use windmill_audit::ActionKind;
use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap};
use windmill_common::{
assets::{AssetUsageKind, AssetWithAltAccessType, clear_asset_usage, insert_asset_usage},
assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType},
error::to_anyhow,
s3_helpers::upload_artifact_to_store,
scripts::hash_script,
utils::{WarnAfterExt, paginate_without_limits},
utils::{paginate_without_limits, WarnAfterExt},
worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING},
};
@@ -60,9 +60,7 @@ use windmill_common::{
ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred,
},
users::username_to_permissioned_as,
utils::{
not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath,
},
utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath},
worker::to_raw_value,
HUB_BASE_URL,
};
@@ -1045,14 +1043,10 @@ async fn create_script_internal<'c>(
let permissioned_as2 = permissioned_as.clone();
let script_path2 = script_path.clone();
let parent_path = p_path_opt.clone();
let lock = ns.lock.clone();
let deployment_message = ns.deployment_message.clone();
let content = ns.content.clone();
let language = ns.language.clone();
tokio::spawn(async move {
// TODO: I don't think we want this. We might want to send dependency job. But skip any calculations if lock is already present.
// It will allow us to make code more consistent and predictable.
// wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
if let Err(e) = process_relative_imports(
@@ -1068,7 +1062,6 @@ async fn create_script_internal<'c>(
&authed2.email,
&authed2.username,
&permissioned_as2,
lock,
)
.await
{
backend/windmill-api/src/workspace_dependencies.rs
+145
View File
@@ -0,0 +1,145 @@
use axum::{
extract::{Path, Query},
routing::{get, post},
Extension, Json, Router,
};
use http::StatusCode;
use serde::Deserialize;
use windmill_common::{
error::{self, JsonResult},
scripts::ScriptLang,
users::username_to_permissioned_as,
utils::require_admin,
workspace_dependencies::WorkspaceDependencies,
DB,
};
use windmill_worker::{
scoped_dependency_map, trigger_dependents_to_recompute_dependencies,
workspace_dependencies::NewWorkspaceDependencies,
};
use crate::db::ApiAuthed;
pub fn workspaced_service() -> Router {
Router::new()
.route("/create", post(create))
.route("/list", get(list))
.route("/archive/:language", post(archive))
.route("/get_latest/:language", get(get_latest))
.route("/delete/:language", post(delete))
}
#[axum::debug_handler]
async fn create(
authed: ApiAuthed,
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Json(nwd): Json<NewWorkspaceDependencies>,
) -> error::Result<(StatusCode, String)> {
tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
Ok((
StatusCode::CREATED,
format!(
"{}",
nwd.create(
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
&db
)
.await?
),
))
}
#[axum::debug_handler]
async fn list(
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<WorkspaceDependencies>> {
tracing::info!(workspace_id = %w_id, "list workspace dependencies");
Ok(Json(WorkspaceDependencies::list(&w_id, &db).await?))
}
#[derive(Deserialize)]
pub(super) struct NameQuery {
name: Option<String>,
}
#[axum::debug_handler]
pub(super) async fn get_latest(
Extension(db): Extension<DB>,
Path((w_id, language)): Path<(String, ScriptLang)>,
Query(params): Query<NameQuery>,
) -> JsonResult<Option<WorkspaceDependencies>> {
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "get latest workspace dependencies");
Ok(Json(
WorkspaceDependencies::get_latest(params.name, language, &w_id, db.into()).await?,
))
}
#[axum::debug_handler]
async fn archive(
authed: ApiAuthed,
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, language)): Path<(String, ScriptLang)>,
Query(params): Query<NameQuery>,
) -> error::Result<()> {
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let db = &db;
WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?;
trigger_dependents_to_recompute_dependencies(
&w_id,
scoped_dependency_map::ScopedDependencyMap::get_dependents(
WorkspaceDependencies::to_path(&params.name, language)?.as_str(),
&w_id,
db,
)
.await?,
None,
None,
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
db,
vec![],
)
.await
}
#[axum::debug_handler]
async fn delete(
authed: ApiAuthed,
// Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, language)): Path<(String, ScriptLang)>,
Query(params): Query<NameQuery>,
) -> error::Result<()> {
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let db = &db;
WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?;
trigger_dependents_to_recompute_dependencies(
&w_id,
scoped_dependency_map::ScopedDependencyMap::get_dependents(
WorkspaceDependencies::to_path(&params.name, language)?.as_str(),
&w_id,
db,
)
.await?,
None,
None,
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
db,
vec![],
)
.await
}
backend/windmill-api/src/workspaces.rs
+100 -3
View File
@@ -51,7 +51,9 @@ use windmill_common::{
utils::{paginate, rd_string, require_admin, Pagination},
};
use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject};
use windmill_worker::scoped_dependency_map::{DependencyMap, ScopedDependencyMap};
use windmill_worker::scoped_dependency_map::{
DependencyDependent, DependencyMap, ScopedDependencyMap,
};
#[cfg(feature = "enterprise")]
use windmill_common::utils::require_admin_or_devops;
@@ -82,6 +84,8 @@ pub fn workspaced_service() -> Router {
.route("/delete_invite", post(delete_invite))
.route("/rebuild_dependency_map", post(rebuild_dependency_map))
.route("/get_dependency_map", get(get_dependency_map))
.route("/get_dependents/*imported_path", get(get_dependents))
.route("/get_dependents_amounts", post(get_dependents_amounts))
.route("/get_settings", get(get_settings))
.route("/get_deploy_to", get(get_deploy_to))
.route("/edit_slack_command", post(edit_slack_command))
@@ -2553,8 +2557,11 @@ async fn clone_workspace_data(
clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?;
// Clone workspace runnable dependencies and dependency map
clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?;
clone_workspace_runnable_dependencies(tx, source_workspace_id, target_workspace_id).await?;
// TODO: Enable when git sync is implemented for workspace dependencies.
// // Clone workspace dependencies
// clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?;
Ok(())
}
@@ -3017,7 +3024,7 @@ async fn clone_raw_apps(
Ok(())
}
async fn clone_workspace_dependencies(
async fn clone_workspace_runnable_dependencies(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
target_workspace_id: &str,
@@ -3049,6 +3056,27 @@ async fn clone_workspace_dependencies(
Ok(())
}
#[allow(dead_code)]
async fn clone_workspace_dependencies(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
target_workspace_id: &str,
) -> Result<()> {
// Clone workspace_runnable_dependencies
sqlx::query!(
"INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at)
SELECT $1, language, name, description, content, archived, created_at
FROM workspace_dependencies
WHERE workspace_id = $2",
target_workspace_id,
source_workspace_id
)
.execute(&mut **tx)
.await?;
Ok(())
}
async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result<String> {
return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string()));
}
@@ -3583,6 +3611,75 @@ async fn rebuild_dependency_map(
ScopedDependencyMap::rebuild_map(&w_id, &db).await
}
#[axum::debug_handler]
async fn get_dependents(
Extension(db): Extension<DB>,
Path((w_id, imported_path)): Path<(String, String)>,
_authed: ApiAuthed,
) -> JsonResult<Vec<DependencyDependent>> {
tracing::debug!(
workspace_id = %w_id,
imported_path = %imported_path,
"API: Getting dependents for imported path"
);
let dependents = ScopedDependencyMap::get_dependents(&imported_path, &w_id, &db).await?;
tracing::debug!(
workspace_id = %w_id,
imported_path = %imported_path,
dependents_count = dependents.len(),
"API: Found dependents: {:?}",
dependents
);
Ok(Json(dependents))
}
#[derive(Serialize, Debug)]
struct DependentsAmount {
imported_path: String,
count: i64,
}
#[axum::debug_handler]
async fn get_dependents_amounts(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(imported_paths): Json<Vec<String>>,
) -> JsonResult<Vec<DependentsAmount>> {
tracing::debug!(
workspace_id = %w_id,
imported_paths = ?imported_paths,
"API: Getting dependents amounts for imported paths"
);
let results = sqlx::query_as!(
DependentsAmount,
r#"
SELECT
imported_path,
COUNT(DISTINCT importer_path) as "count!"
FROM dependency_map
WHERE workspace_id = $1 AND imported_path = ANY($2)
GROUP BY imported_path
"#,
w_id,
&imported_paths
)
.fetch_all(&db)
.await?;
tracing::debug!(
workspace_id = %w_id,
results_count = results.len(),
"API: Found dependents amounts: {:?}",
results
);
Ok(Json(results))
}
#[derive(Deserialize)]
struct ChangeWorkspaceName {
new_name: String,
backend/windmill-api/src/workspaces_export.rs
+39
View File
@@ -44,6 +44,7 @@ use axum::{
use http::HeaderName;
use itertools::Itertools;
use windmill_common::utils::require_admin;
use windmill_common::variables::decrypt;
use windmill_common::{
db::UserDB,
@@ -52,6 +53,7 @@ use windmill_common::{
schedule::Schedule,
scripts::{Schema, Script, ScriptLang},
variables::{build_crypt, ExportableListableVariable},
workspace_dependencies::WorkspaceDependencies,
};
use hyper::header;
@@ -177,6 +179,7 @@ pub(crate) struct ArchiveQueryParams {
include_groups: Option<bool>,
include_settings: Option<bool>,
include_key: Option<bool>,
include_workspace_dependencies: Option<bool>,
default_ts: Option<String>,
}
@@ -309,11 +312,20 @@ pub(crate) async fn tarball_workspace(
include_groups,
include_settings,
include_key,
include_workspace_dependencies,
default_ts,
}): Query<ArchiveQueryParams>,
) -> Result<([(HeaderName, String); 2], impl IntoResponse)> {
// require_admin(authed.is_admin, &authed.username)?;
tracing::info!(
"tarball_workspace called for workspace {}: include_workspace_dependencies={:?}, skip_variables={:?}, skip_resources={:?}",
w_id,
include_workspace_dependencies,
skip_variables,
skip_resources
);
let mut tx = user_db.begin(&authed).await?;
let tmp_dir = TempDir::new_in("/tmp/windmill/")?;
@@ -545,6 +557,33 @@ pub(crate) async fn tarball_workspace(
}
}
if include_workspace_dependencies.unwrap_or(false)
&& require_admin(authed.is_admin, &authed.username).is_ok()
{
tracing::info!("Including workspace dependencies in tarball export");
let workspace_dependencies = WorkspaceDependencies::list(&w_id, &db).await?;
tracing::info!(
"Found {} workspace dependencies",
workspace_dependencies.len()
);
for dep in workspace_dependencies {
// let dep_str = &to_string_without_metadata(&dep, false, None).unwrap();
let filename = WorkspaceDependencies::to_path(&dep.name, dep.language)?;
tracing::info!(
"Adding workspace dependency: name={:?}, language={:?}, filename={}",
dep.name,
dep.language,
filename
);
archive.write_to_archive(&dep.content, &filename).await?;
}
} else {
tracing::info!(
"Skipping workspace dependencies: include_workspace_dependencies={:?}",
include_workspace_dependencies
);
}
if include_schedules.unwrap_or(false) {
let schedules = sqlx::query_as::<_, Schedule>(
"SELECT * FROM schedule
backend/windmill-common/Cargo.toml
+3
View File
@@ -33,6 +33,7 @@ thiserror.workspace = true
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_yml.workspace = true
chrono.workspace = true
chrono-tz.workspace = true
hex.workspace = true
@@ -68,6 +69,7 @@ aws-credential-types.workspace = true
aws-smithy-types.workspace = true
base64.workspace = true
bitflags.workspace = true
phf.workspace = true
aws-smithy-types-convert = { workspace = true, optional = true }
aws-sdk-rds = { workspace = true, optional = true }
@@ -91,6 +93,7 @@ strum_macros.workspace = true
url.workspace = true
urlencoding.workspace = true
async-recursion.workspace = true
pep440_rs.workspace = true
semver.workspace = true
croner = "2.2.0"
backend/windmill-common/src/error.rs
+6
View File
@@ -86,6 +86,12 @@ pub enum Error {
Generic(StatusCode, String),
#[error("{feature} is unavailable due to some workers being behind. Do not use the feature or make sure all workers run at least {min_version}")]
WorkersAreBehind { feature: String, min_version: String },
#[error(
"Breaking change was introduced in v{version} ({feature}). Follow this migration guide: {guide_url}"
)]
MigrationNeeded { version: String, feature: String, guide_url: url::Url },
#[error("{0} is unavailable. It is possible for this worker to be behind.")]
FeatureUnavailable(String),
}
impl Error {
backend/windmill-common/src/flows.rs
+7 -1
View File
@@ -246,7 +246,13 @@ impl FlowValue {
}
}
}
BranchOne { branches, .. } | BranchAll { branches, .. } => {
BranchOne { default, branches, .. } => {
Self::traverse_leafs(default.iter().collect(), cb)?;
for branch in branches {
Self::traverse_leafs(branch.modules.iter().collect(), cb)?;
}
}
BranchAll { branches, .. } => {
for branch in branches {
Self::traverse_leafs(branch.modules.iter().collect(), cb)?;
}
backend/windmill-common/src/global_settings.rs
+15
View File
@@ -133,3 +133,18 @@ pub async fn load_value_from_global_settings(
.map(|x| x.value);
Ok(r)
}
pub async fn set_value_in_global_settings(
db: &Pool<Postgres>,
setting_name: &str,
value: serde_json::Value,
) -> error::Result<()> {
sqlx::query!(
"INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
setting_name,
value
)
.execute(db)
.await?;
Ok(())
}
backend/windmill-common/src/jobs.rs
+1 -1
View File
@@ -412,7 +412,7 @@ pub enum JobPayload {
/// Dependency Job, exposed with API. Requirements can be predefined
RawScriptDependencies {
script_path: String,
/// Will reflect raw requirements content (e.g. requirements.txt)
/// Will reflect raw requirements content (e.g. requirements.in)
content: String,
language: ScriptLang,
},
backend/windmill-common/src/lib.rs
+1 -1
View File
@@ -55,7 +55,7 @@ pub mod job_metrics;
pub mod job_s3_helpers_ee;
#[cfg(feature = "parquet")]
pub mod job_s3_helpers_oss;
pub mod lockfiles;
pub mod workspace_dependencies;
#[cfg(feature = "private")]
pub mod git_sync_ee;
backend/windmill-common/src/schema.rs
+1 -9
View File
@@ -345,15 +345,7 @@ fn find_annotation(comm_lit: &str, annotation: &str, code: &str) -> bool {
pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
let annotation = "schema_validation";
use ScriptLang::*;
let comment = match lang {
Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
Rust => "//!",
// for related places search: ADD_NEW_LANG
};
find_annotation(comment, annotation, code)
find_annotation(&lang.as_comment_lit(), annotation, code)
}
#[derive(Serialize, Deserialize, Debug, Clone)]
backend/windmill-common/src/scripts.rs
+83 -1
View File
@@ -17,6 +17,7 @@ use crate::{
assets::AssetWithAltAccessType,
error::{to_anyhow, Error},
utils::http_get_from_hub,
workspace_dependencies::WorkspaceDependenciesAnnotatedRefs,
DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION,
};
@@ -25,12 +26,26 @@ use anyhow::Context;
use backon::ConstantBuilder;
use backon::{BackoffBuilder, Retryable};
use itertools::Itertools;
use regex::Regex;
use serde::de::Error as _;
use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
use crate::utils::StripPath;
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, Default)]
#[derive(
Serialize,
Deserialize,
Debug,
PartialEq,
Copy,
Clone,
Hash,
Eq,
sqlx::Type,
Default,
Ord,
PartialOrd,
)]
#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum ScriptLang {
@@ -90,6 +105,71 @@ impl ScriptLang {
// for related places search: ADD_NEW_LANG
}
}
pub fn as_dependencies_filename(&self) -> Option<String> {
use ScriptLang::*;
Some(
match self {
Bun | Bunnative => "package.json",
Python3 => "requirements.in",
// Go => "go.mod",
Php => "composer.json",
_ => return None,
}
.to_owned(),
)
}
pub fn as_comment_lit(&self) -> String {
use ScriptLang::*;
match self {
Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//",
Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
Rust => "//!",
// for related places search: ADD_NEW_LANG
}
.to_owned()
}
pub fn extract_workspace_dependencies_annotated_refs(
&self,
code: &str,
runnable_path: &str,
) -> Option<WorkspaceDependenciesAnnotatedRefs<String>> {
use ScriptLang::*;
lazy_static::lazy_static! {
static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap();
}
match self {
// TODO: Maybe use regex
Bun | Bunnative => WorkspaceDependenciesAnnotatedRefs::parse(
"//",
"package_json",
code,
None,
runnable_path,
),
Python3 => WorkspaceDependenciesAnnotatedRefs::parse(
"#",
"requirements",
code,
Some(&RE_PYTHON),
runnable_path,
),
Go => {
WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path)
}
Php => WorkspaceDependenciesAnnotatedRefs::parse(
"//",
"composer_json",
code,
None,
runnable_path,
),
_ => return None,
}
}
}
impl FromStr for ScriptLang {
@@ -739,12 +819,14 @@ pub struct ClonedScript {
pub old_script: NewScript,
pub new_hash: i64,
}
// TODO: What if dependency job fails, there is script with NULL in the lock
pub async fn clone_script<'c>(
base_hash: ScriptHash,
w_id: &str,
deployment_message: Option<String>,
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
) -> crate::error::Result<ClonedScript> {
// TODO:!
let s = sqlx::query_as::<_, Script>(
"SELECT * FROM script WHERE hash = $1 AND workspace_id = $2 AND archived = false FOR UPDATE",
)
backend/windmill-common/src/worker.rs
+121 -1
View File
@@ -264,6 +264,10 @@ lazy_static::lazy_static! {
.unwrap_or(false);
pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
/// Global flag indicating if all workers support workspace dependencies feature (>= 1.583.0)
/// This flag is updated during worker initialization by checking the minimum version across all workers
/// When false, creation of workspace dependencies is forbidden and extraction of external workspace dependencies will error
pub static ref MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
/// Global flag indicating if all workers support the debouncing feature (>= 1.566.0)
/// Debouncing consolidates multiple dependency job requests within a time window to avoid redundant work
/// This flag is updated during worker initialization by checking the minimum version across all workers
@@ -505,7 +509,10 @@ pub const ROOT_CACHE_DIR: &str = concatcp!(TMP_DIR, "/cache/");
pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result<File> {
let path = format!("{}/{}", dir, path);
let mut file = File::create(&path)?;
let mut file = File::create(&path).map_err(|e| {
tracing::error!("Failed to create file at {path}: {:?}", &e);
e
})?;
file.write_all(content.as_bytes())?;
file.flush()?;
Ok(file)
@@ -1264,6 +1271,10 @@ pub async fn update_min_version(conn: &Connection) -> bool {
tracing::info!("Minimal worker version: {min_version}");
}
// Workspace dependencies feature requires minimum version across all workers
*MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES.write().await = min_version
>= Version::parse(crate::workspace_dependencies::MIN_VERSION_WORKSPACE_DEPENDENCIES)
.unwrap();
// Debouncing feature requires minimum version 1.566.0 across all workers
// This ensures all workers can handle debounce keys and stale data accumulation
*MIN_VERSION_SUPPORTS_DEBOUNCING.write().await = min_version >= Version::new(1, 566, 0);
@@ -1949,6 +1960,115 @@ pub fn to_raw_value_owned(result: serde_json::Value) -> Box<RawValue> {
.unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap())
}
pub fn split_python_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
requirements
.as_ref()
.lines()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.map(String::from)
.collect()
}
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
#[repr(u32)]
pub enum PyVAlias {
Py310 = 10,
#[default]
Py311,
Py312,
Py313,
}
impl Into<pep440_rs::Version> for PyVAlias {
fn into(self) -> pep440_rs::Version {
pep440_rs::Version::new([self.major() as u64, self as u64])
}
}
impl Into<u32> for PyVAlias {
fn into(self) -> u32 {
self.major() * 100 + self as u32
}
}
impl PyVAlias {
pub fn all<T: From<PyVAlias>>() -> Vec<T> {
use PyVAlias::*;
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
}
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
fn major(&self) -> u32 {
use PyVAlias::*;
match self {
Py310 | Py311 | Py312 | Py313 => 3,
// Py400 | Py401 => 4
}
}
/// Converts numeric format to alias
/// Example:
/// 310u32 (in) -> PyVAlias::Py310 (out)
pub fn try_from_v1<T: ToString>(numeric: T) -> Option<Self> {
use PyVAlias::*;
match numeric.to_string().as_str() {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
_ => None,
}
}
}
/// Parse lockfile for assigned python version.
/// If not found returns None
pub fn try_parse_locked_python_version_from_requirements<S: AsRef<str>>(
requirements_lines: &[S],
) -> Option<pep440_rs::Version> {
let parse_version = |s: &str| -> Option<pep440_rs::Version> {
// Possible inputs:
// V2:
// # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0
//
// V1:
// # py311 or #py311
let version_unparsed = s
.to_owned()
// Remove whitespaces. That leaves us with:
// V2: #py:3.11.0
// V1: #py311
//
// Remove #
// V2: py:3.11.0
// V1: py311
//
// Remove :
// V2: py3.11.0
// V1: py311
.replace([' ', '#', ':'], "")
// Remove "py"
// V2: 3.11.0
// V1: 311
.replace("py", "");
// We will support reading V1 syntax, but it will be overwritten next deploy
PyVAlias::try_from_v1(&version_unparsed)
.map(PyVAlias::into)
.or(pep440_rs::Version::from_str(&version_unparsed)
.ok()
.map(pep440_rs::Version::into))
};
requirements_lines
.iter()
.find(|s| {
let s = s.as_ref();
s.starts_with("#py") || s.starts_with("# py")
})
.map(S::as_ref)
.and_then(parse_version)
}
#[cfg(test)]
mod tests {
use super::*;
backend/windmill-common/src/workspace_dependencies.rs
+1186
View File
File diff suppressed because it is too large Load Diff
backend/windmill-macros/Cargo.toml
+7
View File
@@ -11,6 +11,9 @@ proc-macro = true
proc-macro2.workspace = true
quote.workspace = true
syn.workspace = true
serde.workspace = true
serde_yml.workspace = true
serde_derive.workspace = true
# Dependencies for tests
[dev-dependencies]
@@ -18,3 +21,7 @@ syn.workspace = true
lazy_static.workspace = true
itertools.workspace = true
regex.workspace = true
serde.workspace = true
serde_yml.workspace = true
serde_derive.workspace = true
pep440_rs.workspace = true
backend/windmill-macros/tests/annotations.rs
+96
View File
@@ -3,6 +3,7 @@ mod annotations_tests {
extern crate windmill_macros;
use itertools::Itertools;
// use pep440_rs::Version;
use windmill_macros::annotations;
// Previous implementation.
@@ -166,4 +167,99 @@ mod annotations_tests {
assert_eq!(expected, Annotations::parse(cont));
}
}
// // #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// // #[annotations("#")]
// // pub struct SerAnnotations {
// // pub ann1: bool,
// // pub stt: String,
// // }
// #[test]
// fn non_bool_1() {
// let cont = r#"#ann1, stt: "hey""#;
// // non-bool take entire line, so you can't have one normal and than parsed.
// let a = SerAnnotations { ann1: false, stt: "".to_owned() };
// assert_eq!(a, SerAnnotations::parse(cont));
// }
// #[test]
// fn non_bool_2() {
// let cont = "#ann1, \n#stt: hey";
// let a = SerAnnotations { ann1: true, stt: "hey".to_owned() };
// assert_eq!(a, SerAnnotations::parse(cont));
// }
// #[test]
// fn non_bool_different_idents() {
// #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// #[annotations("#")]
// pub struct A {
// pub s: String,
// }
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s:hey"));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s : hey"));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s :hey"));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("#s : hey "));
// assert_eq!(A { s: "hey".to_owned() }, A::parse("# s : hey "));
// assert_eq!(A { s: "".to_owned() }, A::parse(" # s : hey "));
// }
// #[test]
// fn non_bool_unparseable_last() {
// #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// #[annotations("#")]
// pub struct A {
// pub v: i32,
// }
// let cont = "#v: 1\n#v: non int";
// let a = A {
// v: 1, // Should still be first, second unparsable one should have no affect on existing values
// };
// assert_eq!(a, A::parse(cont));
// }
// #[test]
// fn non_bool_different_types() {
// #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)]
// #[annotations("#")]
// pub struct A {
// pub s: String,
// pub i: i32,
// pub o: Option<String>,
// pub a: Vec<i32>,
// pub v: Option<pep440_rs::Version>, // Custom deser
// pub e: E,
// }
// #[derive(
// serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq, Clone, Default, Debug,
// )]
// pub enum E {
// #[default]
// One,
// Two(String),
// Three,
// }
// assert_eq!(
// A {
// s: "foo".to_owned(),
// i: 33,
// o: None,
// a: vec![1, 2, 3],
// v: Some(Version::new([1, 0, 0])),
// e: E::Three
// },
// A::parse(
// "#
// #s: foo
// #i: 33
// #o:
// #a: [1, 2, 3]
// #v: 1.0.0
// #e: Three
// "
// )
// );
// }
}
backend/windmill-queue/src/jobs.rs
+2 -9
View File
@@ -38,7 +38,6 @@ use windmill_common::add_time;
use windmill_common::auth::JobPerms;
#[cfg(feature = "benchmark")]
use windmill_common::bench::BenchmarkIter;
use windmill_common::lockfiles::is_generated_from_raw_requirements;
use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL};
use windmill_common::utils::{configure_client, now_from_db};
use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY};
@@ -2659,13 +2658,6 @@ impl PulledJobResult {
)
.await?;
if is_generated_from_raw_requirements(&Some(cloned_script.old_script.language), &cloned_script.old_script.lock.map(|v| v.to_string())) {
return Err(Error::BadRequest(format!(
"Script at path {} is generated from raw requirements, not overriding",
pulled_job.runnable_path()
)));
}
cloned_script.new_hash
}
JobKind::FlowDependencies => {
@@ -3632,7 +3624,7 @@ pub enum PushIsolationLevel<'c> {
}
impl<'c> PushIsolationLevel<'c> {
async fn into_tx(self) -> error::Result<Transaction<'c, Postgres>> {
pub async fn into_tx(self) -> error::Result<Transaction<'c, Postgres>> {
match self {
PushIsolationLevel::Isolated(db, authed) => Ok((db.begin(&authed).await?).into()),
PushIsolationLevel::IsolatedRoot(db) => Ok(db.begin().await?),
@@ -5867,3 +5859,4 @@ pub async fn get_same_worker_job(
))
})
}
backend/windmill-worker/src/ansible_executor.rs
+19 -13
View File
@@ -13,10 +13,10 @@ use tokio::process::Command;
use uuid::Uuid;
use windmill_common::{
error,
git_sync_oss::{prepend_token_to_github_url},
git_sync_oss::prepend_token_to_github_url,
worker::{
is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location,
Connection, WORKER_CONFIG,
is_allowed_file_location, split_python_requirements, to_raw_value, write_file,
write_file_at_user_defined_location, Connection, PyVAlias, WORKER_CONFIG,
},
};
use windmill_queue::MiniPulledJob;
@@ -34,8 +34,8 @@ use crate::{
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, PY_INSTALL_DIR, PyVAlias, TZ_ENV,
DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
PY_INSTALL_DIR, TZ_ENV,
};
use windmill_common::client::AuthedClient;
@@ -399,7 +399,7 @@ async fn handle_ansible_python_deps(
if requirements.len() > 0 {
let mut venv_path = handle_python_reqs(
crate::python_executor::split_requirements(requirements),
split_python_requirements(requirements),
job_id,
w_id,
mem_peak,
@@ -958,7 +958,11 @@ pub async fn handle_ansible_job(
#[cfg(feature = "enterprise")]
if is_github_app {
if let Connection::Sql(db) = conn {
let token = windmill_common::git_sync_oss::get_github_app_token_internal(db, &client.token).await?;
let token = windmill_common::git_sync_oss::get_github_app_token_internal(
db,
&client.token,
)
.await?;
secret_url = prepend_token_to_github_url(&secret_url, &token)?;
} else {
return Err(windmill_common::error::Error::BadRequest("Github App authentication is currently unavailable for agent workers. Contact the windmill team to request this feature".to_string()));
@@ -970,8 +974,12 @@ pub async fn handle_ansible_job(
let target_path = "delegate_git_repository".to_string();
let repo =
GitRepo { url: secret_url, commit: delegated_git_repo.commit.clone(), branch, target_path };
let repo = GitRepo {
url: secret_url,
commit: delegated_git_repo.commit.clone(),
branch,
target_path,
};
append_logs(
&job.id,
&job.workspace_id,
@@ -1241,10 +1249,8 @@ fi
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await?
} else {
let ansible_args: Vec<&str> = cmd_args.iter().map(|s| s.as_ref()).collect();
let mut ansible_cmd = build_command_with_isolation(
ANSIBLE_PLAYBOOK_PATH.as_str(),
&ansible_args,
);
let mut ansible_cmd =
build_command_with_isolation(ANSIBLE_PLAYBOOK_PATH.as_str(), &ansible_args);
ansible_cmd
.current_dir(job_dir)
.env_clear()
backend/windmill-worker/src/bun_executor.rs
+90 -84
View File
@@ -16,7 +16,7 @@ use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
parse_npm_config, read_file, read_file_content, read_result, start_child_process,
write_file_binary, OccupancyMetrics, StreamNotifier,
write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier,
},
handle_child::handle_child,
BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH,
@@ -27,6 +27,7 @@ use windmill_common::{
client::AuthedClient,
s3_helpers::BundleFormat,
scripts::{id_to_codebase_info, CodebaseInfo},
workspace_dependencies::WorkspaceDependenciesPrefetched,
};
#[cfg(windows)]
@@ -40,7 +41,7 @@ use windmill_common::{
error::{self, Result},
get_latest_hash_for_path,
scripts::ScriptLang,
worker::{exists_in_cache, save_cache, to_raw_value, write_file, Connection, DISABLE_BUNDLING},
worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING},
DB,
};
@@ -101,7 +102,7 @@ pub async fn gen_bun_lockfile(
base_internal_url: &str,
worker_name: &str,
export_pkg: bool,
raw_deps: Option<String>,
workspace_dependencies: &WorkspaceDependenciesPrefetched,
npm_mode: bool,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<Option<String>> {
@@ -109,11 +110,11 @@ pub async fn gen_bun_lockfile(
let mut empty_deps = false;
if let Some(raw_deps) = raw_deps.as_ref() {
if let Some(package_json_content) = workspace_dependencies.get_bun()? {
gen_bunfig(job_dir).await?;
write_file(job_dir, "package.json", raw_deps.as_str())?;
write_file(job_dir, "package.json", package_json_content.as_str())?;
} else {
let _ = write_file(
write_file(
&job_dir,
"build.js",
&format!(
@@ -205,16 +206,7 @@ pub async fn gen_bun_lockfile(
let mut file = File::open(format!("{job_dir}/package.json")).await?;
let mut buf = String::default();
file.read_to_string(&mut buf).await?;
if raw_deps.is_some() {
let mut json_map: HashMap<String, Box<RawValue>> = serde_json::from_str(&buf)?;
json_map.insert(
"generatedFromPackageJson".to_string(),
to_raw_value(&"true".to_string()),
);
content = serde_json::to_string_pretty(&json_map)?;
} else {
content = buf;
}
content = buf;
}
if !npm_mode {
#[cfg(any(target_os = "linux", target_os = "macos"))]
@@ -703,7 +695,7 @@ fn extract_saved_codebase(
pub async fn prebundle_bun_script(
inner_content: &str,
lockfile: Option<&String>,
lock: &str,
script_path: &str,
job_id: &Uuid,
w_id: &str,
@@ -715,7 +707,7 @@ pub async fn prebundle_bun_script(
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<()> {
let (local_path, remote_path) =
compute_bundle_local_and_remote_path(inner_content, lockfile, script_path, db, w_id).await;
compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await;
if exists_in_cache(&local_path, &remote_path).await {
return Ok(());
}
@@ -779,16 +771,12 @@ async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) ->
pub async fn compute_bundle_local_and_remote_path(
inner_content: &str,
requirements_o: Option<&String>,
lock: &str,
script_path: &str,
db: Option<&DB>,
w_id: &str,
) -> (String, String) {
let mut input_src = format!(
"{}{}",
inner_content,
requirements_o.as_ref().map(|x| x.as_str()).unwrap_or("")
);
let mut input_src = format!("{inner_content}{lock}",);
if let Some(db) = db {
let relative_imports = crate::worker_lockfiles::extract_relative_imports(
@@ -844,7 +832,7 @@ async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> R
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_bun_job(
requirements_o: Option<&String>,
maybe_lock: MaybeLock,
codebase: Option<&String>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -865,16 +853,15 @@ pub async fn handle_bun_job(
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some()
&& !annotation.nobundling
&& !*DISABLE_BUNDLING
&& codebase.is_none()
{
let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = (
maybe_lock.get_lock(),
!annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(),
) {
let (local_path, remote_path) = match conn {
Connection::Sql(db) => {
compute_bundle_local_and_remote_path(
inner_content,
requirements_o,
lock,
job.runnable_path(),
Some(db),
&job.workspace_id,
@@ -940,59 +927,70 @@ pub async fn handle_bun_job(
if pulled_codebase.is_esm {
format = BundleFormat::Esm;
}
} else if let Some(reqs) = requirements_o.as_ref() {
let (pkg, lock, empty, is_binary) = split_lockfile(reqs);
if lock.is_none() && !annotation.npm {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{reqs}|", if is_binary {"b"} else {""})
));
}
let _ = write_file(job_dir, "package.json", pkg)?;
let lock = if annotation.npm { "" } else { lock.unwrap() };
if !empty {
if !annotation.npm {
let _ = write_lock(lock, job_dir, is_binary).await?;
}
install_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
job_dir,
worker_name,
common_bun_proc_envs.clone(),
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
}
} else {
// if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
let _ = gen_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
&client.token,
job.runnable_path(),
job_dir,
base_internal_url,
worker_name,
false,
None,
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
match &maybe_lock {
MaybeLock::Resolved { lock } => {
let (package_json, bun_lock, empty, is_binary) = split_lockfile(lock);
// }
if bun_lock.is_none() && !annotation.npm {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{lock}|", if is_binary {"b"} else {""})
));
}
write_file(job_dir, "package.json", package_json)?;
let bun_lock = if annotation.npm {
""
} else {
bun_lock.unwrap()
};
if !empty {
if !annotation.npm {
write_lock(bun_lock, job_dir, is_binary).await?;
}
install_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
job_dir,
worker_name,
common_bun_proc_envs.clone(),
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
}
}
MaybeLock::Unresolved { ref workspace_dependencies } => {
// if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
gen_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(conn),
&client.token,
job.runnable_path(),
job_dir,
base_internal_url,
worker_name,
false,
workspace_dependencies,
annotation.npm,
&mut Some(occupancy_metrics),
)
.await?;
// }
}
}
}
if codebase.is_some() && format == BundleFormat::Cjs {
@@ -1184,7 +1182,7 @@ try {{
&& !annotation.nobundling
&& !*DISABLE_BUNDLING
&& !codebase.is_some()
&& (requirements_o.is_some() || annotation.native);
&& (maybe_lock.get_lock().is_some() || annotation.native);
let write_loader_f = async {
if build_cache {
@@ -1689,7 +1687,15 @@ pub async fn start_worker(
base_internal_url,
worker_name,
false,
None,
&WorkspaceDependenciesPrefetched::extract(
inner_content,
ScriptLang::Bun,
w_id,
&None,
&script_path,
db.into(),
)
.await?,
annotation.npm,
&mut None,
)
backend/windmill-worker/src/common.rs
+34 -4
View File
@@ -24,6 +24,7 @@ use windmill_common::worker::{
to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType,
CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
};
use windmill_common::workspace_dependencies::WorkspaceDependenciesPrefetched;
use windmill_common::{
cache::{Cache, RawData},
error::{self, Error},
@@ -629,10 +630,7 @@ lazy_static! {
static ref DISABLE_PROCESS_GROUP: bool = std::env::var("DISABLE_PROCESS_GROUP").is_ok();
}
pub fn build_command_with_isolation(
program: &str,
args: &[&str],
) -> Command {
pub fn build_command_with_isolation(program: &str, args: &[&str]) -> Command {
use tokio::process::Command;
if *crate::ENABLE_UNSHARE_PID {
@@ -1371,3 +1369,35 @@ pub fn s3_mode_args_to_worker_data(
workspace_id: job.workspace_id.clone(),
}
}
#[derive(Debug)]
pub enum MaybeLock {
/// Deployed Scripts
Resolved { lock: String },
/// Previews
Unresolved { workspace_dependencies: WorkspaceDependenciesPrefetched },
}
impl MaybeLock {
pub fn map_unresolved<B, F>(&self, mut f: F) -> Option<B>
where
Self: Sized,
F: FnMut(&WorkspaceDependenciesPrefetched) -> B,
{
self.get_workspace_dependencies().map(|wd| f(wd))
}
pub fn get_workspace_dependencies(&self) -> Option<&WorkspaceDependenciesPrefetched> {
match self {
MaybeLock::Resolved { .. } => None,
MaybeLock::Unresolved { ref workspace_dependencies } => Some(workspace_dependencies),
}
}
pub fn get_lock(&self) -> Option<&String> {
match self {
MaybeLock::Resolved { ref lock } => Some(lock),
MaybeLock::Unresolved { .. } => None,
}
}
}
backend/windmill-worker/src/go_executor.rs
+80 -92
View File
@@ -1,4 +1,4 @@
use crate::PROXY_ENVS;
use crate::{common::MaybeLock, PROXY_ENVS};
use std::{collections::HashMap, fs::DirBuilder, process::Stdio};
use itertools::Itertools;
@@ -19,8 +19,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, read_result,
start_child_process, OccupancyMetrics,
build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables,
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV,
@@ -91,12 +91,12 @@ pub async fn handle_go_job(
parent_runnable_path: Option<String>,
inner_content: &str,
job_dir: &str,
requirements_o: Option<&String>,
shared_mount: &str,
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
occupation_metrics: &mut OccupancyMetrics,
maybe_lock: MaybeLock,
) -> Result<Box<RawValue>, Error> {
//go does not like executing modules at temp root
let job_dir = &format!("{job_dir}/go");
@@ -105,14 +105,7 @@ pub async fn handle_go_job(
.create(&job_dir)
.expect("could not create go job dir");
let hash = calculate_hash(&format!(
"{}{}v2",
inner_content,
requirements_o
.as_ref()
.map(|x| x.to_string())
.unwrap_or_default()
));
let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock));
let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR);
let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}");
let (cache, cache_logs) =
@@ -120,8 +113,8 @@ pub async fn handle_go_job(
let (skip_go_mod, skip_tidy) = if cache {
(true, true)
} else if let Some(requirements) = requirements_o {
gen_go_mod(inner_content, job_dir, &requirements).await?
} else if let Some(lock) = maybe_lock.get_lock() {
gen_go_mod(inner_content, job_dir, &lock).await?
} else {
(false, false)
};
@@ -133,6 +126,7 @@ pub async fn handle_go_job(
install_go_dependencies(
&job.id,
inner_content,
maybe_lock,
mem_peak,
canceled_by,
job_dir,
@@ -140,7 +134,6 @@ pub async fn handle_go_job(
true,
skip_go_mod,
skip_tidy,
false,
worker_name,
&job.workspace_id,
occupation_metrics,
@@ -430,20 +423,16 @@ func Run(req Req) (interface{{}}, error){{
read_result(job_dir, handle_result.result_stream).await
}
async fn gen_go_mod(
inner_content: &str,
job_dir: &str,
requirements: &str,
) -> error::Result<(bool, bool)> {
async fn gen_go_mod(inner_content: &str, job_dir: &str, lock: &str) -> error::Result<(bool, bool)> {
gen_go_mymod(inner_content, job_dir).await?;
let md = requirements.split_once(GO_REQ_SPLITTER);
let md = lock.split_once(GO_REQ_SPLITTER);
if let Some((req, sum)) = md {
write_file(job_dir, "go.mod", &req)?;
write_file(job_dir, "go.sum", &sum)?;
Ok((true, true))
} else {
write_file(job_dir, "go.mod", &requirements)?;
write_file(job_dir, "go.mod", &lock)?;
Ok((true, false))
}
}
@@ -454,92 +443,99 @@ use std::io::prelude::*;
pub async fn install_go_dependencies(
job_id: &Uuid,
code: &str,
maybe_lock: MaybeLock,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
conn: &Connection,
non_dep_job: bool,
// NOTE: this is impossible for skip_go_mod be `false` and maybe_lock be `Resolved`.
// TODO: make it comptime gurantee
skip_go_mod: bool,
has_sum: bool,
raw_deps: bool,
worker_name: &str,
w_id: &str,
occupation_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
let anns = GoAnnotations::parse(code);
if raw_deps {
let go_mod =
if let Some(module) = code.lines().find(|l| l.trim_start().starts_with("module ")) {
code.replace(module, "module mymod")
let hash_input = match maybe_lock {
MaybeLock::Resolved { ref lock } => lock.clone(),
MaybeLock::Unresolved { ref workspace_dependencies } => {
// NOTE: This will always be none, go workspace dependencies are disabled for now.
// read more on discord (internal):
// https://discord.com/channels/930051556043276338/1031563866641018910/1443541229349634189
if let Some(go_mod) = workspace_dependencies.get_go()? {
if !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
fs::write(format!("{job_dir}/go.mod"), &go_mod).await?;
}
go_mod
} else {
format!("module mymod\n{code}")
};
fs::write(format!("{job_dir}/go.mod"), go_mod).await?;
}
if !raw_deps && !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
let mut child_cmd = Command::new(GO_PATH.as_str());
child_cmd
.current_dir(job_dir)
.env_clear()
.args(vec!["mod", "init", "mymod"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
let mut child_cmd = Command::new(GO_PATH.as_str());
child_cmd
.current_dir(job_dir)
.env_clear()
.args(vec!["mod", "init", "mymod"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
child_cmd.env("GOPATH", windows_gopath());
#[cfg(unix)]
child_cmd.env("GOPATH", GO_CACHE_DIR);
#[cfg(windows)]
child_cmd.env("GOPATH", windows_gopath());
#[cfg(unix)]
child_cmd.env("GOPATH", GO_CACHE_DIR);
#[cfg(windows)]
set_windows_env_vars(&mut child_cmd);
let child_process = start_child_process(child_cmd, GO_PATH.as_str(), false).await?;
#[cfg(windows)]
set_windows_env_vars(&mut child_cmd);
let child_process =
start_child_process(child_cmd, GO_PATH.as_str(), false).await?;
handle_child(
job_id,
conn,
mem_peak,
canceled_by,
child_process,
false,
worker_name,
w_id,
"go init",
None,
false,
&mut Some(occupation_metrics),
None,
None,
)
.await?;
handle_child(
job_id,
conn,
mem_peak,
canceled_by,
child_process,
false,
worker_name,
w_id,
"go init",
None,
false,
&mut Some(occupation_metrics),
None,
None,
)
.await?;
for x in REQUIRE_PARSE.captures_iter(code) {
let mut file = OpenOptions::new()
.write(true)
.append(true)
.open(format!("{job_dir}/go.mod"))
.unwrap();
for x in REQUIRE_PARSE.captures_iter(code) {
let mut file = OpenOptions::new()
.write(true)
.append(true)
.open(format!("{job_dir}/go.mod"))
.unwrap();
writeln!(file, "require {}\n", &x[1])?;
writeln!(file, "require {}\n", &x[1])?;
}
}
if !has_sum {
calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
} else {
"".to_owned()
}
}
}
}
let mut new_lockfile = false;
let hash = if raw_deps {
calculate_hash(code)
} else if !has_sum {
calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
} else {
"".to_string()
};
let hash = format!(
"go{}-{}",
if anns.go1_22_compat { "1.22" } else { "" },
hash
calculate_hash(&hash_input)
);
let mut skip_tidy = has_sum;
let (mut new_lockfile, mut skip_tidy) = (false, has_sum);
if !has_sum {
if let Some(db) = conn.as_sql() {
@@ -561,15 +557,7 @@ pub async fn install_go_dependencies(
}
}
let mod_command = if skip_tidy ||
// If there is go.mod provided we want to use `download` only.
// Unlike `tidy` it does not modify local go.mod
raw_deps
{
"download"
} else {
"tidy"
};
let mod_command = if skip_tidy { "download" } else { "tidy" };
let mut child_cmd = Command::new(GO_PATH.as_str());
child_cmd
.current_dir(job_dir)
backend/windmill-worker/src/lib.rs
+2 -1
View File
@@ -70,6 +70,7 @@ mod worker;
mod worker_flow;
mod worker_lockfiles;
mod worker_utils;
pub mod workspace_dependencies;
pub use worker::*;
pub use worker_lockfiles::{
@@ -85,4 +86,4 @@ pub use bun_executor::{
pub use deno_executor::generate_deno_lock;
#[cfg(feature = "python")]
pub use python_versions::{PyV, PyVAlias};
pub use python_versions::PyV;
backend/windmill-worker/src/php_executor.rs
+27 -14
View File
@@ -7,7 +7,9 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command};
use uuid::Uuid;
use windmill_common::{
error::{self, to_anyhow, Result},
scripts::ScriptLang,
worker::{write_file, Connection},
workspace_dependencies::clean_lock_from_annotations,
};
use windmill_queue::MiniPulledJob;
@@ -17,11 +19,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, start_child_process, OccupancyMetrics,
get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics,
},
handle_child::handle_child,
COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
NSJAIL_PATH, PHP_PATH,
COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
};
use windmill_common::client::AuthedClient;
@@ -135,9 +136,23 @@ $args->{arg_name} = new {rt_name}($args->{arg_name});"
)
}
fn split_reqs_and_lock(content: &String) -> error::Result<(Option<String>, Option<String>)> {
let splitted = content.split(COMPOSER_LOCK_SPLIT).collect_vec();
if splitted.len() != 2 {
return Err(error::Error::ExecutionErr(format!(
"Invalid requirements, expected to find LOCK split pattern in reqs. Found: |{content}|"
)));
}
Ok((
Some(clean_lock_from_annotations(splitted[0], ScriptLang::Php)),
Some(splitted[1].to_string()),
))
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_php_job(
requirements_o: Option<&String>,
maybe_lock: MaybeLock,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
@@ -154,16 +169,14 @@ pub async fn handle_php_job(
) -> error::Result<Box<RawValue>> {
check_executor_binary_exists("php", PHP_PATH.as_str(), "php")?;
let (composer_json, composer_lock) = match requirements_o {
Some(reqs_and_lock) if !reqs_and_lock.is_empty() => {
let splitted = reqs_and_lock.split(COMPOSER_LOCK_SPLIT).collect_vec();
if splitted.len() != 2 {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find LOCK split pattern in reqs. Found: |{reqs_and_lock}|")
));
}
(Some(splitted[0].to_string()), Some(splitted[1].to_string()))
}
let (composer_json, composer_lock) = match &maybe_lock {
MaybeLock::Resolved { lock } if !lock.is_empty() => split_reqs_and_lock(lock)?,
MaybeLock::Unresolved { workspace_dependencies } => (
workspace_dependencies
.get_php()?
.or(parse_php_imports(inner_content)?),
None,
),
_ => (parse_php_imports(inner_content)?, None),
};
backend/windmill-worker/src/python_executor.rs
+30 -34
View File
@@ -30,7 +30,8 @@ use windmill_common::{
},
utils::calculate_hash,
worker::{
copy_dir_recursively, pad_string, write_file, Connection, PythonAnnotations, WORKER_CONFIG,
copy_dir_recursively, pad_string, split_python_requirements, write_file, Connection,
PyVAlias, PythonAnnotations, WORKER_CONFIG,
},
};
@@ -122,13 +123,13 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, read_result,
start_child_process, OccupancyMetrics, StreamNotifier,
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file,
read_result, start_child_process, OccupancyMetrics, StreamNotifier,
},
handle_child::handle_child,
worker_utils::ping_job_status,
PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
PyV, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
};
use windmill_common::client::AuthedClient;
@@ -566,7 +567,7 @@ pub async fn handle_python_job(
canceled_by,
&mut Some(occupancy_metrics),
precomputed_agent_info,
annotations,
annotations.clone(),
)
.await?;
@@ -838,10 +839,7 @@ mount {{
} else {
let args = vec!["-u", "-m", "wrapper"];
let mut python_cmd = build_command_with_isolation(
&python_path,
&args,
);
let mut python_cmd = build_command_with_isolation(&python_path, &args);
python_cmd
.current_dir(job_dir)
.env_clear()
@@ -1177,36 +1175,43 @@ async fn handle_python_deps(
let (pyv, resolved_lines) = match requirements_o {
// Deployed
Some(r) => {
let rl = split_requirements(r);
let rl = split_python_requirements(r);
(PyV::parse_from_requirements(&rl), rl)
}
// Preview
None => {
let (v, requirements_lines, error_hint) = match conn {
Connection::Sql(db) => {
let mut version_specifiers = vec![];
let (mut version_specifiers, mut locked_v) = (vec![], None);
let (r, h) = Box::pin(windmill_parser_py_imports::parse_python_imports(
inner_content,
w_id,
script_path,
db,
&mut version_specifiers,
&mut locked_v,
&None,
))
.await?;
let v = PyV::resolve(
version_specifiers,
job_id,
w_id,
annotations.py_select_latest,
Some(conn.clone()),
None,
None,
)
.await?;
let v = if let Some(v) = locked_v {
v.into()
} else {
PyV::resolve(
version_specifiers,
job_id,
w_id,
annotations.py_select_latest,
Some(conn.clone()),
None,
None,
)
.await?
};
(v, r, h)
}
Connection::Http(_) => match precomputed_agent_info {
Some(PrecomputedAgentInfo::Python {
requirements,
@@ -1237,7 +1242,7 @@ Returned from server: py_version - {:?}, py_version_v2 - {:?}
}
};
let r = split_requirements(requirements.unwrap_or_default());
let r = split_python_requirements(requirements.unwrap_or_default());
let h = None;
(v, r, h)
@@ -2113,15 +2118,6 @@ pub async fn handle_python_reqs(
};
}
pub fn split_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
requirements
.as_ref()
.lines()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.map(String::from)
.collect()
}
// Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed
fn get_result_postprocessor<'a>(skip: bool) -> &'a str {
if skip {
@@ -2157,7 +2153,7 @@ pub async fn start_worker(
killpill_rx: tokio::sync::broadcast::Receiver<()>,
client: windmill_common::client::AuthedClient,
) -> error::Result<()> {
use crate::{PyV, PyVAlias};
use crate::PyV;
tracing::info!("script path: {}", script_path);
let mut mem_peak: i32 = 0;
@@ -2317,7 +2313,7 @@ for line in sys.stdin:
proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string());
let py_version = if let Some(requirements) = requirements_o {
PyV::parse_from_requirements(&split_requirements(requirements.as_str()))
PyV::parse_from_requirements(&split_python_requirements(requirements.as_str()))
} else {
tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11");
PyVAlias::Py311.into()
backend/windmill-worker/src/python_versions.rs
+2 -97
View File
@@ -12,8 +12,7 @@ use tokio::{fs::DirBuilder, process::Command, sync::RwLock};
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT,
worker::Connection,
worker::{try_parse_locked_python_version_from_requirements, Connection, PyVAlias},
};
use anyhow::{anyhow, bail};
@@ -26,28 +25,6 @@ use crate::{
HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS,
};
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
#[repr(u32)]
pub enum PyVAlias {
Py310 = 10,
#[default]
Py311,
Py312,
Py313,
}
impl Into<pep440_rs::Version> for PyVAlias {
fn into(self) -> pep440_rs::Version {
pep440_rs::Version::new([self.major() as u64, self as u64])
}
}
impl Into<u32> for PyVAlias {
fn into(self) -> u32 {
self.major() * 100 + self as u32
}
}
impl From<PyV> for PyVAlias {
fn from(value: PyV) -> Self {
match value.release() {
@@ -66,34 +43,6 @@ impl From<PyV> for PyVAlias {
Self::default()
}
}
impl PyVAlias {
fn all<T: From<PyVAlias>>() -> Vec<T> {
use PyVAlias::*;
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
}
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
fn major(&self) -> u32 {
use PyVAlias::*;
match self {
Py310 | Py311 | Py312 | Py313 => 3,
// Py400 | Py401 => 4
}
}
/// Converts numeric format to alias
/// Example:
/// 310u32 (in) -> PyVAlias::Py310 (out)
pub(crate) fn try_from_v1<T: ToString>(numeric: T) -> Option<Self> {
use PyVAlias::*;
match numeric.to_string().as_str() {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
_ => None,
}
}
}
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
@@ -431,51 +380,7 @@ impl PyV {
/// Parse lockfile for assigned python version.
/// If not found returns None
pub fn try_parse_from_requirements<S: AsRef<str>>(requirements_lines: &[S]) -> Option<Self> {
let parse_version = |s: &str| -> Option<PyV> {
// Possible inputs:
// V2:
// # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0
//
// V1:
// # py311 or #py311
let version_unparsed = s
.to_owned()
// Remove whitespaces. That leaves us with:
// V2: #py:3.11.0
// V1: #py311
//
// Remove #
// V2: py:3.11.0
// V1: py311
//
// Remove :
// V2: py3.11.0
// V1: py311
.replace([' ', '#', ':'], "")
// Remove "py"
// V2: 3.11.0
// V1: 311
.replace("py", "");
// We will support reading V1 syntax, but it will be overwritten next deploy
PyVAlias::try_from_v1(&version_unparsed)
.map(PyVAlias::into)
.or(pep440_rs::Version::from_str(&version_unparsed)
.ok()
.map(pep440_rs::Version::into))
};
let index = if requirements_lines.get(0).map_or(false, |line| {
line.as_ref()
.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT)
}) {
1
} else {
0
};
requirements_lines
.get(index)
.map(S::as_ref)
.and_then(parse_version)
try_parse_locked_python_version_from_requirements(requirements_lines).map(PyV::from)
}
pub async fn get_python(
backend/windmill-worker/src/scoped_dependency_map.rs
+93 -62
View File
@@ -1,22 +1,24 @@
use serde::Serialize;
use sqlx::PgExecutor;
use tokio::sync::RwLock;
use windmill_common::{
apps::traverse_app_inline_scripts,
cache,
error::{Error, Result},
flows::{FlowModuleValue, FlowValue},
scripts::ScriptLang,
};
use std::collections::HashSet;
use crate::worker_lockfiles::extract_relative_imports;
use windmill_common::lockfiles::is_generated_from_raw_requirements;
use crate::worker_lockfiles::extract_referenced_paths;
// TODO: To be removed in future versions
lazy_static::lazy_static! {
pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok();
}
// TODO: Rename to DependencyRelation
#[derive(Serialize)]
pub struct DependencyMap {
pub workspace_id: Option<String>,
@@ -26,9 +28,17 @@ pub struct DependencyMap {
pub importer_node_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct DependencyDependent {
pub importer_path: String,
pub importer_kind: String,
pub importer_node_ids: Option<Vec<String>>,
}
#[derive(Debug)]
pub struct ScopedDependencyMap {
dmap: HashSet<(String, String)>,
/// (importer_node_id, imported_path)
to_delete: HashSet<(String, String)>,
w_id: String,
importer_path: String,
importer_kind: String,
@@ -72,7 +82,7 @@ RETURNING importer_node_id, imported_path
.fetch_all(executor)
.await?;
Ok(Self {
dmap: HashSet::from_iter(dmap.into_iter()),
to_delete: HashSet::from_iter(dmap.into_iter()),
w_id: w_id.to_owned(),
importer_path: importer_path.to_owned(),
importer_kind: importer_kind.to_owned(),
@@ -104,7 +114,7 @@ SELECT importer_node_id, imported_path
.await?;
Ok(Self {
dmap: HashSet::from_iter(dmap.into_iter()),
to_delete: HashSet::from_iter(dmap.into_iter()),
w_id: w_id.to_owned(),
importer_path: importer_path.to_owned(),
importer_kind: importer_kind.to_owned(),
@@ -115,22 +125,23 @@ SELECT importer_node_id, imported_path
/// Remove matching entries
pub(crate) async fn patch<'c>(
&mut self,
relative_imports: Option<Vec<String>>,
referenced_paths: Option<Vec<String>>,
node_id: String, // Flow Step/Node ID
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
self.patch_tx_ref(relative_imports, &node_id, &mut tx)
self.patch_tx_ref(referenced_paths, &node_id, &mut tx)
.await?;
Ok(tx)
}
pub(crate) async fn patch_tx_ref<'c>(
&mut self,
relative_imports: Option<Vec<String>>,
// NOTE: Referenced_paths should include all of the paths.
referenced_paths: Option<Vec<String>>,
node_id: &str, // Flow Step/Node ID
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
) -> Result<()> {
let Some(mut relative_imports) = relative_imports else {
let Some(mut referenced_paths) = referenced_paths else {
tracing::info!("relative imports are not found for: importer - {}, importer_node_id - {}, importer_kind - {}",
&self.importer_path,
&node_id,
@@ -149,9 +160,9 @@ SELECT importer_node_id, imported_path
// After all `reduce`'s called ScopedDependencyMap has only extra/orphan imports
// these are going to be clean up by calling [dissolve]
// NOTE: `retain` iterates over vec and remove the ones whose closures returned false.
relative_imports.retain(|imported_path| {
referenced_paths.retain(|imported_path| {
!self
.dmap
.to_delete
// As dmap is HashSet, removing is O(1) operation
// thus making entire process very efficient
// NOTE: `remove` returns true if item was removed and false if wasn't.
@@ -159,15 +170,15 @@ SELECT importer_node_id, imported_path
});
// As mentioned above, usually this will always be empty.
if !relative_imports.is_empty() {
if !referenced_paths.is_empty() {
tracing::info!("adding missing entries to dependency_map: importer_node_id - {}, importer_kind - {}, new_imported_paths - {:?}",
&node_id,
&self.importer_kind,
&relative_imports,
&referenced_paths,
);
}
for import in relative_imports {
for import in referenced_paths {
sqlx::query!(
"INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)
VALUES ($1, $2, $3::text::IMPORTER_KIND, $4, $5) ON CONFLICT DO NOTHING",
@@ -200,7 +211,7 @@ SELECT importer_node_id, imported_path
tracing::info!("dissolving dependency_map: {:?}", &self);
// We _could_ shove it into single query, but this query is rarely called AND let's keep it simple for redability.
for (importer_node_id, imported_path) in self.dmap.into_iter() {
for (importer_node_id, imported_path) in self.to_delete.into_iter() {
tracing::info!("cleaning orphan entry from dependency_map: importer_kind - {}, imported_path - {}, importer_node_id - {}",
&self.importer_kind,
&imported_path,
@@ -275,14 +286,14 @@ SELECT importer_node_id, imported_path
tx
}
/// Run if you want to rebuild maps on specific workspace.
/// Potentially takes much time
pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool<sqlx::Postgres>) -> Result<String> {
async fn inner<'c>(w_id: &str, db: &sqlx::Pool<sqlx::Postgres>) -> Result<String> {
// Scripts
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts");
for r in sqlx::query!(
"SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false",
pub(crate) async fn rebuild_map_unchecked<'c>(
w_id: &str,
db: &sqlx::Pool<sqlx::Postgres>,
) -> Result<String> {
// Scripts
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts");
for r in sqlx::query!(
r#"SELECT path, hash, language AS "language: ScriptLang" FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false"#,
w_id
)
.fetch_all(db)
@@ -292,29 +303,24 @@ SELECT importer_node_id, imported_path
let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "script", db).await?;
let mut tx = db.begin().await?;
if is_generated_from_raw_requirements(&smd.language, &sd.lock) {
// if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map
// because we do not want to have dependencies be recomputed automatically. Empty relative imports passed
// to update_script_dependency_map will clear the dependency map.
} else {
tx = dmap
.patch(
extract_relative_imports(&sd.code, &r.path, &smd.language),
"".into(),
tx,
)
.await?;
}
tx = dmap
.patch(
extract_referenced_paths(&sd.code, &r.path, smd.language),
"".into(),
tx,
)
.await?;
if !*WMDEBUG_NO_DMAP_DISSOLVE {
dmap.dissolve(tx).await.commit().await?;
}
tracing::info!(workspace_id = w_id, "Rebuilt for script {}", &r.path);
}
// Fetch only top level versions and paths
// It is not fetching value
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? {
// Fetch only top level versions and paths
// It is not fetching value
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? {
if let Some(version) = r.version {
// To reduce stress on db try to fetch from cache
// Since our flow versions are immutable it is safe to assume if we have cache for specific version/id it is up to date.
@@ -337,17 +343,15 @@ SELECT importer_node_id, imported_path
FlowValue::traverse_leafs(modules_to_check, &mut |fmv, id| {
match fmv {
// Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript.
FlowModuleValue::RawScript { content, language, lock ,.. } => {
if !is_generated_from_raw_requirements(&Some(*language), lock) {
to_process.push((
extract_relative_imports(
content,
&(r.path.clone() + "/flow"),
&Some(language.clone()),
),
id.clone(),
));
}
FlowModuleValue::RawScript { content, language, .. } => {
to_process.push((
extract_referenced_paths(
content,
&(r.path.clone() + "/flow"),
Some(*language),
),
id.clone(),
));
}
// But just in case we will also handle other cases.
FlowModuleValue::FlowScript { .. } => {
@@ -359,8 +363,8 @@ SELECT importer_node_id, imported_path
Ok(())
})?;
for (ri, id) in to_process {
tx = dmap.patch(ri, id, tx).await?;
for (rp, id) in to_process {
tx = dmap.patch(rp, id, tx).await?;
}
if !*WMDEBUG_NO_DMAP_DISSOLVE {
@@ -374,9 +378,9 @@ SELECT importer_node_id, imported_path
}
}
// Apps
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? {
// Apps
tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps");
for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? {
if let Some(version) = r.version {
// TODO: Use cache when implemented.
let value = sqlx::query_scalar!(
@@ -391,10 +395,10 @@ SELECT importer_node_id, imported_path
let mut to_process = vec![];
traverse_app_inline_scripts(&value, None, &mut |ais, id| {
to_process.push((
extract_relative_imports(
extract_referenced_paths(
&ais.content,
&(r.path.clone() + "/app"),
&ais.language,
ais.language,
),
id,
));
@@ -417,9 +421,11 @@ SELECT importer_node_id, imported_path
}
}
Ok("Success".into())
}
Ok("Success".into())
}
/// Run if you want to rebuild maps on specific workspace.
/// Potentially takes much time
pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool<sqlx::Postgres>) -> Result<String> {
lazy_static::lazy_static! {
pub static ref LOCKED: RwLock<bool> = RwLock::new(false);
}
@@ -438,9 +444,34 @@ SELECT importer_node_id, imported_path
}
*LOCKED.write().await = true;
let r = inner(w_id, db).await;
let r = Self::rebuild_map_unchecked(w_id, db).await;
*LOCKED.write().await = false;
r
}
}
/// Get dependents of any imported path - returns scripts/flows/apps that depend on it
pub async fn get_dependents<'c>(
imported_path: &str,
workspace_id: &str,
e: impl PgExecutor<'c>,
) -> Result<Vec<DependencyDependent>> {
sqlx::query_as!(
DependencyDependent,
r#"
SELECT
importer_path,
importer_kind::text as "importer_kind!", -- sqlx thinks this is nullable somehow, so enfore with !
array_agg(importer_node_id) as importer_node_ids
FROM dependency_map
WHERE workspace_id = $1 AND imported_path = $2
GROUP BY importer_path, importer_kind
"#,
workspace_id,
imported_path
)
.fetch_all(e)
.await
.map_err(Error::from)
}
}
backend/windmill-worker/src/worker.rs
+58 -25
View File
@@ -20,6 +20,8 @@ use windmill_common::scripts::is_special_codebase_hash;
use windmill_common::utils::report_critical_error;
use windmill_common::utils::retrieve_common_worker_prefix;
use windmill_common::worker::error_to_value;
use windmill_common::workspace_dependencies::RawWorkspaceDependencies;
use windmill_common::workspace_dependencies::WorkspaceDependenciesPrefetched;
use windmill_common::{
agent_workers::DECODED_AGENT_TOKEN,
apps::AppScriptId,
@@ -109,6 +111,7 @@ use tokio::{
use rand::Rng;
use crate::ai_executor::handle_ai_agent_job;
use crate::common::MaybeLock;
use crate::common::StreamNotifier;
use crate::{
agent_workers::{queue_init_job, queue_periodic_job},
@@ -153,10 +156,9 @@ use crate::ruby_executor::{handle_ruby_job, JobHandlerInput as JobHandlerInputRu
use crate::php_executor::handle_php_job;
#[cfg(feature = "python")]
use crate::{
python_executor::handle_python_job,
python_versions::{PyV, PyVAlias},
};
use crate::{python_executor::handle_python_job, python_versions::PyV};
#[cfg(feature = "python")]
use windmill_common::worker::PyVAlias;
#[cfg(feature = "python")]
use crate::ansible_executor::handle_ansible_job;
@@ -2857,6 +2859,16 @@ pub async fn handle_queued_job(
let mut column_order: Option<Vec<String>> = None;
let mut new_args: Option<HashMap<String, Box<RawValue>>> = None;
let mut has_stream = false;
let raw_workspace_dependencies_o = if job.kind.is_dependency() {
job.args
.as_ref()
.and_then(|x| x.get("raw_workspace_dependencies"))
.map(|v| v.get())
.and_then(|v| serde_json::from_str::<RawWorkspaceDependencies>(v).ok())
} else {
None
};
// Box::pin all async branches to prevent large match enum on stack
let result = match job.kind {
JobKind::Dependencies => match conn {
@@ -2873,6 +2885,7 @@ pub async fn handle_queued_job(
base_internal_url,
&client.token,
occupancy_metrics,
raw_workspace_dependencies_o,
))
.await
}
@@ -2896,6 +2909,7 @@ pub async fn handle_queued_job(
base_internal_url,
&client.token,
occupancy_metrics,
raw_workspace_dependencies_o,
))
.await
}
@@ -2917,6 +2931,7 @@ pub async fn handle_queued_job(
base_internal_url,
&client.token,
occupancy_metrics,
raw_workspace_dependencies_o,
))
.await
.map(|()| serde_json::from_str("{}").unwrap()),
@@ -3614,14 +3629,32 @@ mount {{
let envs = build_envs(envs.as_ref())?;
let Some(language) = language else {
return Err(Error::ExecutionErr(
"Require language to be not null".to_string(),
))?;
};
let maybe_lock = if let Some(lock) = lock.clone() {
MaybeLock::Resolved { lock }
} else {
MaybeLock::Unresolved {
workspace_dependencies: WorkspaceDependenciesPrefetched::extract(
code,
language,
&job.workspace_id,
// TODO: implement
&None,
job.runnable_path(),
conn.clone(),
)
.await?,
}
};
// Box::pin all language handlers to prevent large match enum on stack
let result: error::Result<Box<RawValue>> = match language {
None => {
return Err(Error::ExecutionErr(
"Require language to be not null".to_string(),
))?;
}
Some(ScriptLang::Python3) => {
ScriptLang::Python3 => {
#[cfg(not(feature = "python"))]
return Err(Error::internal_err(
"Python requires the python feature to be enabled".to_string(),
@@ -3650,7 +3683,7 @@ mount {{
))
.await
}
Some(ScriptLang::Deno) => {
ScriptLang::Deno => {
Box::pin(handle_deno_job(
lock.as_ref(),
mem_peak,
@@ -3670,9 +3703,9 @@ mount {{
))
.await
}
Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) => {
ScriptLang::Bun | ScriptLang::Bunnative => {
Box::pin(handle_bun_job(
lock.as_ref(),
maybe_lock,
codebase.as_ref(),
mem_peak,
canceled_by,
@@ -3693,7 +3726,7 @@ mount {{
))
.await
}
Some(ScriptLang::Go) => {
ScriptLang::Go => {
Box::pin(handle_go_job(
mem_peak,
canceled_by,
@@ -3703,16 +3736,16 @@ mount {{
parent_runnable_path,
&code,
job_dir,
lock.as_ref(),
&shared_mount,
base_internal_url,
worker_name,
envs,
occupancy_metrics,
maybe_lock,
))
.await
}
Some(ScriptLang::Bash) => {
ScriptLang::Bash => {
Box::pin(handle_bash_job(
mem_peak,
canceled_by,
@@ -3731,7 +3764,7 @@ mount {{
))
.await
}
Some(ScriptLang::Powershell) => {
ScriptLang::Powershell => {
Box::pin(handle_powershell_job(
mem_peak,
canceled_by,
@@ -3749,7 +3782,7 @@ mount {{
))
.await
}
Some(ScriptLang::Php) => {
ScriptLang::Php => {
#[cfg(not(feature = "php"))]
return Err(Error::internal_err(
"PHP requires the php feature to be enabled".to_string(),
@@ -3757,7 +3790,7 @@ mount {{
#[cfg(feature = "php")]
Box::pin(handle_php_job(
lock.as_ref(),
maybe_lock,
mem_peak,
canceled_by,
job,
@@ -3774,7 +3807,7 @@ mount {{
))
.await
}
Some(ScriptLang::Rust) => {
ScriptLang::Rust => {
#[cfg(not(feature = "rust"))]
return Err(Error::internal_err(
"Rust requires the rust feature to be enabled".to_string(),
@@ -3799,7 +3832,7 @@ mount {{
))
.await
}
Some(ScriptLang::Ansible) => {
ScriptLang::Ansible => {
#[cfg(not(feature = "python"))]
return Err(Error::internal_err(
"Ansible requires the python feature to be enabled".to_string(),
@@ -3825,7 +3858,7 @@ mount {{
))
.await
}
Some(ScriptLang::CSharp) => {
ScriptLang::CSharp => {
Box::pin(handle_csharp_job(
mem_peak,
canceled_by,
@@ -3844,7 +3877,7 @@ mount {{
))
.await
}
Some(ScriptLang::Nu) => {
ScriptLang::Nu => {
#[cfg(not(feature = "nu"))]
return Err(
anyhow::anyhow!("Nu is not available because the feature is not enabled").into(),
@@ -3869,7 +3902,7 @@ mount {{
}))
.await
}
Some(ScriptLang::Java) => {
ScriptLang::Java => {
#[cfg(not(feature = "java"))]
return Err(anyhow::anyhow!(
"Java is not available because the feature is not enabled"
@@ -3895,7 +3928,7 @@ mount {{
}))
.await
}
Some(ScriptLang::Ruby) => {
ScriptLang::Ruby => {
#[cfg(not(feature = "ruby"))]
return Err(anyhow::anyhow!(
"Ruby is not available because the feature is not enabled"
backend/windmill-worker/src/worker_lockfiles.rs
+287 -323
View File
File diff suppressed because it is too large Load Diff
backend/windmill-worker/src/workspace_dependencies.rs
+268
View File
@@ -0,0 +1,268 @@
use serde::{Deserialize, Serialize};
use windmill_common::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies};
use crate::{
scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies,
};
#[derive(sqlx::FromRow, Clone, Serialize, Deserialize, Hash, Debug)]
pub struct NewWorkspaceDependencies {
pub workspace_id: String,
pub language: ScriptLang,
pub name: Option<String>,
/// If None, will use description of previous version
/// If there is no older versions, will set to default
pub description: Option<String>,
// TODO: Make Option, or optimize it in any other way.
pub content: String,
}
impl NewWorkspaceDependencies {
/// Creates a new workspace dependencies entry in the database.
///
/// Archives any existing dependencies with the same name/language/workspace,
/// then inserts the new dependencies. Triggers recomputation of dependent scripts
/// and rebuilds the dependency map if this is the first unnamed dependency for the workspace.
pub async fn create<'c>(
self,
email: &str,
created_by: &str,
permissioned_as: &str,
db: &sqlx::Pool<sqlx::Postgres>,
) -> error::Result<i64> {
// Check if all workers support workspace dependencies feature
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
.await?;
let path = WorkspaceDependencies::to_path(&self.name, self.language)?;
// If it is unnamed then we want to rebuild dependency map. Otherwise trigger dependents to recompute locks will not work
// NOTE: We rebuild first, even before creating new w deps. We want to make sure that if rebuild failed, then no new default workspace dependencies were created.
if self.name.is_none() {
// Check if we already rebuilt the map for this workspace by checking if the setting exists
let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id);
let already_rebuilt =
windmill_common::global_settings::load_value_from_global_settings(
db,
&setting_name,
)
.await?
.is_some();
if !already_rebuilt {
tracing::info!(
workspace_id = %self.workspace_id,
"Rebuilding workspace dependencies map for first unnamed workspace dependencies"
);
ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, db).await?;
// Mark as rebuilt by creating the setting
windmill_common::global_settings::set_value_in_global_settings(
db,
&setting_name,
serde_json::json!({}),
)
.await?;
tracing::info!(
workspace_id = %self.workspace_id,
"Marked workspace dependencies map as rebuilt"
);
} else {
tracing::info!(
workspace_id = %self.workspace_id,
"Skipping workspace dependencies map rebuild - already rebuilt for this workspace"
);
}
};
let mut tx = db.begin().await?;
let prev_description = sqlx::query_scalar!(
"
UPDATE workspace_dependencies
SET archived = true
WHERE archived = false
AND name IS NOT DISTINCT FROM $1
AND workspace_id = $2
AND language = $3
RETURNING description
",
self.name,
self.workspace_id,
self.language as ScriptLang
)
.fetch_optional(&mut *tx)
.await?;
let new_id = sqlx::query_scalar!(
"
INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)
VALUES ($1, $2, $3, $4, $5)
RETURNING id
",
self.name.clone(),
self.workspace_id,
self.content,
self.language as ScriptLang,
self.description
.or(prev_description.clone())
.unwrap_or("Default Workspace Dependencies".to_owned())
)
.fetch_one(&mut *tx)
.await?;
tx.commit().await?;
// Make sure trigger dependents will have latest view.
// NOTE: Uncomment for tests
// #[cfg(test)]
// assert_eq!(
// sqlx::query_scalar!(
// "
// SELECT id FROM workspace_dependencies
// WHERE archived = false
// AND name IS NOT DISTINCT FROM $1
// AND workspace_id = $2
// AND language = $3
// ",
// self.name,
// self.workspace_id,
// self.language as ScriptLang,
// )
// .fetch_one(db) // Use db
// .await?,
// new_id
// );
// It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies
trigger_dependents_to_recompute_dependencies(
&self.workspace_id,
crate::scoped_dependency_map::ScopedDependencyMap::get_dependents(
path.as_str(),
&self.workspace_id,
db,
)
.await?,
None,
None,
email,
created_by,
permissioned_as,
db,
vec![],
)
.await?;
Ok(new_id)
}
}
// Type aliases for backward compatibility
pub type RawRequirements = WorkspaceDependencies;
pub type NewRawRequirements = NewWorkspaceDependencies;
#[cfg(test)]
mod workspace_dependencies_tests {
// // TODO: test all cases when it should reject.
// #[cfg(feature = "python")]
// mod new_workspace_dependencies {
// use windmill_common::scripts::ScriptLang;
// use crate::workspace_dependencies::NewWorkspaceDependencies;
// #[sqlx::test(
// fixtures("../../tests/fixtures/base.sql",),
// migrations = "../migrations"
// )]
// async fn test_create(db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
// assert_eq!(
// NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::Python3,
// name: None,
// description: None,
// content: "global:rev1".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .unwrap(),
// 1
// );
// assert_eq!(
// NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::Python3,
// name: Some("rrs1".to_owned()),
// description: None,
// content: "rrs1:rev1".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .unwrap(),
// 2
// );
// assert!(NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::DuckDb,
// description: None,
// name: None,
// content: "".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .is_err());
// // Will act as redeployment
// assert_eq!(
// NewWorkspaceDependencies {
// workspace_id: "test-workspace".into(),
// language: ScriptLang::Python3,
// description: None,
// name: Some("rrs1".to_owned()),
// content: "rrs1:rev2".to_owned(),
// }
// .create("", "", "", &db)
// .await
// .unwrap(),
// // It will just increment id
// 3
// );
// Ok(())
// }
// #[sqlx::test(
// fixtures("../../tests/fixtures/base.sql",),
// migrations = "../migrations"
// )]
// async fn violate_constraints(db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
// let db = &db;
// let create = |name| {
// sqlx::query_scalar!(
// "
// INSERT INTO workspace_dependencies(name, workspace_id, content, language)
// VALUES ($1, 'test-workspace', 'test', 'python3')
// RETURNING id
// ",
// name
// )
// .fetch_one(db)
// };
// assert_eq!(create(Some("test".to_owned())).await.unwrap(), 1);
// assert_eq!(create(None).await.unwrap(), 2);
// assert!(create(Some("test".to_owned())).await.is_err());
// assert!(create(None).await.is_err());
// assert_eq!(
// sqlx::query_scalar!("SELECT COUNT(*) FROM workspace_dependencies",)
// .fetch_one(db)
// .await
// .unwrap()
// .unwrap(),
// 2
// );
// Ok(())
// }
// }
}
cli/src/commands/dependencies/dependencies.ts
+105
View File
@@ -0,0 +1,105 @@
// deno-lint-ignore-file no-explicit-any
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { GlobalOptions } from "../../types.ts";
import { colors, Command, log } from "../../../deps.ts";
import * as wmill from "../../../gen/services.gen.ts";
import type { ScriptLang } from "../../../gen/types.gen.ts";
import fs from "node:fs";
import { generateHash } from "../../utils/utils.ts";
import { checkifMetadataUptodate, updateMetadataGlobalLock, workspaceDependenciesPathToLanguageAndFilename } from "../../utils/metadata.ts";
async function push(
opts: GlobalOptions,
filePath: string,
language?: ScriptLang,
name?: string
): Promise<void> {
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
if (!fs.existsSync(filePath)) {
throw new Error(`File not found: ${filePath}`);
}
const content = fs.readFileSync(filePath, "utf8");
// Use the existing pushWorkspaceDependencies function
await pushWorkspaceDependencies(workspace.workspaceId, filePath, null, content);
}
const command = new Command()
.alias("deps")
.description("workspace dependencies related commands")
.command(
"push",
"Push workspace dependencies from a local file"
)
.arguments("<file_path:string>")
.option(
"--language <language:string>",
"Programming language (python3, typescript, go, php). If not specified, will be inferred from file extension."
)
.option(
"--name <name:string>",
"Name for the dependencies. If not specified, creates workspace default dependencies."
)
.action(push as any);
export async function pushWorkspaceDependencies(
workspace: string,
path: string,
_befObj: any,
newDependenciesContent: string
): Promise<void> {
try {
let res = workspaceDependenciesPathToLanguageAndFilename(path);
if (!res) {
throw new Error(`Unknown workspace dependencies file format: ${path}`);
}
let {
language,
name
} = res;
// TODO: include workspace?
// Generate hash for workspace dependencies content and metadata
const contentHash = await generateHash(newDependenciesContent + path);
// Check if dependencies are up-to-date using wmill-lock.yaml tracking
const isUpToDate = await checkifMetadataUptodate(path, contentHash, undefined);
if (isUpToDate) {
const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`;
log.info(colors.green(`${displayName} for ${language} are up-to-date, skipping push`));
return;
}
log.info(colors.yellow(`Pushing ${name ? 'named' : 'workspace default'} dependencies for ${language}...`));
await wmill.createWorkspaceDependencies({
workspace,
requestBody: {
name,
content: newDependenciesContent,
language,
workspace_id: workspace,
// Description is not supported in cli, it will use old description
description: undefined
}
});
// Update wmill-lock.yaml with new hash after successful push
await updateMetadataGlobalLock(path, contentHash);
const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`;
log.info(colors.green(`Successfully pushed ${displayName} for ${language}`));
} catch (error: any) {
log.error(colors.red(`Failed to push workspace dependencies: ${error.message}`));
throw error;
}
}
export default command;
cli/src/commands/flow/flow.ts
+3 -15
View File
@@ -200,9 +200,6 @@ async function generateLocks(
} & SyncOptions,
folder: string | undefined
) {
const useRawReqs =
opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true";
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
@@ -212,10 +209,7 @@ async function generateLocks(
folder,
false,
workspace,
opts,
undefined,
undefined,
useRawReqs
opts
);
} else {
const ignore = await ignoreF(opts);
@@ -241,10 +235,7 @@ async function generateLocks(
folder,
true,
workspace,
opts,
undefined,
undefined,
useRawReqs
opts
);
if (candidate) {
hasAny = true;
@@ -271,10 +262,7 @@ async function generateLocks(
folder,
false,
workspace,
opts,
undefined,
undefined,
useRawReqs
opts
);
}
}
cli/src/commands/script/script.ts
+15 -42
View File
@@ -26,13 +26,14 @@ import {
import { Workspace } from "../workspace/workspace.ts";
import {
generateScriptMetadataInternal,
getRawWorkspaceDependencies,
parseMetadataFile,
} from "../../utils/metadata.ts";
import {
LanguageWithRawReqsSupport,
WorkspaceDependenciesLanguage,
ScriptLanguage,
inferContentTypeFromFilePath,
languagesWithRawReqsSupport,
workspaceDependenciesLanguages,
} from "../../utils/script_common.ts";
import {
elementsToMap,
@@ -88,14 +89,13 @@ async function push(opts: PushOptions, filePath: string) {
await requireLogin(opts);
const codebases = await listSyncCodebases(opts as SyncOptions);
const globalDeps = await findGlobalDeps();
await handleFile(
filePath,
workspace,
[],
undefined,
opts,
globalDeps,
await getRawWorkspaceDependencies(),
codebases
);
log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
@@ -156,7 +156,7 @@ export async function handleScriptMetadata(
workspace: Workspace,
alreadySynced: string[],
message: string | undefined,
globalDeps: GlobalDeps,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[],
opts: GlobalOptions
): Promise<boolean> {
@@ -172,7 +172,7 @@ export async function handleScriptMetadata(
alreadySynced,
message,
opts,
globalDeps,
rawWorkspaceDependencies,
codebases
);
} else {
@@ -194,7 +194,7 @@ export async function handleFile(
alreadySynced: string[],
message: string | undefined,
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined,
globalDeps: GlobalDeps,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[]
): Promise<boolean> {
if (
@@ -326,7 +326,7 @@ export async function handleFile(
path,
workspaceRemote: workspace,
schemaOnly: codebase ? true : undefined,
globalDeps,
rawWorkspaceDependencies,
codebases,
}
: undefined
@@ -943,39 +943,10 @@ async function bootstrap(
}
export type GlobalDeps = Map<
LanguageWithRawReqsSupport,
WorkspaceDependenciesLanguage,
Record<string, string>
>;
export async function findGlobalDeps(): Promise<GlobalDeps> {
var globalDeps: GlobalDeps = new Map();
const els = await FSFSElement(Deno.cwd(), [], false);
for await (const entry of readDirRecursiveWithIgnore((p, isDir) => {
p = SEP + p;
return (
!isDir &&
// Skip if the filename is not one of lockfile names
!languagesWithRawReqsSupport.some((lockfile) =>
p.endsWith(SEP + lockfile.rrFilename)
)
);
}, els)) {
if (entry.isDirectory || entry.ignored) continue;
const content = await entry.getContentText();
// Iterate over available languages to find which lockfile
languagesWithRawReqsSupport.map((lock) => {
if (entry.path.endsWith(lock.rrFilename)) {
const current = globalDeps.get(lock) ?? {};
current[
entry.path.substring(0, entry.path.length - lock.rrFilename.length)
] = content;
globalDeps.set(lock, current);
}
});
}
return globalDeps;
}
async function generateMetadata(
opts: GlobalOptions & {
lockOnly?: boolean;
@@ -999,7 +970,7 @@ async function generateMetadata(
opts = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(opts);
const globalDeps = await findGlobalDeps();
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
if (scriptPath) {
// read script metadata file
await generateScriptMetadataInternal(
@@ -1008,11 +979,12 @@ async function generateMetadata(
opts,
false,
false,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
} else {
// TODO: test this as well.
const ignore = await ignoreF(opts);
const elems = await elementsToMap(
await FSFSElement(Deno.cwd(), codebases, false),
@@ -1036,7 +1008,7 @@ async function generateMetadata(
opts,
true,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
@@ -1063,6 +1035,7 @@ async function generateMetadata(
log.info(colors.green.bold("No metadata to update"));
return;
}
// TODO: test this
for (const e of Object.keys(elems)) {
await generateScriptMetadataInternal(
e,
@@ -1070,7 +1043,7 @@ async function generateMetadata(
opts,
false,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
cli/src/commands/sync/pull.ts
+6 -4
View File
@@ -17,6 +17,7 @@ export async function downloadZip(
includeGroups?: boolean,
includeSettings?: boolean,
includeKey?: boolean,
skipWorkspaceDependencies?: boolean,
defaultTs?: "bun" | "deno"
): Promise<JSZip | undefined> {
const requestHeaders = new Headers();
@@ -30,8 +31,8 @@ export async function downloadZip(
}
}
const zipResponse = await fetch(
workspace.remote +
const includeWorkspaceDependenciesValue = !(skipWorkspaceDependencies ?? false);
const url = workspace.remote +
"api/w/" +
workspace.workspaceId +
`/workspaces/tarball?archive_type=zip&plain_secret=${plainSecrets ?? false
@@ -39,8 +40,9 @@ export async function downloadZip(
}&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false
}&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false
}&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false
}&include_key=${includeKey ?? false}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`,
{
}&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`;
const zipResponse = await fetch(url, {
headers: requestHeaders,
method: "GET",
}
cli/src/commands/sync/sync.ts
+80 -25
View File
@@ -28,14 +28,13 @@ import { downloadZip } from "./pull.ts";
import {
exts,
findContentFile,
findGlobalDeps,
findResourceFile,
handleScriptMetadata,
removeExtensionToPath,
} from "../script/script.ts";
import { handleFile } from "../script/script.ts";
import { deepEqual, isFileResource } from "../../utils/utils.ts";
import { deepEqual, isFileResource, isWorkspaceDependencies } from "../../utils/utils.ts";
import {
SyncOptions,
getEffectiveSettings,
@@ -58,7 +57,9 @@ import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts";
import {
generateFlowLockInternal,
generateScriptMetadataInternal,
getRawWorkspaceDependencies,
readLockfile,
workspaceDependenciesPathToLanguageAndFilename,
} from "../../utils/metadata.ts";
import { OpenFlow } from "../../../gen/types.gen.ts";
import { pushResource } from "../resource/resource.ts";
@@ -311,7 +312,7 @@ function ZipFSElement(
p: string,
f: JSZip.JSZipObject
): Promise<DynFSElement[]> {
const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith(
const kind: "flow" | "app" | "script" | "resource" | "dependencies" | "other" = p.endsWith(
"flow.json"
)
? "flow"
@@ -321,6 +322,8 @@ function ZipFSElement(
? "script"
: p.endsWith("resource.json")
? "resource"
: p.startsWith("dependencies/")
? "dependencies"
: "other";
const isJson = p.endsWith(".json");
@@ -330,6 +333,8 @@ function ZipFSElement(
return p.replace("flow.json", "flow");
} else if (kind == "app") {
return p.replace("app.json", "app");
} else if (kind == "dependencies") {
return p;
} else {
return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p;
}
@@ -489,7 +494,7 @@ function ZipFSElement(
: JSON.stringify(parsed, null, 2);
}
return useYaml && isJson
return useYaml && isJson && kind != "dependencies"
? (() => {
try {
return yamlStringify(JSON.parse(content), yamlOptions);
@@ -622,7 +627,6 @@ export async function* readDirRecursiveWithIgnore(
while (stack.length > 0) {
const e = stack.pop()!;
// console.log(e.path);
yield e;
for await (const e2 of e.c()) {
if (e2.isDirectory) {
@@ -631,10 +635,9 @@ export async function* readDirRecursiveWithIgnore(
continue;
}
}
// console.log(e2.path);
stack.push({
path: e2.path,
ignored: e.ignored || ignore(e2.path, e2.isDirectory),
ignored: e.ignored || e2.isDirectory && e2.path == "dependencies" ? false : ignore(e2.path, e2.isDirectory),
isDirectory: e2.isDirectory,
// getContentBytes: e2.getContentBytes,
getContentText: e2.getContentText,
@@ -661,16 +664,20 @@ export async function elementsToMap(
ignore: (path: string, isDirectory: boolean) => boolean,
json: boolean,
skips: Skips,
specificItems?: SpecificItemsConfig
specificItems?: SpecificItemsConfig,
): Promise<{ [key: string]: string }> {
const map: { [key: string]: string } = {};
const processedBasePaths = new Set<string>();
for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
if (entry.isDirectory || entry.ignored) continue;
if (entry.isDirectory || entry.ignored) {
if (entry.path.includes("dependencies/")) {
log.info(`Ignoring dependencies-related path: ${entry.path} (isDirectory: ${entry.isDirectory}, ignored: ${entry.ignored})`);
}
continue;
}
const path = entry.path;
if (json && path.endsWith(".yaml") && !isFileResource(path)) continue;
if (!json && path.endsWith(".json") && !isFileResource(path)) continue;
if (json && path.endsWith(".yaml") && !isFileResource(path) && !isWorkspaceDependencies(path)) continue;
if (!json && path.endsWith(".json") && !isFileResource(path) && !isWorkspaceDependencies(path)) continue;
const ext = json ? ".json" : ".yaml";
if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue;
if (
@@ -694,6 +701,7 @@ export async function elementsToMap(
if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext))
continue;
// Use getTypeStrFromPath for consistent type detection
try {
const fileType = getTypeStrFromPath(path);
@@ -702,6 +710,7 @@ export async function elementsToMap(
if (skips.skipFlows && fileType === "flow") continue;
if (skips.skipApps && fileType === "app") continue;
if (skips.skipFolders && fileType === "folder") continue;
if (skips.skipWorkspaceDependencies && fileType === "workspace_dependencies") continue;
} catch {
// If getTypeStrFromPath can't determine the type, continue processing the file
}
@@ -728,6 +737,8 @@ export async function elementsToMap(
"nu",
"java",
"rb",
"in", // Python requirements.in files
"mod", // Go go.mod files
// for related places search: ADD_NEW_LANG
].includes(path.split(".").pop() ?? "") &&
!isFileResource(path)
@@ -797,7 +808,7 @@ export async function elementsToMap(
// No specific items configuration, use regular path
map[entry.path] = content;
}
}
}
return map;
}
@@ -810,6 +821,7 @@ export interface Skips {
skipFlows?: boolean | undefined;
skipApps?: boolean | undefined;
skipFolders?: boolean | undefined;
skipWorkspaceDependencies?: boolean | undefined;
skipScriptsMetadata?: boolean | undefined;
includeSchedules?: boolean | undefined;
includeTriggers?: boolean | undefined;
@@ -878,11 +890,14 @@ async function compareDynFSElement(
if (skipMetadata) {
continue;
}
if (k.startsWith("dependencies/")) {
log.info(`Adding workspace dependencies file: ${k}`);
}
changes.push({ name: "added", path: k, content: v });
} else {
if (m2[k] == v) {
continue;
} else if (k.endsWith(".json")) {
} else if (k.endsWith(".json") && !isWorkspaceDependencies(k)) {
let parsedV, parsedM2;
try {
parsedV = JSON.parse(v);
@@ -1093,6 +1108,7 @@ export async function ignoreF(wmillconf: {
excludes?: string[];
extraIncludes?: string[];
skipResourceTypes?: boolean;
skipWorkspaceDependencies?: boolean;
json?: boolean;
includeUsers?: boolean;
includeGroups?: boolean;
@@ -1151,6 +1167,9 @@ export async function ignoreF(wmillconf: {
if (wmillconf.includeKey && fileType === "encryption_key") {
return false; // Don't ignore, always include
}
if (!wmillconf.skipWorkspaceDependencies && fileType === "workspace_dependencies") {
return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped)
}
} catch {
// If getTypeStrFromPath can't determine the type, fall through to normal logic
}
@@ -1269,8 +1288,7 @@ export async function pull(
} catch {
// ignore
}
const remote = ZipFSElement(
(await downloadZip(
const zipFile = await downloadZip(
workspace,
opts.plainSecrets,
opts.skipVariables,
@@ -1283,8 +1301,12 @@ export async function pull(
opts.includeGroups,
opts.includeSettings,
opts.includeKey,
opts.skipWorkspaceDependencies,
opts.defaultTs
))!,
);
const remote = ZipFSElement(
zipFile!,
!opts.json,
opts.defaultTs ?? "bun",
resourceTypeToFormatExtension,
@@ -1308,6 +1330,19 @@ export async function pull(
log.info(
`remote (${workspace.name}) -> local: ${changes.length} changes to apply`
);
// Debug: show all changes for push operation
if (changes.length > 0) {
log.info("All changes:");
changes.forEach(change => {
if (change.path.startsWith("dependencies/")) {
log.info(` ${change.name}: ${change.path} [WORKSPACE DEPS]`);
} else {
log.info(` ${change.name}: ${change.path}`);
}
});
}
// Handle JSON output for dry-run
if (opts.dryRun && opts.jsonOutput) {
@@ -1495,9 +1530,9 @@ export async function pull(
}
log.info("All local changes pulled, now updating wmill-lock.yaml");
await readLockfile(); // ensure wmill-lock.yaml exists
const globalDeps = await findGlobalDeps();
const tracker: ChangeTracker = await buildTracker(changes);
const rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
for (const change of tracker.scripts) {
await generateScriptMetadataInternal(
@@ -1506,14 +1541,14 @@ export async function pull(
opts,
false,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
true
);
}
for (const change of tracker.flows) {
log.info(`Updating lock for flow ${change}`);
await generateFlowLockInternal(change, false, workspace, opts, true);
await generateFlowLockInternal(change, false, workspace, opts, true, false);
}
if (tracker.apps.length > 0) {
log.info(
@@ -1733,6 +1768,7 @@ export async function push(
opts.includeGroups,
opts.includeSettings,
opts.includeKey,
opts.skipWorkspaceDependencies,
opts.defaultTs
))!,
!opts.json,
@@ -1754,7 +1790,8 @@ export async function push(
specificItems
);
const globalDeps = await findGlobalDeps();
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
const tracker: ChangeTracker = await buildTracker(changes);
@@ -1767,7 +1804,7 @@ export async function push(
opts,
true,
true,
globalDeps,
rawWorkspaceDependencies,
codebases,
false
);
@@ -1932,7 +1969,7 @@ export async function push(
workspace,
alreadySynced,
opts.message,
globalDeps,
rawWorkspaceDependencies,
codebases,
opts
)
@@ -1948,7 +1985,7 @@ export async function push(
alreadySynced,
opts.message,
opts,
globalDeps,
rawWorkspaceDependencies,
codebases
)
) {
@@ -2040,7 +2077,7 @@ export async function push(
alreadySynced,
opts.message,
opts,
globalDeps,
rawWorkspaceDependencies,
codebases
)
) {
@@ -2225,6 +2262,22 @@ export async function push(
".group.json"
),
});
break;
case "workspace_dependencies":
const relativePath = removePathPrefix(change.path, "dependencies");
const res = workspaceDependenciesPathToLanguageAndFilename(change.path);
if (!res) {
throw new Error(`Unknown workspace dependencies file format: ${change.path}`);
}
const { name, language } = res;
await wmill.deleteWorkspaceDependencies({
workspace: workspaceId,
language,
name
});
break;
default:
break;
@@ -2323,6 +2376,7 @@ const command = new Command()
.option("--skip-flows", "Skip syncing flows")
.option("--skip-apps", "Skip syncing apps")
.option("--skip-folders", "Skip syncing folders")
.option("--skip-workspace-dependencies", "Skip syncing workspace dependencies")
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
.option("--include-schedules", "Include syncing schedules")
.option("--include-triggers", "Include syncing triggers")
@@ -2371,6 +2425,7 @@ const command = new Command()
.option("--skip-flows", "Skip syncing flows")
.option("--skip-apps", "Skip syncing apps")
.option("--skip-folders", "Skip syncing folders")
.option("--skip-workspace-dependencies", "Skip syncing workspace dependencies")
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
.option("--include-schedules", "Include syncing schedules")
.option("--include-triggers", "Include syncing triggers")
cli/src/core/conf.ts
+3
View File
@@ -26,6 +26,7 @@ export interface SyncOptions {
skipResources?: boolean;
skipResourceTypes?: boolean;
skipSecrets?: boolean;
skipWorkspaceDependencies?: boolean;
skipScripts?: boolean;
skipFlows?: boolean;
skipApps?: boolean;
@@ -317,6 +318,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly<
| "skipSecrets"
| "includeSchedules"
| "includeTriggers"
| "skipWorkspaceDependencies"
| "skipScripts"
| "skipFlows"
| "skipApps"
@@ -346,6 +348,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly<
includeGroups: false,
includeSettings: false,
includeKey: false,
skipWorkspaceDependencies: false,
} as const;
export async function mergeConfigWithConfigFile<T>(
cli/src/main.ts
+2
View File
@@ -36,6 +36,7 @@ import { pull, push } from "./commands/sync/sync.ts";
import { add as workspaceAdd } from "./commands/workspace/workspace.ts";
import workers from "./commands/workers/workers.ts";
import queues from "./commands/queues/queues.ts";
import dependencies from "./commands/dependencies/dependencies.ts";
import init from "./commands/init/init.ts";
export {
@@ -126,6 +127,7 @@ const command = new Command()
.command("worker-groups", workerGroups)
.command("workers", workers)
.command("queues", queues)
.command("dependencies", dependencies)
.command("version --version", "Show version information")
.action(async (opts: any) => {
console.log("CLI version: " + VERSION);
cli/src/types.ts
+12 -3
View File
@@ -17,10 +17,11 @@ import { pushResourceType } from "./commands/resource-type/resource-type.ts";
import { pushVariable } from "./commands/variable/variable.ts";
import { yamlOptions } from "./commands/sync/sync.ts";
import { showDiffs } from "./core/conf.ts";
import { deepEqual, isFileResource } from "./utils/utils.ts";
import { deepEqual, isFileResource, isWorkspaceDependencies } from "./utils/utils.ts";
import { pushSchedule } from "./commands/schedule/schedule.ts";
import { pushWorkspaceUser } from "./commands/user/user.ts";
import { pushGroup } from "./commands/user/user.ts";
import { pushWorkspaceDependencies } from "./commands/dependencies/dependencies.ts";
import { pushWorkspaceSettings, pushWorkspaceKey } from "./core/settings.ts";
import { pushTrigger } from "./commands/trigger/trigger.ts";
@@ -187,6 +188,8 @@ export async function pushObj(
await pushWorkspaceUser(workspace, p, befObj, newObj);
} else if (typeEnding === "group") {
await pushGroup(workspace, p, befObj, newObj);
} else if (typeEnding === "workspace_dependencies") {
await pushWorkspaceDependencies(workspace, p, befObj, newObj);
} else if (typeEnding === "settings") {
await pushWorkspaceSettings(workspace, p, befObj, newObj);
} else if (typeEnding === "encryption_key") {
@@ -199,7 +202,9 @@ export async function pushObj(
}
export function parseFromPath(p: string, content: string): any {
return p.endsWith(".yaml")
return isWorkspaceDependencies(p)
? content
: p.endsWith(".yaml")
? yamlParseContent(p, content)
: p.endsWith(".json")
? JSON.parse(content)
@@ -237,13 +242,17 @@ export function getTypeStrFromPath(
| "user"
| "group"
| "settings"
| "encryption_key" {
| "encryption_key"
| "workspace_dependencies" {
if (p.includes(".flow" + SEP)) {
return "flow";
}
if (p.includes(".app" + SEP)) {
return "app";
}
if (p.startsWith("dependencies" + SEP)) {
return "workspace_dependencies";
}
const parsed = path.parse(p);
if (
parsed.ext == ".go" ||
cli/src/utils/metadata.ts
+86 -83
View File
@@ -14,12 +14,12 @@ import {
} from "../../bootstrap/script_bootstrap.ts";
import { Workspace } from "../commands/workspace/workspace.ts";
import {
languagesWithRawReqsSupport,
LanguageWithRawReqsSupport,
workspaceDependenciesLanguages,
WorkspaceDependenciesLanguage,
ScriptLanguage,
} from "./script_common.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { GlobalDeps, exts, findGlobalDeps } from "../commands/script/script.ts";
import { exts } from "../commands/script/script.ts";
import {
FSFSElement,
findCodebase,
@@ -48,29 +48,51 @@ export class LockfileGenerationError extends Error {
export async function generateAllMetadata() {}
function findClosestRawReqs(
lang: LanguageWithRawReqsSupport | undefined,
remotePath: string,
globalDeps: GlobalDeps
): string | undefined {
let bestCandidate: { k: string; v: string } | undefined = undefined;
if (lang) {
Object.entries(globalDeps.get(lang) ?? {}).forEach(([k, v]) => {
if (
remotePath.startsWith(k) &&
k.length >= (bestCandidate?.k ?? "").length
) {
bestCandidate = { k, v };
export async function getRawWorkspaceDependencies(): Promise<Record<string, string>> {
const rawWorkspaceDeps: Record<string, string> = {};
try {
for await (const entry of Deno.readDir("dependencies")) {
if (entry.isDirectory) continue;
const filePath = `dependencies/${entry.name}`;
const content = await Deno.readTextFile(filePath);
// Find matching language
for (const lang of workspaceDependenciesLanguages) {
if (entry.name.endsWith(lang.filename)) {
// Check if out of sync
const contentHash = await generateHash(content + filePath);
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
if (!isUpToDate) {
rawWorkspaceDeps[filePath] = content;
}
break;
}
}
});
}
} catch {
// dependencies directory doesn't exist
}
// @ts-ignore
return bestCandidate?.v;
return rawWorkspaceDeps;
}
export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined {
const relativePath = path.replace("dependencies/", "");
for (const { filename, language } of workspaceDependenciesLanguages) {
if (relativePath.endsWith(filename)) {
return {
name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""),
language
};
}
}
}
const TOP_HASH = "__flow_hash";
async function generateFlowHash(
rawReqs: Record<string, string> | undefined,
rawWorkspaceDependencies: Record<string, string>,
folder: string,
defaultTs: "bun" | "deno" | undefined
) {
@@ -78,17 +100,9 @@ async function generateFlowHash(
const hashes: Record<string, string> = {};
for await (const f of elems.getChildren()) {
if (exts.some((e) => f.path.endsWith(e))) {
let reqs: string | undefined;
if (rawReqs) {
// Get language name from path
const lang = inferContentTypeFromFilePath(f.path, defaultTs);
// Get lock for that language
[, reqs] =
Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? [];
}
// Embed lock into hash
// Embed workspace dependencies into hash
hashes[f.path] = await generateHash(
(await f.getContentText()) + (reqs ?? "")
(await f.getContentText()) + JSON.stringify(rawWorkspaceDependencies)
);
}
}
@@ -102,8 +116,7 @@ export async function generateFlowLockInternal(
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean,
useRawReqs?: boolean
noStaleMessage?: boolean
): Promise<string | void> {
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
@@ -115,24 +128,9 @@ export async function generateFlowLockInternal(
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
}
let rawReqs: Record<string, string> | undefined = undefined;
if (useRawReqs) {
// Find all dependency files in the workspace
const globalDeps = await findGlobalDeps();
// Find closest dependency files for this flow
rawReqs = {};
// TODO: PERF: Only include raw reqs for the languages that are in the flow
languagesWithRawReqsSupport.map((lang) => {
const dep = findClosestRawReqs(lang, folder, globalDeps);
if (dep) {
// @ts-ignore
rawReqs[lang.language] = dep;
}
});
}
let hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs);
// Always get out-of-sync workspace dependencies
let rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
const conf = await readLockfile();
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
@@ -146,15 +144,12 @@ export async function generateFlowLockInternal(
return remote_path;
}
if (useRawReqs) {
log.warn(
"If using local lockfiles, following redeployments from Web App will inevitably override generated lockfiles by CLI. To maintain your script's lockfiles you will need to redeploy only from CLI. (Behavior is subject to change)"
);
if (Object.keys(rawWorkspaceDependencies).length > 0) {
log.info(
(await blueColor())(
`Found raw requirements (${languagesWithRawReqsSupport
.map((l) => l.rrFilename)
.join("/")}) for ${folder}, using it`
`Found workspace dependencies (${workspaceDependenciesLanguages
.map((l) => l.filename)
.join("/")}) for ${folder}, using them`
)
);
}
@@ -192,7 +187,7 @@ export async function generateFlowLockInternal(
workspace,
flowValue.value,
remote_path,
rawReqs
rawWorkspaceDependencies
);
const inlineScripts = extractInlineScriptsForFlows(
@@ -212,7 +207,7 @@ export async function generateFlowLockInternal(
);
}
hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs);
hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
await clearGlobalLock(folder);
for (const [path, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(folder, hash, path);
@@ -236,7 +231,7 @@ export async function generateScriptMetadataInternal(
},
dryRun: boolean,
noStaleMessage: boolean,
globalDeps: GlobalDeps,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[],
justUpdateMetadataLock?: boolean
): Promise<string | undefined> {
@@ -246,6 +241,14 @@ export async function generateScriptMetadataInternal(
const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
// Filter workspace dependencies to only include those matching the script's language
const filteredRawWorkspaceDependencies: Record<string, string> = {};
for (const [depPath, depContent] of Object.entries(rawWorkspaceDependencies)) {
const depInfo = workspaceDependenciesPathToLanguageAndFilename(depPath);
if (depInfo && depInfo.language === language) {
filteredRawWorkspaceDependencies[depPath] = depContent;
}
}
const metadataWithType = await parseMetadataFile(
remotePath,
@@ -256,14 +259,8 @@ export async function generateScriptMetadataInternal(
const scriptContent = await Deno.readTextFile(scriptPath);
const metadataContent = await Deno.readTextFile(metadataWithType.path);
const rrLang = languagesWithRawReqsSupport.find(
(l) => language == l.language
);
const rawReqs = findClosestRawReqs(rrLang, scriptPath, globalDeps);
let hash = await generateScriptHash(rawReqs, scriptContent, metadataContent);
// Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically
let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent);
if (await checkifMetadataUptodate(remotePath, hash, undefined)) {
if (!noStaleMessage) {
@@ -304,7 +301,7 @@ export async function generateScriptMetadataInternal(
language,
remotePath,
metadataParsedContent,
rawReqs
filteredRawWorkspaceDependencies
);
} else {
metadataParsedContent.lock = "";
@@ -323,7 +320,7 @@ export async function generateScriptMetadataInternal(
const metadataContentUsedForHash = newMetadataContent;
hash = await generateScriptHash(
rawReqs,
filteredRawWorkspaceDependencies,
scriptContent,
metadataContentUsedForHash
);
@@ -366,11 +363,11 @@ async function updateScriptLock(
language: ScriptLanguage,
remotePath: string,
metadataContent: Record<string, any>,
rawDeps: string | undefined
rawWorkspaceDependencies: Record<string, string>
): Promise<void> {
if (
!(
languagesWithRawReqsSupport.some((l) => l.language == language) ||
workspaceDependenciesLanguages.some((l) => l.language == language) ||
language == "deno" ||
language == "rust" ||
language == "ansible"
@@ -379,9 +376,12 @@ async function updateScriptLock(
return;
}
if (rawDeps) {
log.info(`Generating script lock for ${remotePath} with raw deps`);
if (Object.keys(rawWorkspaceDependencies).length > 0) {
const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', ');
log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
}
// generate the script lock running a dependency job in Windmill and update it inplace
// TODO: update this once the client is released
const extraHeaders = getHeaders();
@@ -402,7 +402,8 @@ async function updateScriptLock(
script_path: remotePath,
},
],
raw_deps: rawDeps,
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies : null,
entrypoint: remotePath,
}),
}
@@ -451,12 +452,12 @@ export async function updateFlow(
workspace: Workspace,
flow_value: FlowValue,
remotePath: string,
rawDeps?: Record<string, string>
rawWorkspaceDependencies: Record<string, string>
): Promise<FlowValue | undefined> {
let rawResponse;
if (rawDeps != undefined) {
log.info(colors.blue("Using raw requirements for flow dependencies"));
if (Object.keys(rawWorkspaceDependencies).length > 0) {
log.info(colors.blue("Using raw workspace dependencies for flow dependencies"));
// generate the script lock running a dependency job in Windmill and update it inplace
const extraHeaders = getHeaders();
@@ -473,7 +474,9 @@ export async function updateFlow(
flow_value,
path: remotePath,
use_local_lockfiles: true,
raw_deps: rawDeps,
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies
: null,
}),
}
);
@@ -767,7 +770,7 @@ export async function parseMetadataFile(
path: string;
workspaceRemote: Workspace;
schemaOnly?: boolean;
globalDeps: GlobalDeps;
rawWorkspaceDependencies: Record<string, string>;
codebases: SyncCodebase[]
})
| undefined,
@@ -829,7 +832,7 @@ export async function parseMetadataFile(
generateMetadataIfMissing,
false,
false,
generateMetadataIfMissing.globalDeps,
generateMetadataIfMissing.rawWorkspaceDependencies,
generateMetadataIfMissing.codebases,
false
);
@@ -911,12 +914,12 @@ export async function checkifMetadataUptodate(
}
export async function generateScriptHash(
rawReqs: string | undefined,
rawWorkspaceDependencies: Record<string, string>,
scriptContent: string,
newMetadataContent: string
) {
return await generateHash(
(rawReqs ?? "") + scriptContent + newMetadataContent
JSON.stringify(rawWorkspaceDependencies) + scriptContent + newMetadataContent
);
}
cli/src/utils/script_common.ts
+10 -11
View File
@@ -26,18 +26,17 @@ export type ScriptLanguage =
// To make language support raw requirements:
// 1. Add value here
// 2. Modify backend to allow raw deps
export type LanguageWithRawReqsSupport =
| { language: "bun", rrFilename /** (raw requirements filename) */: "package.json" }
// TODO: Add `requirements.in` - more intuitive and reflects better what actually happens
| { language: "python3", rrFilename: "requirements.txt" }
| { language: "php", rrFilename: "composer.json" }
| { language: "go", rrFilename: "go.mod" };
export type WorkspaceDependenciesLanguage =
| { language: "bun", filename /** (raw requirements filename) */: "package.json" }
| { language: "python3", filename: "requirements.in" }
| { language: "php", filename: "composer.json" }
| { language: "go", filename: "go.mod" };
export const languagesWithRawReqsSupport: LanguageWithRawReqsSupport[] = [
{ language: "bun", rrFilename: "package.json" },
{ language: "python3", rrFilename: "requirements.txt" },
{ language: "php", rrFilename: "composer.json" },
{ language: "go", rrFilename: "go.mod" },
export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
{ language: "bun", filename: "package.json" },
{ language: "python3", filename: "requirements.in" },
{ language: "php", filename: "composer.json" },
{ language: "go", filename: "go.mod" },
] as const;
export function inferContentTypeFromFilePath(
cli/src/utils/utils.ts
+5 -1
View File
@@ -145,6 +145,10 @@ export function isFileResource(path: string): boolean {
);
}
export function isWorkspaceDependencies(path: string): boolean {
return path.startsWith("dependencies/")
}
export function printSync(input: string | Uint8Array, to = Deno.stdout) {
let bytesWritten = 0
const bytes = typeof input === 'string' ? new TextEncoder().encode(input) : input
@@ -235,4 +239,4 @@ export function writeIfChanged(path: string, content: string): boolean {
// console.log(`Writing content to ${path}`);
Deno.writeTextFileSync(path, content);
return true; // File was written
}
}
flake.nix
+13 -12
View File
@@ -50,7 +50,7 @@
xmlsec.dev
libxslt.dev
libclang.dev
libffi # For deno_ffi
libffi # For deno_ffi
libtool
nodejs
postgresql
@@ -118,20 +118,17 @@
fi
wm-cli-deps
'';
buildInputs = buildInputs ++ [
pkgs.deno
];
buildInputs = buildInputs ++ [ pkgs.deno ];
packages = [
(pkgs.writeScriptBin "wm-cli" ''
deno run -A --no-check $FLAKE_ROOT/cli/src/main.ts $*
'')
(pkgs.writeScriptBin "wm-cli-deps" ''
pushd $FLAKE_ROOT/cli/
${
if pkgs.stdenv.isDarwin
then "./gen_wm_client_mac.sh && ./windmill-utils-internal/gen_wm_client_mac.sh"
else "./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh"
}
${if pkgs.stdenv.isDarwin then
"./gen_wm_client_mac.sh && ./windmill-utils-internal/gen_wm_client_mac.sh"
else
"./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh"}
popd
'')
];
@@ -278,6 +275,8 @@
PHP_PATH = "${pkgs.php}/bin/php";
COMPOSER_PATH = "${pkgs.php84Packages.composer}/bin/composer";
BUN_PATH = "${pkgs.bun}/bin/bun";
NODE_PATH = "${pkgs.nodejs}/bin/node";
NODE_BIN_PATH = "${pkgs.nodejs}/bin/node";
UV_PATH = "${pkgs.uv}/bin/uv";
NU_PATH = "${pkgs.nushell}/bin/nu";
JAVA_PATH = "${pkgs.jdk21}/bin/java";
@@ -314,9 +313,11 @@
# See https://web.archive.org/web/20220523141208/https://hoverbear.org/blog/rust-bindgen-in-nix/
BINDGEN_EXTRA_CLANG_ARGS =
# Prevent clang from using system headers - only use Nix headers
"-nostdinc ${builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"} ${
builtins.readFile "${stdenv.cc}/nix-support/libc-cflags"
} ${builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"} ${
"-nostdinc ${
builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"
} ${builtins.readFile "${stdenv.cc}/nix-support/libc-cflags"} ${
builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"
} ${
builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags"
} -idirafter ${pkgs.libiconv}/include ${
lib.optionalString stdenv.cc.isClang
frontend/src/lib/components/DependenciesDeploymentWarning.svelte
+295
View File
@@ -0,0 +1,295 @@
<script lang="ts">
import { WorkspaceService } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { Button } from './common'
import Alert from './common/alert/Alert.svelte'
import Modal from './common/modal/Modal.svelte'
import {
ChevronRight,
ChevronDown,
FileText,
GitBranch,
Layers,
ArrowRight,
Package
} from 'lucide-svelte'
interface DependencyNode {
path: string
kind: 'script' | 'flow' | 'app'
nodeIds?: string[]
children?: DependencyNode[]
expanded?: boolean
loading?: boolean
childrenCount?: number
}
let {
importedPath,
title = 'Dependency Warning',
confirmText = 'Proceed Anyway',
cancelText = 'Cancel',
onConfirm,
onCancel,
open = true
}: {
importedPath: string
title?: string
confirmText?: string
cancelText?: string
onConfirm: () => void
onCancel: () => void
open?: boolean
} = $props()
let dependencies: DependencyNode[] = $state([])
let loading = $state(true)
// Load initial dependencies
async function loadInitialDependencies() {
try {
loading = true
console.log('Loading dependents for:', importedPath)
const dependents = await WorkspaceService.getDependents({
workspace: $workspaceStore!,
importedPath
})
console.log('Found dependents:', dependents)
if (dependents.length === 0) {
dependencies = []
return
}
// Get counts for all dependents to see which have children
const dependentPaths = dependents.map(dep => dep.importer_path)
const amounts = await WorkspaceService.getDependentsAmounts({
workspace: $workspaceStore!,
requestBody: dependentPaths
})
console.log('Dependents amounts:', amounts)
const amountMap = new Map(amounts.map(a => [a.imported_path, a.count]))
dependencies = dependents.map(dep => ({
path: dep.importer_path,
kind: dep.importer_kind as 'script' | 'flow' | 'app',
nodeIds: dep.importer_node_ids ?? undefined,
expanded: false,
childrenCount: amountMap.get(dep.importer_path) || 0
}))
} catch (error) {
console.error('Error loading dependencies:', error)
dependencies = []
} finally {
loading = false
}
}
// Load children for a specific dependency
async function loadChildren(node: DependencyNode) {
if (node.children || node.loading) return
try {
node.loading = true
const childImportedPath = node.path // Use the dependency path directly
const dependents = await WorkspaceService.getDependents({
workspace: $workspaceStore!,
importedPath: childImportedPath
})
// Get counts for child dependents
const dependentPaths = dependents.map(dep => dep.importer_path)
const amounts = await WorkspaceService.getDependentsAmounts({
workspace: $workspaceStore!,
requestBody: dependentPaths
})
const amountMap = new Map(amounts.map(a => [a.imported_path, a.count]))
node.children = dependents.map(dep => ({
path: dep.importer_path,
kind: dep.importer_kind as 'script' | 'flow' | 'app',
nodeIds: dep.importer_node_ids ?? undefined,
expanded: false,
childrenCount: amountMap.get(dep.importer_path) || 0
}))
} catch (error) {
console.error('Error loading children:', error)
node.children = []
} finally {
node.loading = false
}
}
function getIcon(kind: string) {
switch (kind) {
case 'script':
return FileText
case 'flow':
return GitBranch
case 'app':
return Layers
default:
return FileText
}
}
function getKindLabel(kind: string) {
return kind.charAt(0).toUpperCase() + kind.slice(1)
}
async function toggleExpand(node: DependencyNode) {
if (!node.expanded && !node.children) {
await loadChildren(node)
}
node.expanded = !node.expanded
}
function getTotalDependentsCount(): number {
function countNode(node: DependencyNode): number {
let count = 1
if (node.children) {
count += node.children.reduce((sum, child) => sum + countNode(child), 0)
}
return count
}
return dependencies.reduce((sum, dep) => sum + countNode(dep), 0)
}
// Load dependencies when component mounts
$effect(() => {
if ($workspaceStore && importedPath) {
loadInitialDependencies()
}
})
function handleCancel() {
onCancel()
}
async function handleConfirm() {
await onConfirm()
}
</script>
<Modal {open} {title} {cancelText} kind="button" on:canceled={handleCancel} class="sm:max-w-2xl">
<div class="space-y-4">
{#if loading}
<div class="flex items-center gap-2">
<div class="animate-spin rounded-full h-4 w-4 border-b-2 border-yellow-600"></div>
<span class="text-sm">Loading dependencies...</span>
</div>
{:else}
<!-- Header with workspace dependencies info -->
<div class="bg-yellow-50 dark:bg-yellow-900/20 rounded-lg p-3 border-l-4 border-yellow-400">
<div class="flex items-center gap-2 mb-2">
<Package size={16} class="text-yellow-600 dark:text-yellow-400" />
<span class="font-medium text-yellow-800 dark:text-yellow-200">Workspace Dependencies</span>
</div>
<div class="font-mono text-sm text-yellow-700 dark:text-yellow-300 ml-6">
{importedPath}
</div>
</div>
{#if dependencies.length === 0}
<Alert type="info" title="No Dependent Runnables Found">
{#snippet children()}
<p class="text-sm">No dependent runnables were found for these workspace dependencies, but the action will still proceed.</p>
{/snippet}
</Alert>
{:else}
<!-- Arrow pointing down -->
<div class="flex justify-center">
<ArrowRight size={16} class="text-yellow-600 dark:text-yellow-400 transform rotate-90" />
</div>
<!-- Summary -->
<div class="bg-yellow-50 dark:bg-yellow-900/30 rounded p-3">
<p class="text-sm text-yellow-800 dark:text-yellow-200">
This action will trigger redeployment of <strong>{getTotalDependentsCount()}</strong>
{getTotalDependentsCount() === 1 ? 'dependent runnable' : 'dependent runnables'}:
</p>
</div>
<!-- Dependency tree -->
<div class="space-y-1 max-h-64 overflow-y-auto">
{#each dependencies as dependency}
{@render DependencyNode({ node: dependency, level: 0 })}
{/each}
</div>
{/if}
{/if}
</div>
<svelte:fragment slot="actions">
<Button size="sm" color="warning" on:click={handleConfirm}>
{confirmText}
</Button>
</svelte:fragment>
</Modal>
{#snippet DependencyNode({ node, level }: { node: DependencyNode, level: number })}
{@const Icon = getIcon(node.kind)}
<div style="margin-left: {level * 1}rem;">
<div class="flex items-center gap-2 p-2 bg-white dark:bg-gray-800 rounded border hover:bg-gray-50 dark:hover:bg-gray-700 transition-colors">
<!-- Expand/collapse button -->
{#if (node.childrenCount ?? 0) > 0}
<button
onclick={() => toggleExpand(node)}
class="flex-shrink-0 p-1 hover:bg-gray-200 dark:hover:bg-gray-600 rounded"
disabled={node.loading}
>
{#if node.loading}
<div class="animate-spin rounded-full h-3 w-3 border-b-2 border-gray-400"></div>
{:else if node.expanded}
<ChevronDown size={12} class="text-gray-500" />
{:else}
<ChevronRight size={12} class="text-gray-500" />
{/if}
</button>
{:else}
<div class="w-5"></div>
{/if}
<!-- Kind icon and label -->
<div class="flex items-center gap-1.5 flex-shrink-0">
<Icon size={14} class="text-blue-600 dark:text-blue-400" />
<span class="text-xs font-medium text-blue-700 dark:text-blue-300 uppercase tracking-wide">
{getKindLabel(node.kind)}
</span>
</div>
<!-- Path -->
<div class="flex-1 min-w-0">
<span class="font-mono text-sm text-gray-900 dark:text-gray-100 truncate block">
{node.path}
</span>
{#if node.nodeIds && node.nodeIds.length > 0}
<span class="text-xs text-gray-500 dark:text-gray-400">
{node.nodeIds.length} node{node.nodeIds.length !== 1 ? 's' : ''}
</span>
{/if}
</div>
<!-- Children count -->
{#if (node.childrenCount ?? 0) > 0}
<span class="flex-shrink-0 text-xs bg-gray-100 dark:bg-gray-700 text-gray-600 dark:text-gray-300 px-2 py-1 rounded-full">
{node.childrenCount} dep{node.childrenCount !== 1 ? 's' : ''}
</span>
{/if}
</div>
<!-- Children -->
{#if node.expanded && node.children}
<div class="ml-2 border-l border-gray-200 dark:border-gray-600">
{#each node.children as child}
{@render DependencyNode({ node: child, level: level + 1 })}
{/each}
</div>
{/if}
</div>
{/snippet}
frontend/src/lib/components/HighlightCode.svelte
+5 -3
View File
@@ -19,11 +19,11 @@
import { copyToClipboard } from '$lib/utils'
import { ClipboardCopy } from 'lucide-svelte'
import HighlightTheme from './HighlightTheme.svelte'
import type { LanguageType } from 'svelte-highlight/languages'
import { json, type LanguageType } from 'svelte-highlight/languages'
interface Props {
code?: string
language: Script['language'] | 'bunnative' | 'frontend' | undefined
language: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined
highlightLanguage?: LanguageType<string> | undefined
lines?: boolean
className?: string
@@ -43,7 +43,7 @@
applyButtonIcon = undefined
}: Props = $props()
function getLang(lang: Script['language'] | 'bunnative' | 'frontend' | undefined) {
function getLang(lang: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined) {
switch (lang) {
case 'python3':
return python
@@ -91,6 +91,8 @@
return java
case 'ruby':
return ruby
case 'json':
return json
// for related places search: ADD_NEW_LANG
default:
return typescript
frontend/src/lib/components/ScriptBuilder.svelte
+3 -2
View File
@@ -12,7 +12,7 @@
PostgresTriggerService,
CaptureService,
type ScriptLang,
WorkerService
WorkerService,
} from '$lib/gen'
import { inferArgs } from '$lib/infer'
import {
@@ -189,7 +189,7 @@
: undefined
)
const simplifiedPoll = writable(false)
export function setPrimarySchedule(schedule: ScheduleTrigger | undefined | false) {
primaryScheduleStore.set(schedule)
loadTriggers()
@@ -1759,6 +1759,7 @@
/>
</div>
{/if}
</div>
{#if $enterpriseLicense && initialPath != ''}
frontend/src/lib/components/WorkspaceDependenciesEditor.svelte
+512
View File
@@ -0,0 +1,512 @@
<script lang="ts">
import { createEventDispatcher } from 'svelte'
import { WorkspaceDependenciesService, WorkspaceService, type ScriptLang, type WorkspaceDependencies } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { Button } from './common'
import Drawer from './common/drawer/Drawer.svelte'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import Alert from './common/alert/Alert.svelte'
import DependenciesDeploymentWarning from './DependenciesDeploymentWarning.svelte'
import type SimpleEditor from './SimpleEditor.svelte'
import { sendUserToast } from '$lib/toast'
import Section from './Section.svelte'
import { Loader2, Rocket, Code2, FolderOpen } from 'lucide-svelte'
import Select from './select/Select.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
const dispatch = createEventDispatcher()
// Helper function to get full filename
export function getFullFilename(language: ScriptLang, name: string | null): string | null {
const extension = getFileExtension(language)
if (extension == null) return null;
return name ? `${name}.${extension}` : extension
}
export function getWorkspaceDependenciesPath(name: string | null, language: ScriptLang): string | null {
const extension = getFileExtension(language)
if (extension == null) return null;
return name ? `dependencies/${name}.${extension}` : `dependencies/${extension}`
}
export function getDisplayName(deps: WorkspaceDependencies): string {
return deps.name || `Default (${deps.language})`
}
export function getFileExtension(language: ScriptLang): string | null {
switch (language) {
case 'python3':
return 'requirements.in'
case 'bun':
return 'package.json'
// case 'go':
// return 'go.mod'
case 'php':
return 'composer.json'
default:
return null
}
}
// Load existing workspace defaults from API
async function loadExistingWorkspaceDefaults(): Promise<void> {
if (!$workspaceStore) return
try {
const workspaceDeps = await WorkspaceDependenciesService.listWorkspaceDependencies({
workspace: $workspaceStore
})
// Reset defaults
existingWorkspaceDefaults = {}
workspaceDefaultIds = {}
// Check for workspace defaults (where name is null) for each language
workspaceDeps.forEach((dep) => {
if (dep.name === null) {
existingWorkspaceDefaults[dep.language] = true
workspaceDefaultIds[dep.language] = dep.id
}
})
} catch (error) {
console.error('Error loading existing workspace defaults:', error)
// Fallback to empty defaults on error
existingWorkspaceDefaults = {}
workspaceDefaultIds = {}
}
}
// Check if workspace default exists for a language
function hasWorkspaceDefault(language: string): boolean {
return existingWorkspaceDefaults[language] || false
}
function goToWorkspaceDefault(): void {
const language = workspaceDependencies.language;
const defaultId = workspaceDefaultIds[language];
if (defaultId) {
// Close current drawer and edit the workspace default
drawer?.closeDrawer()
// Call editWorkspaceDependencies with the workspace default info (name = null for workspace default)
setTimeout(() => {
editWorkspaceDependencies(defaultId, undefined, language)
}, 100) // Small delay to allow drawer to close before opening new one
} else {
console.error('No workspace default found for language:', language)
sendUserToast(`No workspace default found for ${language}`, true)
}
}
let workspaceDependenciesName: string = $state('')
let workspaceDependenciesType = $state<'workspace' | 'named'>('workspace')
// Language options for workspace dependencies - only supported languages
const LANGUAGE_OPTIONS = [
{ value: 'python3', label: 'Python' },
{ value: 'bun', label: 'TypeScript (Bun/Bunnative)' },
// { value: 'go', label: 'Go' },
{ value: 'php', label: 'PHP' }
]
// Default templates for each language
const LANGUAGE_TEMPLATES: Record<string, string> = {
python3: `# Python Requirements (requirements.in format)
# # py: 3.11
# ^ Uncomment to pin to python version.
# Core dependencies
requests>=2.31.0
pandas>=2.0.0
numpy>=1.24.0
`,
bun: `{
"dependencies": {
"number-to-words": "1",
"windmill-client": "*",
"date-fns": "^2.30.0",
"uuid": "^9.0.0"
}
}`,
// go: `module mymod
// go 1.25
// require (
// rsc.io/quote v1.5.2
// github.com/gorilla/mux v1.8.1
// github.com/lib/pq v1.10.9
// github.com/joho/godotenv v1.5.1
// github.com/sirupsen/logrus v1.9.3
// github.com/stretchr/testify v1.8.4
// )
// `,
php: `{
"require": {
"guzzlehttp/guzzle": "^7.8",
"monolog/monolog": "^3.5",
"vlucas/phpdotenv": "^5.6",
"symfony/console": "^6.4"
}
}`
}
let workspaceDependencies: {
content: string
language: ScriptLang
description: string
} = $state({
content: '',
language: 'python3',
description: ''
})
let valid = $state(true)
let drawer: Drawer | undefined = $state()
let edit = $state(false)
let initialName: string | undefined = $state(undefined)
let initialLanguage: ScriptLang = $state('python3')
let existingWorkspaceDefaults: Record<string, boolean> = $state({})
let workspaceDefaultIds: Record<string, number> = $state({})
let can_write = $state(true)
let editor: SimpleEditor | undefined = $state(undefined)
let showWarning = $state(false)
let currentImportedPath: string | null = $state(null)
const MAX_WORKSPACE_DEPENDENCIES_LENGTH = 50000
$effect(() => {
valid = workspaceDependencies.content.length <= MAX_WORKSPACE_DEPENDENCIES_LENGTH
})
// Track when editor is ready
let editorReady = $state(false)
// Calculate when deploy button should be disabled
let isDisabled = $derived(!can_write || !valid || (workspaceDependenciesType === 'named' && workspaceDependenciesName.trim() === ''))
$effect(() => {
if (editor && !editorReady) {
editor.setCode(workspaceDependencies.content)
editorReady = true
}
})
// Handle editor content changes
function handleEditorChange(event: { code: string }) {
// SimpleEditor dispatches change event with { code: string }
workspaceDependencies.content = event.code
}
// Watch for language changes and update template
let initialLanguageSet = $state(false)
let previousLanguage = $state<ScriptLang | null>(null)
$effect(() => {
const lang = workspaceDependencies.language
// Only update template when language actually changes, not on every content edit
if (lang && LANGUAGE_TEMPLATES[lang] && !edit && previousLanguage !== lang) {
workspaceDependencies.content = LANGUAGE_TEMPLATES[lang]
previousLanguage = lang
if (!initialLanguageSet) {
initialLanguageSet = true
}
if (editor && editorReady) {
editor.setCode(workspaceDependencies.content)
}
}
})
export async function initNew(): Promise<void> {
workspaceDependencies = {
content: LANGUAGE_TEMPLATES.python3,
language: 'python3',
description: "",
}
edit = false
initialName = undefined
initialLanguage = 'python3'
workspaceDependenciesName = ''
workspaceDependenciesType = 'named' // Start with named by default
can_write = true
editorReady = false
initialLanguageSet = false
previousLanguage = null
// Load existing workspace defaults from API
await loadExistingWorkspaceDefaults()
drawer?.openDrawer()
}
export async function editWorkspaceDependencies(id: number, name: string | undefined, language: ScriptLang): Promise<void> {
edit = true
try {
// Call the get-latest endpoint to get actual content
const workspaceDeps = await WorkspaceDependenciesService.getLatestWorkspaceDependencies({
workspace: $workspaceStore!,
language,
name: name || undefined
})
can_write = true // TODO: Implement proper permissions
if (workspaceDeps) {
workspaceDependencies = {
content: workspaceDeps.content,
language: workspaceDeps.language,
description: workspaceDeps.description || `${name || 'Default'} requirements for ${language}`
}
} else {
sendUserToast('Enforced dependencies not found', true)
return
}
} catch (error) {
console.error('Error loading workspace dependencies:', error)
sendUserToast(`Failed to load enforced dependencies: ${error.message}`, true)
return
}
initialName = name
initialLanguage = language
workspaceDependenciesName = name || ''
workspaceDependenciesType = !name ? 'workspace' : 'named'
editorReady = false
initialLanguageSet = true // Don't override content when editing
previousLanguage = language // Set to current language to prevent template reset
drawer?.openDrawer()
}
async function updateWorkspaceDependencies(): Promise<void> {
try {
await WorkspaceDependenciesService.createWorkspaceDependencies({
workspace: $workspaceStore!,
requestBody: {
name: workspaceDependenciesType === 'workspace' ? undefined : workspaceDependenciesName,
content: workspaceDependencies.content,
language: workspaceDependencies.language as any,
workspace_id: $workspaceStore!,
description: workspaceDependencies.description,
}
})
const displayName = workspaceDependenciesType === 'workspace' ? `workspace default for ${workspaceDependencies.language}` : workspaceDependenciesName
sendUserToast(`Deployed enforced dependencies: ${displayName}`)
dispatch('create')
drawer?.closeDrawer()
} catch (error) {
console.error('Error updating workspace dependencies:', error)
sendUserToast(`Failed to update enforced dependencies: ${error.message}`, true)
} finally {
showWarning = false
}
}
async function handleDeployClick(): Promise<void> {
// For updates, check for dependents and show warning
if (edit) {
const existingPath = getWorkspaceDependenciesPath(initialName ?? null, initialLanguage)
if (existingPath === null) {
sendUserToast('Unsupported language for enforced dependencies path generation', true)
return
}
try {
// Check if there are any dependents
const dependents = await WorkspaceService.getDependents({
workspace: $workspaceStore!,
importedPath: existingPath
})
// Only show warning if there are actually dependents
if (dependents.length > 0) {
currentImportedPath = existingPath
showWarning = true
} else {
// No dependents, proceed directly
await updateWorkspaceDependencies()
}
} catch (error) {
console.error('Error checking dependents:', error)
// On error, proceed without warning
await updateWorkspaceDependencies()
}
} else {
// New workspace dependencies, no need to check for dependents
await updateWorkspaceDependencies()
}
}
function confirmDeploy(): void {
showWarning = false
updateWorkspaceDependencies()
}
function cancelDeploy(): void {
showWarning = false
}
</script>
<Drawer bind:this={drawer} size="1200px">
<DrawerContent
title={edit ? `Deploy ${getFullFilename(workspaceDependencies.language, workspaceDependenciesType === 'workspace' ? null : workspaceDependenciesName)}` : 'Add enforced dependencies'}
on:close={drawer?.closeDrawer}
>
<div class="flex flex-col gap-8">
{#if !can_write}
<Alert type="warning" title="Only read access">
You only have read access to this resource and cannot edit it
</Alert>
{/if}
{#if showWarning && currentImportedPath}
<DependenciesDeploymentWarning
importedPath={currentImportedPath}
title="Deployment Warning"
confirmText="Deploy Anyway"
cancelText="Cancel"
onConfirm={confirmDeploy}
onCancel={cancelDeploy}
/>
{/if}
<Section label="Enforced Dependencies Type">
<div class="flex flex-col gap-4">
{#if hasWorkspaceDefault(workspaceDependencies.language) && !edit}
<ToggleButtonGroup bind:selected={workspaceDependenciesType} disabled={!can_write}>
{#snippet children({ item })}
<ToggleButton
disabled={!can_write}
value="named"
label="Named Dependencies"
{item}
/>
{/snippet}
</ToggleButtonGroup>
<div class="flex items-center gap-2 p-3 bg-blue-50 border border-blue-200 rounded">
<FolderOpen size={16} class="text-blue-600" />
<span class="text-sm text-blue-800 flex-1">
Workspace default already exists for {workspaceDependencies.language}
</span>
<Button size="xs" color="blue" variant="border" on:click={goToWorkspaceDefault}>
Go to default
</Button>
</div>
{:else}
<ToggleButtonGroup bind:selected={workspaceDependenciesType} disabled={!can_write || edit}>
{#snippet children({ item })}
<ToggleButton
disabled={!can_write || edit}
value="named"
label="Named Dependencies"
{item}
/>
<ToggleButton
disabled={!can_write || edit}
value="workspace"
label="Workspace Default"
{item}
/>
{/snippet}
</ToggleButtonGroup>
{/if}
{#if workspaceDependenciesType === 'named'}
<input
type="text"
bind:value={workspaceDependenciesName}
placeholder="Enter dependencies name (e.g., 'data-science', 'web-api')"
disabled={!can_write}
class="input"
/>
{/if}
<div class="text-sm text-tertiary">
<FolderOpen size={16} class="inline mr-2" />
Default Enforced Dependencies are used when no specific Dependencies are referenced from runnables.
Named dependencies can be referenced by scripts using
<a
href="https://www.windmill.dev/docs/core_concepts/workspace_dependencies"
target="_blank"
rel="noopener noreferrer"
class="text-accent"
>
annotations
</a>.
</div>
</div>
</Section>
<Section label="Language">
<div class="flex flex-col gap-4">
<Select
bind:value={workspaceDependencies.language}
items={LANGUAGE_OPTIONS}
disabled={edit}
/>
<div class="text-sm text-tertiary">
<Code2 size={16} class="inline mr-2" />
{edit ? 'Language cannot be changed after creation.' : 'Select the programming language these dependencies are for. This will load a default template.'}
</div>
</div>
</Section>
<Section label="Description">
<input
type="text"
bind:value={workspaceDependencies.description}
placeholder="Brief description of these enforced dependencies (optional)"
disabled={!can_write}
class="input"
/>
<div class="text-sm text-tertiary mt-2">
Provide a brief description to help others understand the purpose of these enforced dependencies.
</div>
</Section>
<Section label="Dependencies File Content">
{#snippet header()}
<span class="text-sm text-tertiary mr-4 font-normal">
({workspaceDependencies.content.length}/{MAX_WORKSPACE_DEPENDENCIES_LENGTH} characters)
</span>
{/snippet}
<div class="border rounded mb-4 w-full">
{#await import('$lib/components/SimpleEditor.svelte')}
<Loader2 class="animate-spin" />
{:then Module}
<Module.default
bind:this={editor}
autoHeight
lang="markdown"
code={workspaceDependencies.content}
on:change={(e) => handleEditorChange(e.detail)}
fixedOverflowWidgets={false}
disabled={!can_write}
/>
{/await}
</div>
</Section>
</div>
{#snippet actions()}
<Button
on:click={handleDeployClick}
disabled={isDisabled}
startIcon={{ icon: Rocket }}
color="dark"
size="sm"
>
Deploy
</Button>
{/snippet}
</DrawerContent>
</Drawer>
frontend/src/lib/components/WorkspaceDependenciesViewer.svelte
+100
View File
@@ -0,0 +1,100 @@
<script lang="ts">
import Drawer from './common/drawer/Drawer.svelte'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import HighlightCode from './HighlightCode.svelte'
import { Button } from './common'
import WorkspaceDependenciesEditor from './WorkspaceDependenciesEditor.svelte'
import type { ScriptLang } from '$lib/gen'
import { Code2, Edit, FileText } from 'lucide-svelte'
import { canWrite } from '$lib/utils'
import { userStore } from '$lib/stores'
// Component state
let drawer: Drawer | undefined = $state()
let workspaceDependenciesEditor: WorkspaceDependenciesEditor | undefined = $state()
// Content state
let viewContent: string = $state('')
let viewLanguage: ScriptLang = $state('python3')
let viewPath: string = $state('')
let viewDescription: string = $state('')
let viewDepsId: number | undefined = $state(undefined)
let viewDepsName: string | undefined = $state(undefined)
let canWriteDeps: boolean = $state(false)
// Export methods for external control
export function openViewer(path: string, content?: string, language?: ScriptLang, description?: string, depsId?: number, depsName?: string) {
viewPath = path
viewContent = content || ''
viewLanguage = language || 'python3'
viewDescription = description || ''
viewDepsId = depsId
viewDepsName = depsName
// TODO: Replace with actual API call to check permissions
// For now, mock the permission check
canWriteDeps = canWrite(path, {}, $userStore)
drawer?.openDrawer()
}
export function closeViewer() {
drawer?.closeDrawer()
}
function editWorkspaceDependencies() {
if (viewDepsId) {
workspaceDependenciesEditor?.editWorkspaceDependencies(viewDepsId, viewDepsName, viewLanguage)
}
closeViewer()
}
function onWorkspaceDependenciesUpdated() {
// TODO: Reload workspace dependencies data
console.log('Workspace dependencies updated, should reload data')
}
</script>
<WorkspaceDependenciesEditor bind:this={workspaceDependenciesEditor} on:create={onWorkspaceDependenciesUpdated} />
<Drawer bind:this={drawer} size="900px">
<DrawerContent title="Workspace Dependencies from {viewPath}" on:close={closeViewer}>
{#snippet actions()}
<div class="flex items-center gap-4">
<div class="flex items-center gap-2">
<Code2 size={16} class="text-secondary" />
<span class="text-sm font-mono text-secondary">{viewLanguage}</span>
</div>
{#if canWriteDeps}
<Button
size="xs"
variant="border"
color="light"
startIcon={{ icon: Edit }}
on:click={editWorkspaceDependencies}
>
Edit
</Button>
{/if}
</div>
{/snippet}
<div class="space-y-4">
{#if viewDescription}
<div class="bg-surface-secondary rounded-md p-3">
<p class="text-sm text-secondary">{viewDescription}</p>
</div>
{/if}
{#if viewContent}
<HighlightCode language={viewLanguage} code={viewContent} />
{:else}
<div class="text-center text-secondary py-8">
<FileText size={48} class="mx-auto mb-4 opacity-50" />
<p>No workspace dependencies found for this path</p>
<p class="text-xs mt-2">Create workspace dependencies to define dependencies for scripts in this directory</p>
</div>
{/if}
</div>
</DrawerContent>
</Drawer>
frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte
+393
View File
@@ -0,0 +1,393 @@
<script lang="ts">
import { Button, Skeleton } from '$lib/components/common'
import ListFilters from '$lib/components/home/ListFilters.svelte'
import SearchItems from '$lib/components/SearchItems.svelte'
import Cell from '$lib/components/table/Cell.svelte'
import DataTable from '$lib/components/table/DataTable.svelte'
import Head from '$lib/components/table/Head.svelte'
import Row from '$lib/components/table/Row.svelte'
import WorkspaceDependenciesEditor from '$lib/components/WorkspaceDependenciesEditor.svelte'
import DependenciesDeploymentWarning from '$lib/components/DependenciesDeploymentWarning.svelte'
import Drawer from '$lib/components/common/drawer/Drawer.svelte'
import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte'
import HighlightCode from '$lib/components/HighlightCode.svelte'
import { workspaceStore, userStore } from '$lib/stores'
import { Plus, FileText, Search, Code2, Edit, Eye } from 'lucide-svelte'
import { WorkspaceDependenciesService, WorkspaceService } from '$lib/gen'
import type { WorkspaceDependencies, ScriptLang } from '$lib/gen'
import { untrack } from 'svelte'
import { sendUserToast } from '$lib/toast'
import TimeAgo from '$lib/components/TimeAgo.svelte'
import Description from '$lib/components/Description.svelte'
let filter = $state('')
let workspaceDependencies: WorkspaceDependencies[] | undefined = $state()
let filteredItems: (WorkspaceDependencies & { marked?: string })[] | undefined = $state()
let workspaceDependenciesEditor: WorkspaceDependenciesEditor | undefined = $state()
// View modal state
let viewDrawer: Drawer | undefined = $state()
let viewContent: string = $state('')
let viewLanguage: ScriptLang = $state('python3')
let viewPath: string = $state('')
// Dependency warning state
let showDependencyWarning = $state(false)
let pendingAction: (() => Promise<void>) | null = $state(null)
let currentImportedPath: string | null = $state(null)
let warningTitle = $state('')
let warningConfirmText = $state('')
let languages = $derived(
Array.from(new Set(filteredItems?.map((x) => x.language).filter(Boolean) ?? [])).sort()
)
let languageFilter: string | undefined = $state(undefined)
$effect(() => {
if ($workspaceStore) {
languageFilter = undefined
}
})
let preFilteredItems = $derived(
languageFilter == undefined
? workspaceDependencies
: workspaceDependencies?.filter((x) => x.language === languageFilter)
)
// Load workspace dependencies using actual API
async function loadWorkspaceDependencies(): Promise<void> {
if (!$workspaceStore) return
try {
workspaceDependencies = await WorkspaceDependenciesService.listWorkspaceDependencies({
workspace: $workspaceStore
})
} catch (error) {
console.error('Failed to load workspace dependencies:', error)
sendUserToast('Failed to load enforced dependencies', true)
}
}
$effect(() => {
if ($workspaceStore && $userStore) {
untrack(() => {
loadWorkspaceDependencies()
})
}
})
async function createNewWorkspaceDependencies() {
await workspaceDependenciesEditor?.initNew()
}
function editWorkspaceDependencies(deps: WorkspaceDependencies) {
workspaceDependenciesEditor?.editWorkspaceDependencies(deps.id, deps.name, deps.language)
}
function viewWorkspaceDependencies(deps: WorkspaceDependencies) {
viewPath = deps.name || `Workspace Default (${deps.language})`
viewContent = deps.content
viewLanguage = deps.language
viewDrawer?.openDrawer()
}
// Archive workspace dependencies
async function archiveWorkspaceDependencies(deps: WorkspaceDependencies): Promise<void> {
const importedPath = workspaceDependenciesEditor?.getWorkspaceDependenciesPath(deps.name ?? null, deps.language)
if (!importedPath) {
sendUserToast('Unable to determine enforced dependencies path', true)
return
}
currentImportedPath = importedPath
warningTitle = `Archive Warning`
warningConfirmText = 'Archive Anyway'
pendingAction = () => executeArchive(deps)
showDependencyWarning = true
}
async function executeArchive(deps: WorkspaceDependencies): Promise<void> {
try {
await WorkspaceDependenciesService.archiveWorkspaceDependencies({
workspace: $workspaceStore!,
language: deps.language as any,
name: deps.name
})
sendUserToast(`Archived enforced dependencies: ${workspaceDependenciesEditor?.getDisplayName(deps)}`)
loadWorkspaceDependencies() // Reload the list
} catch (error) {
console.error('Error archiving workspace dependencies:', error)
sendUserToast(`Failed to archive enforced dependencies: ${error.message}`, true)
}
}
// Delete workspace dependencies
async function deleteWorkspaceDependencies(deps: WorkspaceDependencies): Promise<void> {
const importedPath = workspaceDependenciesEditor?.getWorkspaceDependenciesPath(deps.name ?? null, deps.language)
if (!importedPath) {
sendUserToast('Unable to determine enforced dependencies path', true)
return
}
currentImportedPath = importedPath
warningTitle = `Delete Warning`
warningConfirmText = 'Delete Anyway'
pendingAction = () => executeDelete(deps)
showDependencyWarning = true
}
async function executeDelete(deps: WorkspaceDependencies): Promise<void> {
try {
await WorkspaceDependenciesService.deleteWorkspaceDependencies({
workspace: $workspaceStore!,
language: deps.language as any,
name: deps.name
})
sendUserToast(`Deleted enforced dependencies: ${workspaceDependenciesEditor?.getDisplayName(deps)}`)
loadWorkspaceDependencies() // Reload the list
} catch (error) {
console.error('Error deleting workspace dependencies:', error)
sendUserToast(`Failed to delete enforced dependencies: ${error.message}`, true)
}
}
async function viewReferencedFrom(deps: WorkspaceDependencies): Promise<void> {
try {
const path = workspaceDependenciesEditor?.getWorkspaceDependenciesPath(deps.name ?? null, deps.language)
if (!path) {
sendUserToast('Unable to determine enforced dependencies path', true)
return
}
const dependents = await WorkspaceService.getDependents({
workspace: $workspaceStore!,
importedPath: path
})
if (dependents.length === 0) {
sendUserToast('No dependent runnables found for these enforced dependencies')
} else {
// Show dependents in a modal or navigate to a detailed view
console.log('Dependents:', dependents)
sendUserToast(`Found ${dependents.length} dependent runnable${dependents.length !== 1 ? 's' : ''}`)
}
} catch (error) {
console.error('Error fetching dependent runnables:', error)
sendUserToast('Failed to fetch dependent runnables', true)
}
}
async function handleWarningConfirm(): Promise<void> {
if (pendingAction) {
showDependencyWarning = false
await pendingAction()
pendingAction = null
currentImportedPath = null
}
}
function handleWarningCancel(): void {
showDependencyWarning = false
pendingAction = null
currentImportedPath = null
}
function getLanguageForHighlighting(language: ScriptLang): ScriptLang | 'json' | undefined{
// Map our requirement languages to syntax highlighting languages
switch (language) {
case 'python3':
return 'python3'
case 'bun':
return 'json'
case 'go':
return 'go'
case 'php':
return 'json'
}
}
</script>
<WorkspaceDependenciesEditor bind:this={workspaceDependenciesEditor} on:create={loadWorkspaceDependencies} />
<SearchItems
{filter}
items={preFilteredItems}
bind:filteredItems
f={(x) => (x.name || 'Default') + ' ' + (x.language || '') + ' ' + (x.content || '')}
/>
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class="text-sm font-semibold text-emphasis">Enforced Dependencies</div>
<Description link="https://www.windmill.dev/docs/">
Enforced Dependencies define dependency specifications for scripts by language. Unnamed dependencies serve as workspace defaults, while named dependencies can be referenced by scripts using #raw_reqs annotations.
</Description>
</div>
<div class="flex flex-row justify-end">
<Button size="md" startIcon={{ icon: Plus }} on:click={createNewWorkspaceDependencies}>
New&nbsp;enforced&nbsp;dependencies
</Button>
</div>
</div>
<div class="pt-2">
<div class="relative text-tertiary">
<input
placeholder="Search enforced dependencies by name, language, or content..."
bind:value={filter}
class="bg-surface !h-10 !px-4 !pr-10 !rounded-lg text-sm focus:outline-none w-full"
/>
<button aria-label="Search" type="submit" class="absolute right-0 top-0 mt-3 mr-4">
<Search class="h-4 w-4" />
</button>
</div>
</div>
<div class="min-h-[56px]">
<ListFilters bind:selectedFilter={languageFilter} filters={languages} />
</div>
<div class="relative overflow-x-auto pb-40 pr-4">
{#if !filteredItems}
<Skeleton layout={[0.5, [2], 1]} />
{#each new Array(3) as _}
<Skeleton layout={[[3.5], 0.5]} />
{/each}
{:else if filteredItems.length == 0}
<div class="flex flex-col items-center justify-center h-full py-12">
<FileText size={48} class="text-secondary mb-4" />
<div class="text-md font-medium">No enforced dependencies found</div>
<div class="text-sm text-secondary mb-4">
Try changing the filters or creating new enforced dependencies
</div>
<Button startIcon={{ icon: Plus }} on:click={createNewWorkspaceDependencies}>
Create your first enforced dependencies
</Button>
</div>
{:else}
<DataTable size="xs">
<Head>
<tr>
<Cell head first>Name</Cell>
<Cell head>Language</Cell>
<Cell head>Description</Cell>
<Cell head>Type</Cell>
<Cell head>Edited</Cell>
<Cell head last>Actions</Cell>
</tr>
</Head>
<tbody class="divide-y">
{#each filteredItems as deps}
<Row>
<Cell first>
<div class="flex items-center gap-2">
<FileText size={16} class="text-secondary" />
<div class="flex flex-col">
<button
class="break-all hover:text-primary cursor-pointer font-medium text-left"
onclick={() => editWorkspaceDependencies(deps)}
>
{#if deps.marked}
{@html deps.marked}
{:else}
{workspaceDependenciesEditor?.getDisplayName(deps) || (deps.name || `Default (${deps.language})`)}
{/if}
</button>
<span class="text-xs text-tertiary font-mono">
{workspaceDependenciesEditor?.getFullFilename(deps.language, deps.name ?? null)} • {deps.language}
</span>
</div>
</div>
</Cell>
<Cell>
<div class="flex items-center gap-1">
<Code2 size={14} class="text-secondary" />
<span class="text-xs font-mono text-secondary">
{deps.language || 'python3'}
</span>
</div>
</Cell>
<Cell>
<span class="text-xs text-tertiary" title={deps.description}>
{deps.description || '-'}
</span>
</Cell>
<Cell>
<span class="text-xs px-1.5 py-0.5 rounded bg-opacity-50 font-medium"
class:bg-blue-100="{deps.name === null}"
class:text-blue-700="{deps.name === null}"
class:bg-gray-100="{deps.name !== null}"
class:text-gray-600="{deps.name !== null}"
>
{deps.name === null ? 'Default' : 'Named'}
</span>
</Cell>
<Cell>
<span class="text-2xs text-secondary">
<TimeAgo date={deps.created_at || ''} />
</span>
</Cell>
<Cell last>
<div class="flex gap-1 flex-wrap">
<Button size="xs" variant="border" color="light" startIcon={{ icon: Eye }} on:click={() => viewWorkspaceDependencies(deps)}>
View
</Button>
<Button size="xs" variant="border" color="light" startIcon={{ icon: Edit }} on:click={() => editWorkspaceDependencies(deps)}>
Edit
</Button>
<!-- Placeholder buttons -->
<Button size="xs" variant="border" color="gray" on:click={() => archiveWorkspaceDependencies(deps)} title="Archive">
Archive
</Button>
<Button size="xs" variant="border" color="red" on:click={() => deleteWorkspaceDependencies(deps)} title="Delete">
Delete
</Button>
<Button size="xs" variant="border" color="gray" on:click={() => viewReferencedFrom(deps)} title="Referenced From">
Refs
</Button>
</div>
</Cell>
</Row>
{/each}
</tbody>
</DataTable>
{/if}
</div>
<Drawer bind:this={viewDrawer} size="900px">
<DrawerContent title="View Requirement - {viewPath}" on:close={viewDrawer?.closeDrawer}>
{#snippet actions()}
<div class="flex items-center gap-2">
<Code2 size={16} class="text-secondary" />
<span class="text-sm font-mono text-secondary">{viewLanguage}</span>
</div>
{/snippet}
<div class="space-y-4">
{#if viewContent}
<HighlightCode language={getLanguageForHighlighting(viewLanguage)} code={viewContent} />
{:else}
<div class="text-center text-secondary py-8">
<FileText size={48} class="mx-auto mb-4 opacity-50" />
<p>No content available for this requirement</p>
</div>
{/if}
</div>
</DrawerContent>
</Drawer>
{#if showDependencyWarning && currentImportedPath}
<DependenciesDeploymentWarning
importedPath={currentImportedPath}
title={warningTitle}
confirmText={warningConfirmText}
cancelText="Cancel"
onConfirm={handleWarningConfirm}
onCancel={handleWarningCancel}
/>
{/if}
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+12 -1
View File
@@ -59,6 +59,7 @@
import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte'
import TextInput from '$lib/components/text_input/TextInput.svelte'
import CollapseLink from '$lib/components/CollapseLink.svelte'
import WorkspaceDependenciesSettings from '$lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte'
let slackInitialPath: string = $state('')
let slackScriptPath: string = $state('')
@@ -134,7 +135,8 @@
| 'windmill_lfs'
| 'git_sync'
| 'default_app'
| 'encryption') ?? 'users'
| 'encryption'
| 'dependencies') ?? 'users'
)
let usingOpenaiClientCredentialsOauth = $state(false)
@@ -692,6 +694,13 @@
aiDescription="General workspace settings"
label="General"
/>
<Tab
small
value="dependencies"
aiId="workspace-settings-dependencies"
aiDescription="Workspace dependencies settings"
label="Dependencies"
/>
</Tabs>
</div>
{#if !loadedSettings}
@@ -1124,6 +1133,8 @@
<div class="text-sm text-secondary">Loading workspace...</div>
</div>
{/if}
{:else if tab == 'dependencies'}
<WorkspaceDependenciesSettings />
{:else if tab == 'default_app'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">