diff --git a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json index 152ae38fdc..60d2cfdc01 100644 --- a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json +++ b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json @@ -62,7 +62,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json index e8b39caef7..9462f9b48b 100644 --- a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json +++ b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json @@ -43,7 +43,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json b/backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json index b2d146291b..eafb17af4e 100644 --- a/backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json +++ b/backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json @@ -32,7 +32,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json index 561e202d1d..5f59450dc3 100644 --- a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json +++ b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json @@ -71,7 +71,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json b/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json index 91512257cd..e16cd7db75 100644 --- a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json +++ b/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json @@ -138,7 +138,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json b/backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json new file mode 100644 index 0000000000..9709a354cf --- /dev/null +++ b/backend/.sqlx/query-4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c" +} diff --git a/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json b/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json index 1a30ad011f..9560d17efa 100644 --- a/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json +++ b/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json @@ -62,7 +62,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json index 7933311a4c..6f08d98113 100644 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json @@ -35,7 +35,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json b/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json index 17dc7cfd2e..5f6611dae6 100644 --- a/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json +++ b/backend/.sqlx/query-a6608d47b96d851eb7b04d2e4b472889ff257db0f0e3e9252adda2e4ef2039d6.json @@ -148,7 +148,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json index a3cb5a82b9..279d2f25fa 100644 --- a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -37,7 +37,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json index 16fea8720d..c2c6738167 100644 --- a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json +++ b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json @@ -42,7 +42,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json index c8cbb8e10d..072bc63b66 100644 --- a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json +++ b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json @@ -66,7 +66,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-acfe583fe17604ba72ba4800b62a72de0a9de0d58ef8c28dd709adf3be021597.json b/backend/.sqlx/query-acfe583fe17604ba72ba4800b62a72de0a9de0d58ef8c28dd709adf3be021597.json index 7d7eef1cab..fbd4571adc 100644 --- a/backend/.sqlx/query-acfe583fe17604ba72ba4800b62a72de0a9de0d58ef8c28dd709adf3be021597.json +++ b/backend/.sqlx/query-acfe583fe17604ba72ba4800b62a72de0a9de0d58ef8c28dd709adf3be021597.json @@ -71,7 +71,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json b/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json new file mode 100644 index 0000000000..fb865ca894 --- /dev/null +++ b/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json @@ -0,0 +1,248 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n cj.id AS \"id!\",\n cj.workspace_id AS \"workspace_id!\",\n cj.parent_job,\n cj.created_by AS \"created_by!\",\n cj.duration_ms AS \"duration_ms!\",\n cj.success AS \"success!\",\n cj.script_hash AS \"script_hash!: Option\",\n cj.script_path,\n cj.args AS \"args: sqlx::types::Json>>\",\n cj.result AS \"result: sqlx::types::Json>\",\n cj.deleted AS \"deleted!\",\n cj.canceled AS \"canceled!\",\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind AS \"job_kind!: JobKind\",\n cj.schedule_path,\n cj.permissioned_as AS \"permissioned_as!\",\n cj.is_flow_step AS \"is_flow_step!\",\n cj.language AS \"language: ScriptLang\",\n cj.is_skipped AS \"is_skipped!\",\n cj.email AS \"email!\",\n cj.visible_to_owner AS \"visible_to_owner!\",\n cj.mem_peak,\n cj.tag AS \"tag!\",\n cj.created_at AS \"created_at!\",\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset AS \"log_offset?\",\n job_logs.log_file_index\n\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n ORDER BY cj.created_at ASC LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "script_hash!: Option", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "deleted!", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "canceled!", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "job_kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "schedule_path", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "is_flow_step!", + "type_info": "Bool" + }, + { + "ordinal": 18, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 19, + "name": "is_skipped!", + "type_info": "Bool" + }, + { + "ordinal": 20, + "name": "email!", + "type_info": "Varchar" + }, + { + "ordinal": 21, + "name": "visible_to_owner!", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 23, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 24, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 25, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 26, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 27, + "name": "log_offset?", + "type_info": "Int4" + }, + { + "ordinal": 28, + "name": "log_file_index", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Timestamptz", + "Int8" + ] + }, + "nullable": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + "hash": "f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7" +} diff --git a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json index 4e911403de..b100feb47d 100644 --- a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json +++ b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json @@ -66,7 +66,8 @@ "oracledb", "nu", "java", - "duckdb" + "duckdb", + "ruby" ] } } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1179183971..b4d024cd02 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -13854,6 +13854,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8" +[[package]] +name = "tree-sitter-ruby" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "triomphe" version = "0.1.14" @@ -15214,6 +15224,20 @@ dependencies = [ "windmill-parser", ] +[[package]] +name = "windmill-parser-ruby" +version = "1.523.0" +dependencies = [ + "anyhow", + "lazy_static", + "regex", + "serde_json", + "tree-sitter", + "tree-sitter-ruby", + "wasm-bindgen", + "windmill-parser", +] + [[package]] name = "windmill-parser-rust" version = "1.526.1" @@ -15281,6 +15305,7 @@ dependencies = [ "windmill-parser-nu", "windmill-parser-php", "windmill-parser-py", + "windmill-parser-ruby", "windmill-parser-rust", "windmill-parser-sql", "windmill-parser-ts", @@ -15427,6 +15452,7 @@ dependencies = [ "windmill-parser-php", "windmill-parser-py", "windmill-parser-py-imports", + "windmill-parser-ruby", "windmill-parser-rust", "windmill-parser-sql", "windmill-parser-ts", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index e6da910ebf..8a5cc8ec0d 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -24,6 +24,7 @@ members = [ "./parsers/windmill-parser-csharp", "./parsers/windmill-parser-nu", "./parsers/windmill-parser-java", + "./parsers/windmill-parser-ruby", "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", @@ -48,7 +49,7 @@ incremental = true lto = "thin" [features] -default = [] +default = ["ruby"] private = ["windmill-api/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-queue/private", "windmill-worker/private"] agent_worker_server = ["windmill-api/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"] @@ -95,7 +96,8 @@ php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] -all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +ruby = ["windmill-worker/ruby"] +all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java", "ruby"] # For windows we have another set of languages enabled # NOTE: DuckDB is ignored because of compilation problems all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] @@ -145,7 +147,6 @@ systemstat.workspace = true size.workspace = true strum.workspace = true - [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } @@ -180,6 +181,7 @@ windmill-parser-rust = { path = "./parsers/windmill-parser-rust" } windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" } windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" } windmill-parser-java = { path = "./parsers/windmill-parser-java" } +windmill-parser-ruby = { path = "./parsers/windmill-parser-ruby" } windmill-parser-nu = { path = "./parsers/windmill-parser-nu" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } @@ -406,6 +408,7 @@ tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] } tree-sitter = { version = "0.23.0", features = [] } tree-sitter-c-sharp = "0.23.0" tree-sitter-java = "0.23.0" +tree-sitter-ruby = "0.23.0" oracle = { version = "0.6.3", features = ["chrono"] } rumqttc = { version = "0.24.0", features = ["use-native-tls"]} strum = { version = "0.27", features = ["derive"] } diff --git a/backend/migrations/20250721160151_add_ruby.down.sql b/backend/migrations/20250721160151_add_ruby.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250721160151_add_ruby.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250721160151_add_ruby.up.sql b/backend/migrations/20250721160151_add_ruby.up.sql new file mode 100644 index 0000000000..3cf454718c --- /dev/null +++ b/backend/migrations/20250721160151_add_ruby.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'ruby'; +UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["ruby"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java", "duckdb"]}'::jsonb AND NOT config->'worker_tags' @> '"ruby"'::jsonb; diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 52983872c6..89ebcc659e 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -128,7 +128,7 @@ pub fn parse_python_signature( && default.is_some() && default != Some(json!(FUNCTION_CALL)) { - typ = json_to_typ(default.as_ref().unwrap()); + typ = json_to_typ(default.as_ref().unwrap(), false); } // if the type is still a list of unknowns after checking the default, we set it to a list of strings to not break past behavior diff --git a/backend/parsers/windmill-parser-ruby/Cargo.toml b/backend/parsers/windmill-parser-ruby/Cargo.toml new file mode 100644 index 0000000000..779b0116a0 --- /dev/null +++ b/backend/parsers/windmill-parser-ruby/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "windmill-parser-ruby" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_ruby" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +tree-sitter.workspace = true +tree-sitter-ruby.workspace = true +anyhow.workspace = true +wasm-bindgen.workspace = true +serde_json.workspace = true +regex.workspace = true +lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-ruby/src/lib.rs b/backend/parsers/windmill-parser-ruby/src/lib.rs new file mode 100644 index 0000000000..c736cef86a --- /dev/null +++ b/backend/parsers/windmill-parser-ruby/src/lib.rs @@ -0,0 +1,410 @@ +#![cfg_attr(target_arch = "wasm32", feature(c_variadic))] + +#[cfg(target_arch = "wasm32")] +pub mod wasm_libc; + +use anyhow::anyhow; +use anyhow::bail; +use regex::Regex; +use serde_json::Value; +use tree_sitter::Node; +use tree_sitter::Range; +use windmill_parser::json_to_typ; +use windmill_parser::Arg; +use windmill_parser::MainArgSignature; + +pub fn parse_ruby_sig_meta(code: &str) -> anyhow::Result { + let mut parser = tree_sitter::Parser::new(); + let language = tree_sitter_ruby::LANGUAGE; + parser + .set_language(&language.into()) + .map_err(|e| anyhow!("Error setting Ruby as language: {e}"))?; + + // Parse code + let tree = parser + .parse(code, None) + .ok_or(anyhow!("Failed to parse code"))?; + let root_node = tree.root_node(); + + root_node.clone().to_string(); + // Traverse the AST to find the Main method signature + let args = find_main_signature(root_node, code)?; + let no_main_func = Some(args.is_none()); + + let main_sig = MainArgSignature { + star_args: false, + star_kwargs: false, + args: args.unwrap_or_default(), + has_preprocessor: None, + no_main_func, + }; + + Ok(main_sig) +} + +pub fn parse_ruby_signature(code: &str) -> anyhow::Result { + Ok(parse_ruby_sig_meta(code)?) +} + +pub fn parse_ruby_requirements(code: &str) -> anyhow::Result { + let mut parser = tree_sitter::Parser::new(); + let language = tree_sitter_ruby::LANGUAGE; + parser + .set_language(&language.into()) + .map_err(|e| anyhow!("Error setting Ruby as language: {e}"))?; + + // Parse code + let tree = parser + .parse(code, None) + .ok_or(anyhow!("Failed to parse code"))?; + let root_node = tree.root_node(); + + root_node.clone().to_string(); + let mut cursor = root_node.walk(); + 'top_level: for x in root_node.children(&mut cursor) { + if x.kind() == "call" { + for (i, n) in x.children(&mut x.walk()).enumerate() { + if i == 0 + && n.kind() == "identifier" + && !n + .utf8_text(code.as_bytes()) + .map(|ident| ident == "gemfile") + .unwrap_or_default() + { + continue 'top_level; + } else if n.kind() == "do_block" { + let req = n.utf8_text(code.as_bytes())?.to_owned(); + + lazy_static::lazy_static! { + static ref WINDMILL_RE: Regex = Regex::new(r"(?m)^\s*require\s*'windmill/inline'").unwrap(); + } + + if WINDMILL_RE.find(&code).is_none() { + return Err(anyhow!( + "`require 'windmill/inline'` is not detected - please add `require 'windmill/inline'` in order to use inline gemfile. + Your Gemfile syntax will be compatible with bundler/inline." + ) + .into()); + } + + return req + // gemfile do_block comes with 'do' and 'end' + // we want to omit these by taking slice + .get(2..(req.len() - 3)) + .map(str::to_owned) + .ok_or(anyhow!("Invalid gemfile do block")); + } + } + } + } + Ok(String::new()) +} + +// Function to find the Main method's signature +fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> anyhow::Result>> { + let mut cursor = root_node.walk(); + 'top_level: for x in root_node.children(&mut cursor) { + if x.kind() == "method" { + let mut args = vec![]; + for (i, m) in x.children(&mut x.walk()).skip(1).enumerate() { + if i == 0 + && m.kind() == "identifier" + && !m + .utf8_text(code.as_bytes()) + .map(|ident| ident == "main") + .unwrap_or_default() + { + continue 'top_level; + } else if m.kind() == "method_parameters" { + for a in m.children(&mut m.walk()) { + if a.kind() == "identifier" { + a.utf8_text(code.as_bytes()).inspect(|name| { + args.push(Arg { name: (*name).to_owned(), ..Default::default() }) + })?; + } + if a.kind() == "optional_parameter" { + let mut walk = a.walk(); + let mut it = a.children(&mut walk).into_iter(); + match (it.next().and_then(|n| n.utf8_text(code.as_bytes()).ok()), { + // Skip `=` + it.next(); + it.next().map(|n| n.range()) + }) { + (Some(ident), Some(Range { start_byte, end_byte, .. })) => { + let unparsed = + &code[start_byte..end_byte].replace("nil", "null"); + let default: Value = serde_json::from_str(unparsed).map_err( + |e| + anyhow!("Cannot convert default value to json\n\tvalue: {unparsed}\n\terror: {e}{}", + if e.to_string().contains("key must be a string") { + "\n\nNOTE: If you are trying to declare default hash, use following syntax:\n { \"\": }" + } else { + "" + } + ))?; + args.push(Arg { + name: ident.to_owned(), + typ: json_to_typ(&default, true), + default: Some(default), + has_default: true, + ..Default::default() + }); + } + _ => { + let Range { start_byte, end_byte, .. } = a.range(); + bail!( + "Cannot parse optional parameter: {}", + &code.get(start_byte..end_byte).unwrap_or("CANNOT DISPLAY") + ) + } + } + } + let kind = a.kind(); + if matches!( + kind, + "keyword_parameter" | "splat_parameter" | "hash_splat_parameter" + ) { + let Range { start_byte, end_byte, .. } = a.range(); + bail!( + " - {}\n{}s are not supported", + &code.get(start_byte..end_byte).unwrap_or("CANNOT DISPLAY"), + kind.replace("_", " ") + ); + } + } + } + } + return Ok(Some(args)); + } + } + Ok(None) +} + +#[cfg(test)] +mod test { + + use serde_json::json; + use windmill_parser::{ObjectProperty, ObjectType, Typ}; + + use super::parse_ruby_sig_meta as parse; + use super::*; + #[test] + fn test_parse_ruby_no_main() { + let code = r#" +def not_main end +def private_fn end +"#; + let sig = parse(code).unwrap(); + + assert_eq!( + sig, + MainArgSignature { no_main_func: Some(true), ..Default::default() } + ); + } + #[test] + fn test_parse_ruby_no_args() { + let code = r#" +def main +end +"#; + let sig = parse(code).unwrap(); + + assert_eq!( + sig, + MainArgSignature { no_main_func: Some(false), ..Default::default() } + ); + } + #[test] + fn test_parse_ruby_positional_args() { + let sig = { + let code = r#"def main(a, b, c) end"#; + parse(code).unwrap() + }; + let sig2 = { + let code = r#" + def main a, b, c + end"#; + parse(code).unwrap() + }; + + assert_eq!( + sig, + MainArgSignature { + args: vec![ + Arg { name: "a".into(), ..Default::default() }, + Arg { name: "b".into(), ..Default::default() }, + Arg { name: "c".into(), ..Default::default() } + ], + no_main_func: Some(false), + ..Default::default() + } + ); + assert_eq!(sig2, sig); + } + #[test] + fn test_parse_ruby_lists() { + let sig = { + let code = r#"def main(a = [ 1, 2, 3 ], b = [ 1, false, nil, "test"]) end"#; + parse(code).unwrap() + }; + assert_eq!( + sig, + MainArgSignature { + args: vec![ + Arg { + name: "a".into(), + has_default: true, + typ: Typ::List(Box::new(Typ::Int)), + default: Some(json!([1, 2, 3])), + ..Default::default() + }, + Arg { + name: "b".into(), + has_default: true, + typ: Typ::List(Box::new(Typ::Unknown)), + default: Some(json!([1, false, null, "test"])), + ..Default::default() + }, + ], + no_main_func: Some(false), + ..Default::default() + } + ); + } + #[test] + fn test_parse_ruby_hashes() { + let sig = { + let code = r#"def main(a = { "1": 4, "2": [ 1, 2, 3] }) end"#; + parse(code).unwrap() + }; + assert_eq!( + sig, + MainArgSignature { + args: vec![Arg { + name: "a".into(), + has_default: true, + typ: Typ::Object(ObjectType::new( + None, + Some(vec![ + ObjectProperty { key: "1".into(), typ: Box::new(Typ::Int) }, + ObjectProperty { + key: "2".into(), + typ: Box::new(Typ::List(Box::new(Typ::Int))) + } + ],) + )), + default: Some(json!({"1": 4, "2": [ 1, 2, 3 ]})), + ..Default::default() + },], + no_main_func: Some(false), + ..Default::default() + } + ); + } + #[test] + fn test_parse_ruby_default_args() { + let sig = { + let code = + r#"def main(a = 10, b = "hey", c = false, d = [ 1, 2, 3 ], e = { "a": 43 }) end"#; + parse(code).unwrap() + }; + + assert_eq!( + sig, + MainArgSignature { + args: vec![ + Arg { + name: "a".into(), + has_default: true, + typ: Typ::Int, + default: Some(json!(10)), + ..Default::default() + }, + Arg { + name: "b".into(), + has_default: true, + typ: Typ::Str(None), + default: Some(json!("hey")), + ..Default::default() + }, + Arg { + name: "c".into(), + has_default: true, + typ: Typ::Bool, + default: Some(json!(false)), + ..Default::default() + }, + Arg { + name: "d".into(), + has_default: true, + typ: Typ::List(Box::new(Typ::Int)), + default: Some(json!([1, 2, 3])), + ..Default::default() + }, + Arg { + name: "e".into(), + has_default: true, + typ: Typ::Object(ObjectType::new( + None, + Some(vec![ObjectProperty { + key: "a".into(), + typ: Box::new(Typ::Int) + }]) + )), + default: Some(json!({"a": 43})), + ..Default::default() + }, + ], + no_main_func: Some(false), + ..Default::default() + } + ); + } + + #[test] + fn test_parse_ruby_unsupported() { + assert_eq!( + &parse("def main( a: ) end").unwrap_err().to_string(), + " - a:\nkeyword parameters are not supported" + ); + + assert_eq!( + &parse("def main( *a ) end").unwrap_err().to_string(), + " - *a\nsplat parameters are not supported" + ); + + assert_eq!( + &parse("def main( **a ) end").unwrap_err().to_string(), + " - **a\nhash splat parameters are not supported" + ); + + assert!(&parse("def main( a = Time.now ) end").is_err()); + assert!(&parse("def main( a = :symbol ) end").is_err()); + assert!(&parse("def main( a = { b: 2 } ) end").is_err()); + assert!(&parse("def main( a = { b => 2 } ) end").is_err()); + assert!(&parse(r#"def main( a = { "b" => 2 } ) end"#).is_err()); + } + + // #[test] + // fn test_parse_ruby_requirements() { + // assert_eq!( + // parse_ruby_requirements( + // " + // require 'dep1' + // require 'dep2' + // require 'dep2/submod' + // require 'u/username/script' + // require 'f/folder/script' + // " + // ) + // .unwrap(), + // vec![ + // "dep1".to_owned(), + // "dep2".into(), + // "dep2".into(), + // "u/username/script".into(), + // "f/folder/script".into(), + // ] + // ); + // } +} diff --git a/backend/parsers/windmill-parser-ruby/src/wasm_libc.rs b/backend/parsers/windmill-parser-ruby/src/wasm_libc.rs new file mode 100644 index 0000000000..ddb6705023 --- /dev/null +++ b/backend/parsers/windmill-parser-ruby/src/wasm_libc.rs @@ -0,0 +1,297 @@ +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; +use std::{ + alloc::{self, Layout}, + ffi::{c_char, c_int, c_void}, + mem::align_of, + ptr, +}; +use wasm_bindgen::prelude::*; + +/* -------------------------------- stdlib.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn abort() { + panic!("Aborted from C"); +} + +macro_rules! console_log { + ($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) }) +} + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(js_namespace = console)] + fn log(a: &str); +} + +#[no_mangle] +pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size); + let buf = alloc::alloc(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + if count == 0 || size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size * count); + let buf = alloc::alloc_zeroed(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void { + if buf.is_null() { + malloc(new_size) + } else if new_size == 0 { + free(buf); + ptr::null_mut() + } else { + let (old_buf, old_layout) = retrieve_layout(buf); + let (new_layout, offset_to_data) = layout_for_size_prepended(new_size); + let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size()); + store_layout(new_buf, new_layout, offset_to_data) + } +} + +#[no_mangle] +pub unsafe extern "C" fn free(buf: *mut c_void) { + if buf.is_null() { + return; + } + let (buf, layout) = retrieve_layout(buf); + alloc::dealloc(buf, layout); +} + +// In all these allocations, we store the layout before the data for later retrieval. +// This is because we need to know the layout when deallocating the memory. +// Here are some helper methods for that: + +/// Given a pointer to the data, retrieve the layout and the pointer to the layout. +unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) { + let (_, layout_offset) = Layout::new::() + .extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap()) + .unwrap(); + + let buf = (buf as *mut u8).offset(-(layout_offset as isize)); + let layout = *(buf as *mut Layout); + + (buf, layout) +} + +/// Calculate a layout for a given size with space for storing a layout at the start. +/// Returns the layout and the offset to the data. +fn layout_for_size_prepended(size: usize) -> (Layout, usize) { + Layout::new::() + .extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap()) + .unwrap() +} + +/// Store a layout in the pointer, returning a pointer to where the data should be stored. +unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void { + *(buf as *mut Layout) = layout; + (buf as *mut u8).offset(offset_to_data as isize) as *mut c_void +} + +/* -------------------------------- string.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int { + let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n); + let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n); + + for (a, b) in s1.iter().zip(s2.iter()) { + if *a != *b || *a == 0 { + return (*a as i32) - (*b as i32); + } + } + + 0 +} + +// Implementation by AI: +pub type size_t = usize; +use std::slice; +#[no_mangle] +pub unsafe extern "C" fn memchr( + haystack: *const c_void, + needle: c_int, + len: usize, +) -> *mut c_void { + if haystack.is_null() || len == 0 { + return ptr::null_mut(); // Return null if the input pointer is null or length is zero + } + + let needle_byte = needle as u8; // Convert needle to a byte + + // Create a pointer to the start of the haystack + let mut current = haystack as *const u8; + + // Iterate through the memory block + for _ in 0..len { + if *current == needle_byte { + return current as *mut c_void; // Return the pointer to the found byte + } + current = current.add(1); // Move to the next byte + } + + ptr::null_mut() // Return null if the byte was not found +} + +#[no_mangle] +pub unsafe extern "C" fn strchr(mut s: *const c_char, c: c_int) -> *mut c_char { + if s.is_null() { + return std::ptr::null_mut(); // Return null if the input string is null + } + + let target = c as u8 as char; // Convert c to a char + let mut current = s; + + // Iterate through the string until we find the character or reach the end + while *current != 0 { + if *current as u8 as char == target { + return current as *mut c_char; // Return the pointer to the found character + } + current = current.add(1); // Move to the next character + } + + std::ptr::null_mut() // Return null if the character was not found +} +// End of AI implemetation +/* -------------------------------- wctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn iswspace(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_whitespace()) +} + +#[no_mangle] +pub unsafe extern "C" fn iswalnum(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric()) +} + +// Implementation by AI: +pub type wint_t = u32; + +#[no_mangle] +pub extern "C" fn iswdigit(wc: wint_t) -> c_int { + // Check if the character is a digit ('0' to '9') + if wc >= '0' as wint_t && wc <= '9' as wint_t { + return 1; // Return true (1) + } + 0 // Return false (0) +} + +#[no_mangle] +pub extern "C" fn iswupper(wc: wint_t) -> c_int { + // Check if the character is an uppercase letter ('A' to 'Z') + if wc >= 'A' as wint_t && wc <= 'Z' as wint_t { + return 1; // Return true (1) + } + 0 // Return false (0) +} + +#[no_mangle] +pub extern "C" fn iswalpha(wc: wint_t) -> c_int { + // Check if the character is an alphabetic character ('A' to 'Z' or 'a' to 'z') + if (wc >= 'A' as wint_t && wc <= 'Z' as wint_t) || (wc >= 'a' as wint_t && wc <= 'z' as wint_t) { + return 1; // Return true (1) + } + 0 // Return false (0) +} + +#[no_mangle] +pub extern "C" fn iswlower(wc: wint_t) -> c_int { + // Check if the character is a lowercase letter ('a' to 'z') + if wc >= 'a' as wint_t && wc <= 'z' as wint_t { + return 1; // Return true (1) + } + 0 // Return false (0) +} +// End of AI implemetation + + +/* --------------------------------- time.h --------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn clock() -> u64 { + panic!("clock is not supported"); +} + +/* --------------------------------- ctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn isprint(c: c_int) -> bool { + c >= 32 && c <= 126 +} + +/* --------------------------------- stdio.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int { + panic!("fprintf is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int { + panic!("fputs is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int { + panic!("fputc is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void { + panic!("fdopen is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int { + panic!("fclose is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fwrite( + _ptr: *const c_void, + _size: usize, + _nmemb: usize, + _stream: *mut c_void, +) -> usize { + panic!("fwrite is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn vsnprintf( + _buf: *mut c_char, + _size: usize, + _format: *const c_char, + _args: ... +) -> c_int { + panic!("vsnprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) { + panic!("clock_gettime is not supported"); +} + +// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... ); +#[no_mangle] +pub extern "C" fn snprintf() { + panic!("snprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) { + panic!("oh no"); +} diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 055dce6a81..7d998dbac7 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -388,7 +388,7 @@ fn parse_param( }; if typ == Typ::Unknown && dflt.is_some() { - typ = json_to_typ(dflt.as_ref().unwrap()); + typ = json_to_typ(dflt.as_ref().unwrap(), false); } Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true, oidx: None }) } diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 5b1c884143..0d60735a3a 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -29,6 +29,7 @@ ansible-parser = [ "dep:windmill-parser-yaml"] csharp-parser = [ "dep:windmill-parser-csharp"] nu-parser = [ "dep:windmill-parser-nu"] java-parser = [ "dep:windmill-parser-java"] +ruby-parser = [ "dep:windmill-parser-ruby"] [dependencies] anyhow.workspace = true @@ -45,6 +46,7 @@ windmill-parser-yaml = { workspace = true, optional = true } windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } +windmill-parser-ruby = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true getrandom = { workspace = true, features = ["js"] } diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index 60fdead893..1ab87f9873 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -50,11 +50,11 @@ const targets = [ desc: "Java", features: "java-parser", env: "tree-sitter", - # }, { - # ident: "ruby", - # desc: "Ruby", - # features: "ruby-parser", - # env: "tree-sitter", + }, { + ident: "ruby", + desc: "Ruby", + features: "ruby-parser", + env: "tree-sitter", }, # ^^^ Add new entry here ^^^ ]; @@ -130,3 +130,4 @@ def main [ } } } + diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index e02e280abb..238b207dd2 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -168,6 +168,11 @@ pub fn parse_java(code: &str) -> String { wrap_sig(windmill_parser_java::parse_java_signature(code)) } +#[cfg(feature = "ruby-parser")] +#[wasm_bindgen] +pub fn parse_ruby(code: &str) -> String { + wrap_sig(windmill_parser_ruby::parse_ruby_signature(code)) +} #[cfg(feature = "sql-parser")] #[wasm_bindgen] pub fn parse_assets_sql(code: &str) -> String { diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h index c0687e9c88..30fce92495 100644 --- a/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h @@ -5,3 +5,7 @@ void *memmove(void *dest, const void *src, unsigned long n); void *memset(void *s, int c, unsigned long n); int memcmp(const void *ptr1, const void *ptr2, unsigned long n); int strncmp(const char *s1, const char *s2, unsigned long n); + +// Manually added (needed for Ruby) +void *memchr (void *s, int c, size_t n); +char *strchr (char *s, int c); diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h index 517a954be5..ab0d3fd5ce 100644 --- a/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h @@ -5,3 +5,9 @@ typedef __WINT_TYPE__ wint_t; int iswspace(wchar_t ch); int iswalnum(wint_t _wc); + +// Manually added (needed for Ruby) +int iswdigit(wint_t _wc); +int iswupper(wint_t _wc); +int iswalpha(wint_t _wc); +int iswlower(wint_t _wc); diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 493222c3c3..565be52cd4 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -6,6 +6,8 @@ * LICENSE-AGPL for a copy of the license. */ +use std::mem::discriminant; + use convert_case::{Boundary, Case, Casing}; use serde::Serialize; use serde_json::Value; @@ -53,7 +55,7 @@ pub struct OneOfVariant { pub properties: Vec, } -#[derive(Serialize, Clone, Debug, PartialEq)] +#[derive(Serialize, Clone, Debug, PartialEq, Default)] #[serde(rename_all(serialize = "lowercase"))] pub enum Typ { Str(Option>), @@ -69,10 +71,11 @@ pub enum Typ { DynSelect(String), Object(ObjectType), OneOf(Vec), + #[default] Unknown, } -#[derive(Serialize, Clone, Debug, PartialEq)] +#[derive(Serialize, Clone, Debug, PartialEq, Default)] pub struct Arg { pub name: String, pub otyp: Option, @@ -82,7 +85,7 @@ pub struct Arg { pub oidx: Option, } -pub fn json_to_typ(js: &Value) -> Typ { +pub fn json_to_typ(js: &Value, precise_arrays: bool) -> Typ { match js { Value::String(_) => Typ::Str(None), Value::Number(n) if n.is_i64() => Typ::Int, @@ -92,11 +95,28 @@ pub fn json_to_typ(js: &Value) -> Typ { None, Some( o.iter() - .map(|(k, v)| ObjectProperty { key: k.to_string(), typ: Box::new(json_to_typ(v)) }) + .map(|(k, v)| ObjectProperty { + key: k.to_string(), + typ: Box::new(json_to_typ(v, precise_arrays)), + }) .collect(), ), )), - Value::Array(a) => Typ::List(Box::new(a.first().map(json_to_typ).unwrap_or(Typ::Unknown))), + Value::Array(a) => Typ::List(Box::new({ + // Check if all variant types are the same + if !precise_arrays + || a.windows(2).all(|pair| { + let (l, r) = (&pair[0], &pair[1]); + discriminant(l) == discriminant(r) + }) + { + a.first() + .map(|js| json_to_typ(js, precise_arrays)) + .unwrap_or(Typ::Unknown) + } else { + Typ::Unknown + } + })), _ => Typ::Unknown, } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 02133be4d0..9b80e58eef 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -10,7 +10,7 @@ use monitor::{ load_base_url, load_otel, reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, - reload_no_default_maven_setting, reload_nuget_config_setting, + reload_no_default_maven_setting, reload_nuget_config_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, }; @@ -47,7 +47,7 @@ use windmill_common::{ NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, + RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, @@ -83,7 +83,8 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, - PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, + PY312_CACHE_DIR, PY313_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, + UV_CACHE_DIR, }; use crate::monitor::{ @@ -1014,6 +1015,9 @@ Windmill Community Edition {GIT_VERSION} NO_DEFAULT_MAVEN_SETTING => { reload_no_default_maven_setting(&conn).await }, + RUBY_REPOS_SETTING => { + reload_ruby_repos_setting(&conn).await + }, KEEP_JOB_DIR_SETTING => { load_keep_job_dir(&conn).await; }, @@ -1368,6 +1372,7 @@ pub async fn run_workers( HUB_CACHE_DIR, POWERSHELL_CACHE_DIR, JAVA_CACHE_DIR, + RUBY_CACHE_DIR, TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG ] { DirBuilder::new() diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 2668934c8a..31f376d4b6 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -291,6 +291,7 @@ pub async fn initial_load( reload_nuget_config_setting(&conn).await; reload_maven_repos_setting(&conn).await; reload_no_default_maven_setting(&conn).await; + reload_ruby_repos_setting(&conn).await; } } @@ -1093,6 +1094,7 @@ pub async fn reload_nuget_config_setting(conn: &Connection) { ) .await; } + pub async fn reload_maven_repos_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -1102,6 +1104,7 @@ pub async fn reload_maven_repos_setting(conn: &Connection) { ) .await; } + pub async fn reload_no_default_maven_setting(conn: &Connection) { let value = load_value_from_global_settings_with_conn( conn, @@ -1118,6 +1121,16 @@ pub async fn reload_no_default_maven_setting(conn: &Connection) { }; } +pub async fn reload_ruby_repos_setting(conn: &Connection) { + reload_url_list_setting_with_tracing( + conn, + windmill_common::global_settings::RUBY_REPOS_SETTING, + "RUBY_REPOS", + windmill_worker::RUBY_REPOS.clone(), + ) + .await; +} + pub async fn reload_retention_period_setting(conn: &Connection) { if let Err(e) = reload_setting( conn, @@ -1287,6 +1300,89 @@ pub async fn reload_option_setting( Ok(()) } +pub async fn reload_url_list_setting_with_tracing( + conn: &Connection, + setting_name: &str, + std_env_var: &str, + lock: Arc>>>, +) { + if let Err(e) = reload_url_list_setting(conn, setting_name, std_env_var, lock.clone()).await { + tracing::error!("Error reloading setting {}: {:?}", setting_name, e) + } +} + +pub async fn reload_url_list_setting( + conn: &Connection, + setting_name: &str, + std_env_var: &str, + lock: Arc>>>, +) -> error::Result<()> { + // Check for force environment variable + if let Ok(force_value) = std::env::var(format!("FORCE_{}", std_env_var)) { + let mut urls = Vec::new(); + for url_str in force_value.trim().split_whitespace() { + match url::Url::parse(url_str) { + Ok(url) => urls.push(url), + Err(e) => { + return Err(error::Error::BadRequest(format!("Invalid URL in FORCE_{}: '{}': {}", std_env_var, url_str, e))); + } + } + } + let mut l = lock.write().await; + *l = if urls.is_empty() { None } else { Some(urls) }; + return Ok(()); + } + + let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + + // Check regular environment variable + let mut value = if let Ok(env_value) = std::env::var(std_env_var) { + let mut urls = Vec::new(); + for url_str in env_value.trim().split_whitespace() { + match url::Url::parse(url_str) { + Ok(url) => urls.push(url), + Err(_) => { + // Log error but continue, similar to force variable handling + tracing::error!("Invalid URL in {}: '{}'", std_env_var, url_str); + } + } + } + if urls.is_empty() { None } else { Some(urls) } + } else { + None + }; + + // Check database setting + if let Some(q) = q { + if let Ok(repos_str) = serde_json::from_value::(q.clone()) { + let mut urls = Vec::new(); + for url_str in repos_str.trim().split_whitespace() { + match url::Url::parse(url_str) { + Ok(url) => urls.push(url), + Err(e) => { + tracing::error!("Invalid URL in {}: '{}': {}", setting_name, url_str, e); + // Continue with other URLs, just skip invalid ones + } + } + } + tracing::info!("Loaded setting {} from db config: {} URLs", setting_name, urls.len()); + value = if urls.is_empty() { None } else { Some(urls) }; + } else { + tracing::error!("Could not parse {} found: {:#?}", setting_name, &q); + } + } + + { + if value.is_none() { + tracing::info!("Loaded {} setting to None", setting_name); + } + let mut l = lock.write().await; + *l = value; + } + + Ok(()) +} + pub async fn reload_setting( conn: &Connection, setting_name: &str, diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index cce3f338dc..a8631f3484 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3468,6 +3468,7 @@ async fn test_script_schedule_handlers(db: Pool) { tag: None, paused_until: None, cron_version: None, + description: None, }; let _ = client.create_schedule("test-workspace", &schedule).await; @@ -3538,6 +3539,7 @@ async fn test_script_schedule_handlers(db: Pool) { tag: None, paused_until: None, cron_version: None, + description: None, }, ) .await @@ -3621,6 +3623,7 @@ async fn test_flow_schedule_handlers(db: Pool) { tag: None, paused_until: None, cron_version: None, + description: None, }; let _ = client.create_schedule("test-workspace", &schedule).await; @@ -3692,6 +3695,7 @@ async fn test_flow_schedule_handlers(db: Pool) { tag: None, paused_until: None, cron_version: None, + description: None, }, ) .await @@ -3777,6 +3781,7 @@ async fn run_deployed_relative_imports( codebase: None, has_preprocessor: None, on_behalf_of_email: None, + assets: vec![], }, ) .await @@ -4028,6 +4033,7 @@ async fn assert_lockfile( codebase: None, has_preprocessor: None, on_behalf_of_email: None, + assets: vec![], }, ) .await diff --git a/backend/windmill-api-client/codegen.rs b/backend/windmill-api-client/codegen.rs index 202145159e..4de622c668 100644 --- a/backend/windmill-api-client/codegen.rs +++ b/backend/windmill-api-client/codegen.rs @@ -8213,6 +8213,9 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Csharp, #[serde(rename = "nu")] Nu, + #[serde(rename = "ruby")] + Ruby, + // for related places search: ADD_NEW_LANG } impl From<&ScriptLang> for ScriptLang { fn from(value: &ScriptLang) -> Self { @@ -8241,6 +8244,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Ansible => "ansible".to_string(), Self::Csharp => "csharp".to_string(), Self::Nu => "nu".to_string(), + Self::Ruby => "ruby".to_string(), + // for related places search: ADD_NEW_LANG } } } @@ -8267,6 +8272,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "ansible" => Ok(Self::Ansible), "csharp" => Ok(Self::Csharp), "nu" => Ok(Self::Nu), + "ruby" => Ok(Self::Ruby), + // for related places search: ADD_NEW_LANG _ => Err("invalid value"), } } diff --git a/backend/windmill-api-client/src/codegen.rs b/backend/windmill-api-client/src/codegen.rs index 9bc26262d5..2ef8fe8cac 100644 --- a/backend/windmill-api-client/src/codegen.rs +++ b/backend/windmill-api-client/src/codegen.rs @@ -7,6 +7,20 @@ pub mod types { use serde::{Deserialize, Serialize}; #[allow(unused_imports)] use std::convert::TryFrom; + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct AiConfig { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_completion_model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_model: Option, + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub providers: std::collections::HashMap, + } + impl From<&AiConfig> for AiConfig { + fn from(value: &AiConfig) -> Self { + value.clone() + } + } #[derive( Clone, Copy, @@ -22,6 +36,8 @@ pub mod types { pub enum AiProvider { #[serde(rename = "openai")] Openai, + #[serde(rename = "azure_openai")] + AzureOpenai, #[serde(rename = "anthropic")] Anthropic, #[serde(rename = "mistral")] @@ -34,6 +50,8 @@ pub mod types { Groq, #[serde(rename = "openrouter")] Openrouter, + #[serde(rename = "togetherai")] + Togetherai, #[serde(rename = "customai")] Customai, } @@ -46,12 +64,14 @@ pub mod types { fn to_string(&self) -> String { match *self { Self::Openai => "openai".to_string(), + Self::AzureOpenai => "azure_openai".to_string(), Self::Anthropic => "anthropic".to_string(), Self::Mistral => "mistral".to_string(), Self::Deepseek => "deepseek".to_string(), Self::Googleai => "googleai".to_string(), Self::Groq => "groq".to_string(), Self::Openrouter => "openrouter".to_string(), + Self::Togetherai => "togetherai".to_string(), Self::Customai => "customai".to_string(), } } @@ -61,12 +81,14 @@ pub mod types { fn from_str(value: &str) -> Result { match value { "openai" => Ok(Self::Openai), + "azure_openai" => Ok(Self::AzureOpenai), "anthropic" => Ok(Self::Anthropic), "mistral" => Ok(Self::Mistral), "deepseek" => Ok(Self::Deepseek), "googleai" => Ok(Self::Googleai), "groq" => Ok(Self::Groq), "openrouter" => Ok(Self::Openrouter), + "togetherai" => Ok(Self::Togetherai), "customai" => Ok(Self::Customai), _ => Err("invalid value"), } @@ -91,12 +113,35 @@ pub mod types { } } #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct AiResource { - pub path: String, + pub struct AiProviderConfig { + pub models: Vec, + pub resource_path: String, + } + impl From<&AiProviderConfig> for AiProviderConfig { + fn from(value: &AiProviderConfig) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct AiProviderModel { + pub model: String, pub provider: AiProvider, } - impl From<&AiResource> for AiResource { - fn from(value: &AiResource) -> Self { + impl From<&AiProviderModel> for AiProviderModel { + fn from(value: &AiProviderModel) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct Alert { + pub alert_cooldown_seconds: i64, + pub alert_time_threshold_seconds: i64, + pub jobs_num_threshold: i64, + pub name: String, + pub tags_to_monitor: Vec, + } + impl From<&Alert> for Alert { + fn from(value: &Alert) -> Self { value.clone() } } @@ -210,6 +255,201 @@ pub mod types { } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct Asset { + pub kind: AssetKind, + pub path: String, + } + impl From<&Asset> for Asset { + fn from(value: &Asset) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum AssetKind { + #[serde(rename = "s3object")] + S3object, + #[serde(rename = "resource")] + Resource, + #[serde(rename = "ducklake")] + Ducklake, + } + impl From<&AssetKind> for AssetKind { + fn from(value: &AssetKind) -> Self { + value.clone() + } + } + impl ToString for AssetKind { + fn to_string(&self) -> String { + match *self { + Self::S3object => "s3object".to_string(), + Self::Resource => "resource".to_string(), + Self::Ducklake => "ducklake".to_string(), + } + } + } + impl std::str::FromStr for AssetKind { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "s3object" => Ok(Self::S3object), + "resource" => Ok(Self::Resource), + "ducklake" => Ok(Self::Ducklake), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for AssetKind { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for AssetKind { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for AssetKind { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum AssetUsageAccessType { + #[serde(rename = "r")] + R, + #[serde(rename = "w")] + W, + #[serde(rename = "rw")] + Rw, + } + impl From<&AssetUsageAccessType> for AssetUsageAccessType { + fn from(value: &AssetUsageAccessType) -> Self { + value.clone() + } + } + impl ToString for AssetUsageAccessType { + fn to_string(&self) -> String { + match *self { + Self::R => "r".to_string(), + Self::W => "w".to_string(), + Self::Rw => "rw".to_string(), + } + } + } + impl std::str::FromStr for AssetUsageAccessType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "r" => Ok(Self::R), + "w" => Ok(Self::W), + "rw" => Ok(Self::Rw), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for AssetUsageAccessType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for AssetUsageAccessType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for AssetUsageAccessType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum AssetUsageKind { + #[serde(rename = "script")] + Script, + #[serde(rename = "flow")] + Flow, + } + impl From<&AssetUsageKind> for AssetUsageKind { + fn from(value: &AssetUsageKind) -> Self { + value.clone() + } + } + impl ToString for AssetUsageKind { + fn to_string(&self) -> String { + match *self { + Self::Script => "script".to_string(), + Self::Flow => "flow".to_string(), + } + } + } + impl std::str::FromStr for AssetUsageKind { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "script" => Ok(Self::Script), + "flow" => Ok(Self::Flow), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for AssetUsageKind { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for AssetUsageKind { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for AssetUsageKind { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct AuditLog { pub action_kind: AuditLogActionKind, pub id: i64, @@ -218,6 +458,8 @@ pub mod types { pub parameters: std::collections::HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub resource: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub span: Option, pub timestamp: chrono::DateTime, pub username: String, } @@ -690,6 +932,81 @@ pub mod types { value.parse() } } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum AuthenticationMethod { + #[serde(rename = "none")] + None, + #[serde(rename = "windmill")] + Windmill, + #[serde(rename = "api_key")] + ApiKey, + #[serde(rename = "basic_http")] + BasicHttp, + #[serde(rename = "custom_script")] + CustomScript, + #[serde(rename = "signature")] + Signature, + } + impl From<&AuthenticationMethod> for AuthenticationMethod { + fn from(value: &AuthenticationMethod) -> Self { + value.clone() + } + } + impl ToString for AuthenticationMethod { + fn to_string(&self) -> String { + match *self { + Self::None => "none".to_string(), + Self::Windmill => "windmill".to_string(), + Self::ApiKey => "api_key".to_string(), + Self::BasicHttp => "basic_http".to_string(), + Self::CustomScript => "custom_script".to_string(), + Self::Signature => "signature".to_string(), + } + } + } + impl std::str::FromStr for AuthenticationMethod { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "none" => Ok(Self::None), + "windmill" => Ok(Self::Windmill), + "api_key" => Ok(Self::ApiKey), + "basic_http" => Ok(Self::BasicHttp), + "custom_script" => Ok(Self::CustomScript), + "signature" => Ok(Self::Signature), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for AuthenticationMethod { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for AuthenticationMethod { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for AuthenticationMethod { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct AutoscalingEvent { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -710,6 +1027,65 @@ pub mod types { value.clone() } } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum AwsAuthResourceType { + #[serde(rename = "oidc")] + Oidc, + #[serde(rename = "credentials")] + Credentials, + } + impl From<&AwsAuthResourceType> for AwsAuthResourceType { + fn from(value: &AwsAuthResourceType) -> Self { + value.clone() + } + } + impl ToString for AwsAuthResourceType { + fn to_string(&self) -> String { + match *self { + Self::Oidc => "oidc".to_string(), + Self::Credentials => "credentials".to_string(), + } + } + } + impl std::str::FromStr for AwsAuthResourceType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "oidc" => Ok(Self::Oidc), + "credentials" => Ok(Self::Credentials), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for AwsAuthResourceType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for AwsAuthResourceType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for AwsAuthResourceType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct BranchAll { pub branches: Vec, @@ -874,9 +1250,8 @@ pub mod types { pub struct Capture { pub created_at: chrono::DateTime, pub id: i64, - pub payload: serde_json::Value, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trigger_extra: Option, + pub main_args: serde_json::Value, + pub preprocessor_args: serde_json::Value, pub trigger_kind: CaptureTriggerKind, } impl From<&Capture> for Capture { @@ -930,6 +1305,8 @@ pub mod types { Sqs, #[serde(rename = "mqtt")] Mqtt, + #[serde(rename = "gcp")] + Gcp, } impl From<&CaptureTriggerKind> for CaptureTriggerKind { fn from(value: &CaptureTriggerKind) -> Self { @@ -948,6 +1325,7 @@ pub mod types { Self::Postgres => "postgres".to_string(), Self::Sqs => "sqs".to_string(), Self::Mqtt => "mqtt".to_string(), + Self::Gcp => "gcp".to_string(), } } } @@ -964,6 +1342,7 @@ pub mod types { "postgres" => Ok(Self::Postgres), "sqs" => Ok(Self::Sqs), "mqtt" => Ok(Self::Mqtt), + "gcp" => Ok(Self::Gcp), _ => Err("invalid value"), } } @@ -1064,6 +1443,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub worker: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace_id: Option, } impl From<&CompletedJob> for CompletedJob { @@ -1200,6 +1581,39 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct Configs(pub Option); + impl std::ops::Deref for Configs { + type Target = Option; + fn deref(&self) -> &Option { + &self.0 + } + } + impl From for Option { + fn from(value: Configs) -> Self { + value.0 + } + } + impl From<&Configs> for Configs { + fn from(value: &Configs) -> Self { + value.clone() + } + } + impl From> for Configs { + fn from(value: Option) -> Self { + Self(value) + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct ConfigsInner { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub alerts: Vec, + } + impl From<&ConfigsInner> for ConfigsInner { + fn from(value: &ConfigsInner) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct ContextualVariable { pub description: String, pub is_custom: bool, @@ -1237,9 +1651,12 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct CreateResource { + ///The description of the resource #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, + ///The path to the resource pub path: String, + ///The resource_type associated with the resource pub resource_type: String, pub value: serde_json::Value, } @@ -1250,15 +1667,22 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct CreateVariable { + ///The account identifier #[serde(default, skip_serializing_if = "Option::is_none")] pub account: Option, + ///The description of the variable pub description: String, + ///The expiration date of the variable #[serde(default, skip_serializing_if = "Option::is_none")] pub expires_at: Option>, + ///Whether the variable is an OAuth variable #[serde(default, skip_serializing_if = "Option::is_none")] pub is_oauth: Option, + ///Whether the variable is a secret pub is_secret: bool, + ///The path to the variable pub path: String, + ///The value of the variable pub value: String, } impl From<&CreateVariable> for CreateVariable { @@ -1307,27 +1731,11 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct EditHttpTrigger { - pub http_method: EditHttpTriggerHttpMethod, - pub is_async: bool, - pub is_flow: bool, - pub is_static_website: bool, - pub path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub raw_string: Option, - pub requires_auth: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub route_path: Option, - pub script_path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub static_asset_config: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workspaced_route: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub wrap_body: Option, + pub struct DeleteGcpSubscription { + pub subscription_id: String, } - impl From<&EditHttpTrigger> for EditHttpTrigger { - fn from(value: &EditHttpTrigger) -> Self { + impl From<&DeleteGcpSubscription> for DeleteGcpSubscription { + fn from(value: &DeleteGcpSubscription) -> Self { value.clone() } } @@ -1343,66 +1751,201 @@ the execution of this script will be permissioned_as and by extension its DT_TOK PartialOrd, Serialize )] - pub enum EditHttpTriggerHttpMethod { - #[serde(rename = "get")] - Get, - #[serde(rename = "post")] - Post, - #[serde(rename = "put")] - Put, - #[serde(rename = "delete")] - Delete, - #[serde(rename = "patch")] - Patch, + pub enum DeliveryType { + #[serde(rename = "push")] + Push, + #[serde(rename = "pull")] + Pull, } - impl From<&EditHttpTriggerHttpMethod> for EditHttpTriggerHttpMethod { - fn from(value: &EditHttpTriggerHttpMethod) -> Self { + impl From<&DeliveryType> for DeliveryType { + fn from(value: &DeliveryType) -> Self { value.clone() } } - impl ToString for EditHttpTriggerHttpMethod { + impl ToString for DeliveryType { fn to_string(&self) -> String { match *self { - Self::Get => "get".to_string(), - Self::Post => "post".to_string(), - Self::Put => "put".to_string(), - Self::Delete => "delete".to_string(), - Self::Patch => "patch".to_string(), + Self::Push => "push".to_string(), + Self::Pull => "pull".to_string(), } } } - impl std::str::FromStr for EditHttpTriggerHttpMethod { + impl std::str::FromStr for DeliveryType { type Err = &'static str; fn from_str(value: &str) -> Result { match value { - "get" => Ok(Self::Get), - "post" => Ok(Self::Post), - "put" => Ok(Self::Put), - "delete" => Ok(Self::Delete), - "patch" => Ok(Self::Patch), + "push" => Ok(Self::Push), + "pull" => Ok(Self::Pull), _ => Err("invalid value"), } } } - impl std::convert::TryFrom<&str> for EditHttpTriggerHttpMethod { + impl std::convert::TryFrom<&str> for DeliveryType { type Error = &'static str; fn try_from(value: &str) -> Result { value.parse() } } - impl std::convert::TryFrom<&String> for EditHttpTriggerHttpMethod { + impl std::convert::TryFrom<&String> for DeliveryType { type Error = &'static str; fn try_from(value: &String) -> Result { value.parse() } } - impl std::convert::TryFrom for EditHttpTriggerHttpMethod { + impl std::convert::TryFrom for DeliveryType { type Error = &'static str; fn try_from(value: String) -> Result { value.parse() } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct DucklakeSettings { + pub ducklakes: std::collections::HashMap, + } + impl From<&DucklakeSettings> for DucklakeSettings { + fn from(value: &DucklakeSettings) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct DucklakeSettingsDucklakesValue { + pub catalog: DucklakeSettingsDucklakesValueCatalog, + pub storage: DucklakeSettingsDucklakesValueStorage, + } + impl From<&DucklakeSettingsDucklakesValue> for DucklakeSettingsDucklakesValue { + fn from(value: &DucklakeSettingsDucklakesValue) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct DucklakeSettingsDucklakesValueCatalog { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_path: Option, + pub resource_type: DucklakeSettingsDucklakesValueCatalogResourceType, + } + impl From<&DucklakeSettingsDucklakesValueCatalog> + for DucklakeSettingsDucklakesValueCatalog { + fn from(value: &DucklakeSettingsDucklakesValueCatalog) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum DucklakeSettingsDucklakesValueCatalogResourceType { + #[serde(rename = "postgresql")] + Postgresql, + #[serde(rename = "mysql")] + Mysql, + #[serde(rename = "instance")] + Instance, + } + impl From<&DucklakeSettingsDucklakesValueCatalogResourceType> + for DucklakeSettingsDucklakesValueCatalogResourceType { + fn from(value: &DucklakeSettingsDucklakesValueCatalogResourceType) -> Self { + value.clone() + } + } + impl ToString for DucklakeSettingsDucklakesValueCatalogResourceType { + fn to_string(&self) -> String { + match *self { + Self::Postgresql => "postgresql".to_string(), + Self::Mysql => "mysql".to_string(), + Self::Instance => "instance".to_string(), + } + } + } + impl std::str::FromStr for DucklakeSettingsDucklakesValueCatalogResourceType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "postgresql" => Ok(Self::Postgresql), + "mysql" => Ok(Self::Mysql), + "instance" => Ok(Self::Instance), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> + for DucklakeSettingsDucklakesValueCatalogResourceType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> + for DucklakeSettingsDucklakesValueCatalogResourceType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom + for DucklakeSettingsDucklakesValueCatalogResourceType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct DucklakeSettingsDucklakesValueStorage { + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + } + impl From<&DucklakeSettingsDucklakesValueStorage> + for DucklakeSettingsDucklakesValueStorage { + fn from(value: &DucklakeSettingsDucklakesValueStorage) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct EditHttpTrigger { + pub authentication_method: AuthenticationMethod, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authentication_resource_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub http_method: HttpMethod, + pub is_async: bool, + pub is_flow: bool, + pub is_static_website: bool, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_string: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub route_path: Option, + pub script_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub static_asset_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspaced_route: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub wrap_body: Option, + } + impl From<&EditHttpTrigger> for EditHttpTrigger { + fn from(value: &EditHttpTrigger) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditHttpTriggerStaticAssetConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub filename: Option, @@ -1417,10 +1960,16 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditKafkaTrigger { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub group_id: String, pub is_flow: bool, pub kafka_resource_path: String, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, pub topics: Vec, } @@ -1436,9 +1985,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub client_version: Option, pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, pub mqtt_resource_path: String, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, pub subscribe_topics: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1455,9 +2010,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub struct EditNatsTrigger { #[serde(default, skip_serializing_if = "Option::is_none")] pub consumer_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, pub nats_resource_path: String, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub stream_name: Option, @@ -1472,6 +2033,10 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditPostgresTrigger { pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, pub path: String, pub postgres_resource_path: String, @@ -1479,6 +2044,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub publication: Option, pub publication_name: String, pub replication_slot_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, } impl From<&EditPostgresTrigger> for EditPostgresTrigger { @@ -1488,10 +2055,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditResource { + ///The new description of the resource #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, + ///The path to the resource #[serde(default, skip_serializing_if = "Option::is_none")] pub path: Option, + ///The new resource_type to be associated with the resource + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_type: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub value: Option, } @@ -1515,38 +2087,53 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditSchedule { pub args: ScriptArgs, + ///The version of the cron schedule to use (last is v2) #[serde(default, skip_serializing_if = "Option::is_none")] pub cron_version: Option, + ///The description of the schedule + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + ///Whether the schedule should not run if a flow is already running #[serde(default, skip_serializing_if = "Option::is_none")] pub no_flow_overlap: Option, + ///The path to the script or flow to trigger on failure #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure: Option, + ///Whether the schedule should only run on the exact time #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure_exact: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure_extra_args: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure_times: Option, + ///The path to the script or flow to trigger on recovery #[serde(default, skip_serializing_if = "Option::is_none")] pub on_recovery: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_recovery_extra_args: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_recovery_times: Option, + ///The path to the script or flow to trigger on success #[serde(default, skip_serializing_if = "Option::is_none")] pub on_success: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_success_extra_args: Option, + ///The date and time the schedule will be paused until #[serde(default, skip_serializing_if = "Option::is_none")] pub paused_until: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub retry: Option, + ///The cron schedule to trigger the script or flow. Should include seconds. pub schedule: String, + ///The summary of the schedule #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, + ///The tag of the schedule #[serde(default, skip_serializing_if = "Option::is_none")] pub tag: Option, + ///The timezone to use for the cron schedule pub timezone: String, + ///Whether the WebSocket error handler is muted #[serde(default, skip_serializing_if = "Option::is_none")] pub ws_error_handler_muted: Option, } @@ -1557,13 +2144,20 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditSqsTrigger { + pub aws_auth_resource_type: AwsAuthResourceType, pub aws_resource_path: String, pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub message_attributes: Vec, pub path: String, pub queue_url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, } impl From<&EditSqsTrigger> for EditSqsTrigger { @@ -1573,12 +2167,16 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditVariable { + ///The new description of the variable #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, + ///Whether the variable is a secret #[serde(default, skip_serializing_if = "Option::is_none")] pub is_secret: Option, + ///The path to the variable #[serde(default, skip_serializing_if = "Option::is_none")] pub path: Option, + ///The new value of the variable #[serde(default, skip_serializing_if = "Option::is_none")] pub value: Option, } @@ -1590,11 +2188,17 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EditWebsocketTrigger { pub can_return_message: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub filters: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub initial_messages: Vec, pub is_flow: bool, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, pub url: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1630,6 +2234,104 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct EndpointTool { + ///JSON schema for request body + #[serde(default, skip_serializing_if = "Option::is_none")] + pub body_schema: Option>, + ///Short description of the tool + pub description: String, + ///Detailed instructions for using the tool + pub instructions: String, + ///HTTP method (GET, POST, etc.) + pub method: String, + ///The tool name/operation ID + pub name: String, + ///API endpoint path + pub path: String, + ///JSON schema for path parameters + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path_params_schema: Option< + std::collections::HashMap, + >, + ///JSON schema for query parameters + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query_params_schema: Option< + std::collections::HashMap, + >, + } + impl From<&EndpointTool> for EndpointTool { + fn from(value: &EndpointTool) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum ErrorHandler { + #[serde(rename = "custom")] + Custom, + #[serde(rename = "slack")] + Slack, + #[serde(rename = "teams")] + Teams, + #[serde(rename = "email")] + Email, + } + impl From<&ErrorHandler> for ErrorHandler { + fn from(value: &ErrorHandler) -> Self { + value.clone() + } + } + impl ToString for ErrorHandler { + fn to_string(&self) -> String { + match *self { + Self::Custom => "custom".to_string(), + Self::Slack => "slack".to_string(), + Self::Teams => "teams".to_string(), + Self::Email => "email".to_string(), + } + } + } + impl std::str::FromStr for ErrorHandler { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "custom" => Ok(Self::Custom), + "slack" => Ok(Self::Slack), + "teams" => Ok(Self::Teams), + "email" => Ok(Self::Email), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for ErrorHandler { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for ErrorHandler { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for ErrorHandler { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct ExportedInstanceGroup { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub emails: Vec, @@ -1771,9 +2473,9 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub sleep: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub stop_after_all_iters_if: Option, + pub stop_after_all_iters_if: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub stop_after_if: Option, + pub stop_after_if: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1809,30 +2511,6 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct FlowModuleStopAfterAllItersIf { - pub expr: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub skip_if_stopped: Option, - } - impl From<&FlowModuleStopAfterAllItersIf> for FlowModuleStopAfterAllItersIf { - fn from(value: &FlowModuleStopAfterAllItersIf) -> Self { - value.clone() - } - } - #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct FlowModuleStopAfterIf { - pub expr: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub skip_if_stopped: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error_message: Option - } - impl From<&FlowModuleStopAfterIf> for FlowModuleStopAfterIf { - fn from(value: &FlowModuleStopAfterIf) -> Self { - value.clone() - } - } - #[derive(Clone, Debug, Deserialize, Serialize)] pub struct FlowModuleSuspend { #[serde(default, skip_serializing_if = "Option::is_none")] pub continue_on_disapprove_timeout: Option, @@ -2319,14 +2997,105 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct GitRepositorySettings { + pub struct GcpTrigger { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delivery_config: Option, + pub delivery_type: DeliveryType, + pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub gcp_resource_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub server_id: Option, + pub subscription_id: String, + pub subscription_mode: SubscriptionMode, + pub topic_id: String, + } + impl From<&GcpTrigger> for GcpTrigger { + fn from(value: &GcpTrigger) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GcpTriggerData { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auto_acknowledge_msg: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delivery_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delivery_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub gcp_resource_path: String, + pub is_flow: bool, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + pub script_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subscription_id: Option, + pub subscription_mode: SubscriptionMode, + pub topic_id: String, + } + impl From<&GcpTriggerData> for GcpTriggerData { + fn from(value: &GcpTriggerData) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GenerateOpenapiSpec { #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub exclude_types_override: Vec, + pub http_route_filters: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub info: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub openapi_spec_format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub webhook_filters: Vec, + } + impl From<&GenerateOpenapiSpec> for GenerateOpenapiSpec { + fn from(value: &GenerateOpenapiSpec) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GetAllTopicSubscription { + pub topic_id: String, + } + impl From<&GetAllTopicSubscription> for GetAllTopicSubscription { + fn from(value: &GetAllTopicSubscription) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GitRepositorySettings { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collapsed: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude_types_override: Vec, pub git_repo_resource_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub group_by_folder: Option, pub script_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub settings: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub use_individual_branch: Option, } impl From<&GitRepositorySettings> for GitRepositorySettings { @@ -2334,6 +3103,22 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GitRepositorySettingsSettings { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude_path: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub extra_include_path: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include_path: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include_type: Vec, + } + impl From<&GitRepositorySettingsSettings> for GitRepositorySettingsSettings { + fn from(value: &GitRepositorySettingsSettings) -> Self { + value.clone() + } + } #[derive( Clone, Copy, @@ -2346,7 +3131,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK PartialOrd, Serialize )] - pub enum GitRepositorySettingsExcludeTypesOverrideItem { + pub enum GitSyncObjectType { #[serde(rename = "script")] Script, #[serde(rename = "flow")] @@ -2369,14 +3154,19 @@ the execution of this script will be permissioned_as and by extension its DT_TOK User, #[serde(rename = "group")] Group, + #[serde(rename = "trigger")] + Trigger, + #[serde(rename = "settings")] + Settings, + #[serde(rename = "key")] + Key, } - impl From<&GitRepositorySettingsExcludeTypesOverrideItem> - for GitRepositorySettingsExcludeTypesOverrideItem { - fn from(value: &GitRepositorySettingsExcludeTypesOverrideItem) -> Self { + impl From<&GitSyncObjectType> for GitSyncObjectType { + fn from(value: &GitSyncObjectType) -> Self { value.clone() } } - impl ToString for GitRepositorySettingsExcludeTypesOverrideItem { + impl ToString for GitSyncObjectType { fn to_string(&self) -> String { match *self { Self::Script => "script".to_string(), @@ -2390,10 +3180,13 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Schedule => "schedule".to_string(), Self::User => "user".to_string(), Self::Group => "group".to_string(), + Self::Trigger => "trigger".to_string(), + Self::Settings => "settings".to_string(), + Self::Key => "key".to_string(), } } } - impl std::str::FromStr for GitRepositorySettingsExcludeTypesOverrideItem { + impl std::str::FromStr for GitSyncObjectType { type Err = &'static str; fn from_str(value: &str) -> Result { match value { @@ -2408,31 +3201,79 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "schedule" => Ok(Self::Schedule), "user" => Ok(Self::User), "group" => Ok(Self::Group), + "trigger" => Ok(Self::Trigger), + "settings" => Ok(Self::Settings), + "key" => Ok(Self::Key), _ => Err("invalid value"), } } } - impl std::convert::TryFrom<&str> for GitRepositorySettingsExcludeTypesOverrideItem { + impl std::convert::TryFrom<&str> for GitSyncObjectType { type Error = &'static str; fn try_from(value: &str) -> Result { value.parse() } } - impl std::convert::TryFrom<&String> - for GitRepositorySettingsExcludeTypesOverrideItem { + impl std::convert::TryFrom<&String> for GitSyncObjectType { type Error = &'static str; fn try_from(value: &String) -> Result { value.parse() } } - impl std::convert::TryFrom - for GitRepositorySettingsExcludeTypesOverrideItem { + impl std::convert::TryFrom for GitSyncObjectType { type Error = &'static str; fn try_from(value: String) -> Result { value.parse() } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GithubInstallations(pub Vec); + impl std::ops::Deref for GithubInstallations { + type Target = Vec; + fn deref(&self) -> &Vec { + &self.0 + } + } + impl From for Vec { + fn from(value: GithubInstallations) -> Self { + value.0 + } + } + impl From<&GithubInstallations> for GithubInstallations { + fn from(value: &GithubInstallations) -> Self { + value.clone() + } + } + impl From> for GithubInstallations { + fn from(value: Vec) -> Self { + Self(value) + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GithubInstallationsItem { + pub account_id: String, + pub installation_id: f64, + pub repositories: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_id: Option, + } + impl From<&GithubInstallationsItem> for GithubInstallationsItem { + fn from(value: &GithubInstallationsItem) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct GithubInstallationsItemRepositoriesItem { + pub name: String, + pub url: String, + } + impl From<&GithubInstallationsItemRepositoriesItem> + for GithubInstallationsItemRepositoriesItem { + fn from(value: &GithubInstallationsItemRepositoriesItem) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct GlobalSetting { pub name: String, pub value: std::collections::HashMap, @@ -2538,24 +3379,6 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } - #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct HttpTrigger { - pub http_method: HttpTriggerHttpMethod, - pub is_async: bool, - pub is_static_website: bool, - pub raw_string: bool, - pub requires_auth: bool, - pub route_path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub static_asset_config: Option, - pub workspaced_route: bool, - pub wrap_body: bool, - } - impl From<&HttpTrigger> for HttpTrigger { - fn from(value: &HttpTrigger) -> Self { - value.clone() - } - } #[derive( Clone, Copy, @@ -2568,7 +3391,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK PartialOrd, Serialize )] - pub enum HttpTriggerHttpMethod { + pub enum HttpMethod { #[serde(rename = "get")] Get, #[serde(rename = "post")] @@ -2580,12 +3403,12 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(rename = "patch")] Patch, } - impl From<&HttpTriggerHttpMethod> for HttpTriggerHttpMethod { - fn from(value: &HttpTriggerHttpMethod) -> Self { + impl From<&HttpMethod> for HttpMethod { + fn from(value: &HttpMethod) -> Self { value.clone() } } - impl ToString for HttpTriggerHttpMethod { + impl ToString for HttpMethod { fn to_string(&self) -> String { match *self { Self::Get => "get".to_string(), @@ -2596,7 +3419,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } } - impl std::str::FromStr for HttpTriggerHttpMethod { + impl std::str::FromStr for HttpMethod { type Err = &'static str; fn from_str(value: &str) -> Result { match value { @@ -2609,25 +3432,55 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } } - impl std::convert::TryFrom<&str> for HttpTriggerHttpMethod { + impl std::convert::TryFrom<&str> for HttpMethod { type Error = &'static str; fn try_from(value: &str) -> Result { value.parse() } } - impl std::convert::TryFrom<&String> for HttpTriggerHttpMethod { + impl std::convert::TryFrom<&String> for HttpMethod { type Error = &'static str; fn try_from(value: &String) -> Result { value.parse() } } - impl std::convert::TryFrom for HttpTriggerHttpMethod { + impl std::convert::TryFrom for HttpMethod { type Error = &'static str; fn try_from(value: String) -> Result { value.parse() } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct HttpTrigger { + pub authentication_method: AuthenticationMethod, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authentication_resource_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub http_method: HttpMethod, + pub is_async: bool, + pub is_static_website: bool, + pub raw_string: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + pub route_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub static_asset_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + } + impl From<&HttpTrigger> for HttpTrigger { + fn from(value: &HttpTrigger) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct HttpTriggerStaticAssetConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub filename: Option, @@ -2640,27 +3493,71 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } - #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct HubScriptKind(pub serde_json::Value); - impl std::ops::Deref for HubScriptKind { - type Target = serde_json::Value; - fn deref(&self) -> &serde_json::Value { - &self.0 - } - } - impl From for serde_json::Value { - fn from(value: HubScriptKind) -> Self { - value.0 - } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum HubScriptKind { + #[serde(rename = "script")] + Script, + #[serde(rename = "failure")] + Failure, + #[serde(rename = "trigger")] + Trigger, + #[serde(rename = "approval")] + Approval, } impl From<&HubScriptKind> for HubScriptKind { fn from(value: &HubScriptKind) -> Self { value.clone() } } - impl From for HubScriptKind { - fn from(value: serde_json::Value) -> Self { - Self(value) + impl ToString for HubScriptKind { + fn to_string(&self) -> String { + match *self { + Self::Script => "script".to_string(), + Self::Failure => "failure".to_string(), + Self::Trigger => "trigger".to_string(), + Self::Approval => "approval".to_string(), + } + } + } + impl std::str::FromStr for HubScriptKind { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "script" => Ok(Self::Script), + "failure" => Ok(Self::Failure), + "trigger" => Ok(Self::Trigger), + "approval" => Ok(Self::Approval), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for HubScriptKind { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for HubScriptKind { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for HubScriptKind { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() } } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -2780,6 +3677,21 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct InstanceGroupWithWorkspaces { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub emails: Vec, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub workspaces: Vec, + } + impl From<&InstanceGroupWithWorkspaces> for InstanceGroupWithWorkspaces { + fn from(value: &InstanceGroupWithWorkspaces) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct JavascriptTransform { pub expr: String, #[serde(rename = "type")] @@ -3013,11 +3925,17 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub enabled: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub group_id: String, pub kafka_resource_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub server_id: Option, pub topics: Vec, } @@ -3085,6 +4003,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub azure_blob_resource_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub gcs_resource_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub public_resource: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub s3_resource_path: Option, @@ -3106,6 +4026,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub azure_blob_resource_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub gcs_resource_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub public_resource: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub s3_resource_path: Option, @@ -3135,6 +4057,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK AzureBlobStorage, AzureWorkloadIdentity, S3AwsOidc, + GoogleCloudStorage, } impl From<&LargeFileStorageSecondaryStorageValueType> for LargeFileStorageSecondaryStorageValueType { @@ -3149,6 +4072,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::AzureBlobStorage => "AzureBlobStorage".to_string(), Self::AzureWorkloadIdentity => "AzureWorkloadIdentity".to_string(), Self::S3AwsOidc => "S3AwsOidc".to_string(), + Self::GoogleCloudStorage => "GoogleCloudStorage".to_string(), } } } @@ -3160,6 +4084,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "AzureBlobStorage" => Ok(Self::AzureBlobStorage), "AzureWorkloadIdentity" => Ok(Self::AzureWorkloadIdentity), "S3AwsOidc" => Ok(Self::S3AwsOidc), + "GoogleCloudStorage" => Ok(Self::GoogleCloudStorage), _ => Err("invalid value"), } } @@ -3199,6 +4124,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK AzureBlobStorage, AzureWorkloadIdentity, S3AwsOidc, + GoogleCloudStorage, } impl From<&LargeFileStorageType> for LargeFileStorageType { fn from(value: &LargeFileStorageType) -> Self { @@ -3212,6 +4138,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::AzureBlobStorage => "AzureBlobStorage".to_string(), Self::AzureWorkloadIdentity => "AzureWorkloadIdentity".to_string(), Self::S3AwsOidc => "S3AwsOidc".to_string(), + Self::GoogleCloudStorage => "GoogleCloudStorage".to_string(), } } } @@ -3223,6 +4150,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "AzureBlobStorage" => Ok(Self::AzureBlobStorage), "AzureWorkloadIdentity" => Ok(Self::AzureWorkloadIdentity), "S3AwsOidc" => Ok(Self::S3AwsOidc), + "GoogleCloudStorage" => Ok(Self::GoogleCloudStorage), _ => Err("invalid value"), } } @@ -3253,6 +4181,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub id: i64, pub path: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_app: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub starred: Option, pub summary: String, pub version: i64, @@ -3479,7 +4409,12 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(rename = "str")] Str(Option>), #[serde(rename = "object")] - Object(Vec), + Object { + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + props: Vec, + }, #[serde(rename = "list")] List(MainArgSignatureArgsItemTypList), } @@ -3498,12 +4433,6 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Str(value) } } - impl From> - for MainArgSignatureArgsItemTyp { - fn from(value: Vec) -> Self { - Self::Object(value) - } - } impl From for MainArgSignatureArgsItemTyp { fn from(value: MainArgSignatureArgsItemTypList) -> Self { Self::List(value) @@ -3543,18 +4472,18 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] - pub struct MainArgSignatureArgsItemTypObjectItem { + pub struct MainArgSignatureArgsItemTypObjectPropsItem { pub key: String, - pub typ: MainArgSignatureArgsItemTypObjectItemTyp, + pub typ: MainArgSignatureArgsItemTypObjectPropsItemTyp, } - impl From<&MainArgSignatureArgsItemTypObjectItem> - for MainArgSignatureArgsItemTypObjectItem { - fn from(value: &MainArgSignatureArgsItemTypObjectItem) -> Self { + impl From<&MainArgSignatureArgsItemTypObjectPropsItem> + for MainArgSignatureArgsItemTypObjectPropsItem { + fn from(value: &MainArgSignatureArgsItemTypObjectPropsItem) -> Self { value.clone() } } #[derive(Clone, Debug, Deserialize, Serialize)] - pub enum MainArgSignatureArgsItemTypObjectItemTyp { + pub enum MainArgSignatureArgsItemTypObjectPropsItemTyp { #[serde(rename = "float")] Float, #[serde(rename = "int")] @@ -3576,13 +4505,13 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(rename = "str")] Str(serde_json::Value), } - impl From<&MainArgSignatureArgsItemTypObjectItemTyp> - for MainArgSignatureArgsItemTypObjectItemTyp { - fn from(value: &MainArgSignatureArgsItemTypObjectItemTyp) -> Self { + impl From<&MainArgSignatureArgsItemTypObjectPropsItemTyp> + for MainArgSignatureArgsItemTypObjectPropsItemTyp { + fn from(value: &MainArgSignatureArgsItemTypObjectPropsItemTyp) -> Self { value.clone() } } - impl From for MainArgSignatureArgsItemTypObjectItemTyp { + impl From for MainArgSignatureArgsItemTypObjectPropsItemTyp { fn from(value: serde_json::Value) -> Self { Self::Str(value) } @@ -3807,9 +4736,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, pub mqtt_resource_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub server_id: Option, pub subscribe_topics: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -3839,7 +4774,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub session_expiry_interval: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub topic_alias: Option, + pub topic_alias_maximum: Option, } impl From<&MqttV5Config> for MqttV5Config { fn from(value: &MqttV5Config) -> Self { @@ -3854,9 +4789,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, pub nats_resource_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub server_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub stream_name: Option, @@ -3870,19 +4811,31 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewHttpTrigger { - pub http_method: NewHttpTriggerHttpMethod, + pub authentication_method: AuthenticationMethod, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authentication_resource_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + pub http_method: HttpMethod, pub is_async: bool, pub is_flow: bool, pub is_static_website: bool, pub path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub raw_string: Option, - pub requires_auth: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub route_path: String, pub script_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub static_asset_config: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspaced_route: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub wrap_body: Option, @@ -3892,77 +4845,6 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } - #[derive( - Clone, - Copy, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize - )] - pub enum NewHttpTriggerHttpMethod { - #[serde(rename = "get")] - Get, - #[serde(rename = "post")] - Post, - #[serde(rename = "put")] - Put, - #[serde(rename = "delete")] - Delete, - #[serde(rename = "patch")] - Patch, - } - impl From<&NewHttpTriggerHttpMethod> for NewHttpTriggerHttpMethod { - fn from(value: &NewHttpTriggerHttpMethod) -> Self { - value.clone() - } - } - impl ToString for NewHttpTriggerHttpMethod { - fn to_string(&self) -> String { - match *self { - Self::Get => "get".to_string(), - Self::Post => "post".to_string(), - Self::Put => "put".to_string(), - Self::Delete => "delete".to_string(), - Self::Patch => "patch".to_string(), - } - } - } - impl std::str::FromStr for NewHttpTriggerHttpMethod { - type Err = &'static str; - fn from_str(value: &str) -> Result { - match value { - "get" => Ok(Self::Get), - "post" => Ok(Self::Post), - "put" => Ok(Self::Put), - "delete" => Ok(Self::Delete), - "patch" => Ok(Self::Patch), - _ => Err("invalid value"), - } - } - } - impl std::convert::TryFrom<&str> for NewHttpTriggerHttpMethod { - type Error = &'static str; - fn try_from(value: &str) -> Result { - value.parse() - } - } - impl std::convert::TryFrom<&String> for NewHttpTriggerHttpMethod { - type Error = &'static str; - fn try_from(value: &String) -> Result { - value.parse() - } - } - impl std::convert::TryFrom for NewHttpTriggerHttpMethod { - type Error = &'static str; - fn try_from(value: String) -> Result { - value.parse() - } - } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewHttpTriggerStaticAssetConfig { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -3980,10 +4862,16 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub struct NewKafkaTrigger { #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub group_id: String, pub is_flow: bool, pub kafka_resource_path: String, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, pub topics: Vec, } @@ -4000,9 +4888,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub client_version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, pub mqtt_resource_path: String, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, pub subscribe_topics: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -4021,9 +4915,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub consumer_name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, pub nats_resource_path: String, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub stream_name: Option, @@ -4038,6 +4938,10 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewPostgresTrigger { pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, pub path: String, pub postgres_resource_path: String, @@ -4047,6 +4951,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub publication_name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub replication_slot_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, } impl From<&NewPostgresTrigger> for NewPostgresTrigger { @@ -4057,43 +4963,62 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewSchedule { pub args: ScriptArgs, + ///The version of the cron schedule to use (last is v2) #[serde(default, skip_serializing_if = "Option::is_none")] pub cron_version: Option, + ///The description of the schedule + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + ///Whether the schedule is enabled #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + ///Whether the schedule is for a flow pub is_flow: bool, + ///Whether the schedule should not run if a flow is already running #[serde(default, skip_serializing_if = "Option::is_none")] pub no_flow_overlap: Option, + ///The path to the script or flow to trigger on failure #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure: Option, + ///Whether the schedule should only run on the exact time #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure_exact: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure_extra_args: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure_times: Option, + ///The path to the script or flow to trigger on recovery #[serde(default, skip_serializing_if = "Option::is_none")] pub on_recovery: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_recovery_extra_args: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_recovery_times: Option, + ///The path to the script or flow to trigger on success #[serde(default, skip_serializing_if = "Option::is_none")] pub on_success: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_success_extra_args: Option, + ///The path where the schedule will be created pub path: String, + ///The date and time the schedule will be paused until #[serde(default, skip_serializing_if = "Option::is_none")] pub paused_until: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub retry: Option, + ///The cron schedule to trigger the script or flow. Should include seconds. pub schedule: String, + ///The path to the script or flow to trigger pub script_path: String, + ///The summary of the schedule #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, + ///The tag of the schedule #[serde(default, skip_serializing_if = "Option::is_none")] pub tag: Option, + ///The timezone to use for the cron schedule pub timezone: String, + ///Whether the WebSocket error handler is muted #[serde(default, skip_serializing_if = "Option::is_none")] pub ws_error_handler_muted: Option, } @@ -4104,6 +5029,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewScript { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub assets: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -4163,6 +5090,146 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct NewScriptAssetsItem { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub access_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alt_access_type: Option, + pub kind: AssetKind, + pub path: String, + } + impl From<&NewScriptAssetsItem> for NewScriptAssetsItem { + fn from(value: &NewScriptAssetsItem) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum NewScriptAssetsItemAccessType { + #[serde(rename = "r")] + R, + #[serde(rename = "w")] + W, + #[serde(rename = "rw")] + Rw, + } + impl From<&NewScriptAssetsItemAccessType> for NewScriptAssetsItemAccessType { + fn from(value: &NewScriptAssetsItemAccessType) -> Self { + value.clone() + } + } + impl ToString for NewScriptAssetsItemAccessType { + fn to_string(&self) -> String { + match *self { + Self::R => "r".to_string(), + Self::W => "w".to_string(), + Self::Rw => "rw".to_string(), + } + } + } + impl std::str::FromStr for NewScriptAssetsItemAccessType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "r" => Ok(Self::R), + "w" => Ok(Self::W), + "rw" => Ok(Self::Rw), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for NewScriptAssetsItemAccessType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for NewScriptAssetsItemAccessType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for NewScriptAssetsItemAccessType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum NewScriptAssetsItemAltAccessType { + #[serde(rename = "r")] + R, + #[serde(rename = "w")] + W, + #[serde(rename = "rw")] + Rw, + } + impl From<&NewScriptAssetsItemAltAccessType> for NewScriptAssetsItemAltAccessType { + fn from(value: &NewScriptAssetsItemAltAccessType) -> Self { + value.clone() + } + } + impl ToString for NewScriptAssetsItemAltAccessType { + fn to_string(&self) -> String { + match *self { + Self::R => "r".to_string(), + Self::W => "w".to_string(), + Self::Rw => "rw".to_string(), + } + } + } + impl std::str::FromStr for NewScriptAssetsItemAltAccessType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "r" => Ok(Self::R), + "w" => Ok(Self::W), + "rw" => Ok(Self::Rw), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for NewScriptAssetsItemAltAccessType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for NewScriptAssetsItemAltAccessType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for NewScriptAssetsItemAltAccessType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } #[derive( Clone, Copy, @@ -4253,14 +5320,21 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NewSqsTrigger { + pub aws_auth_resource_type: AwsAuthResourceType, pub aws_resource_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub is_flow: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub message_attributes: Vec, pub path: String, pub queue_url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, } impl From<&NewSqsTrigger> for NewSqsTrigger { @@ -4304,11 +5378,17 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub can_return_message: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub filters: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub initial_messages: Vec, pub is_flow: bool, pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, pub script_path: String, pub url: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -4383,6 +5463,121 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct OpenapiHttpRouteFilters { + pub folder_regex: String, + pub path_regex: String, + pub route_path_regex: String, + } + impl From<&OpenapiHttpRouteFilters> for OpenapiHttpRouteFilters { + fn from(value: &OpenapiHttpRouteFilters) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum OpenapiSpecFormat { + #[serde(rename = "yaml")] + Yaml, + #[serde(rename = "json")] + Json, + } + impl From<&OpenapiSpecFormat> for OpenapiSpecFormat { + fn from(value: &OpenapiSpecFormat) -> Self { + value.clone() + } + } + impl ToString for OpenapiSpecFormat { + fn to_string(&self) -> String { + match *self { + Self::Yaml => "yaml".to_string(), + Self::Json => "json".to_string(), + } + } + } + impl std::str::FromStr for OpenapiSpecFormat { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "yaml" => Ok(Self::Yaml), + "json" => Ok(Self::Json), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for OpenapiSpecFormat { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for OpenapiSpecFormat { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for OpenapiSpecFormat { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct OpenapiV3Info { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub contact: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub license: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub terms_of_service: Option, + pub title: String, + pub version: String, + } + impl From<&OpenapiV3Info> for OpenapiV3Info { + fn from(value: &OpenapiV3Info) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct OpenapiV3InfoContact { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub email: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + } + impl From<&OpenapiV3InfoContact> for OpenapiV3InfoContact { + fn from(value: &OpenapiV3InfoContact) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct OpenapiV3InfoLicense { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identifier: Option, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + } + impl From<&OpenapiV3InfoLicense> for OpenapiV3InfoLicense { + fn from(value: &OpenapiV3InfoLicense) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct OperatorSettings(pub Option); impl std::ops::Deref for OperatorSettings { type Target = Option; @@ -4407,6 +5602,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct OperatorSettingsInner { + ///Whether operators can view assets + pub assets: bool, ///Whether operators can view audit logs pub audit_logs: bool, ///Whether operators can view folders page @@ -4689,11 +5886,17 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, pub postgres_resource_path: String, pub publication_name: String, pub replication_slot_name: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub server_id: Option, } impl From<&PostgresTrigger> for PostgresTrigger { @@ -4717,6 +5920,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub tag: Option, } impl From<&Preview> for Preview { @@ -4799,6 +6004,17 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct PushConfig { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audience: Option, + pub authenticate: bool, + } + impl From<&PushConfig> for PushConfig { + fn from(value: &PushConfig) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct QueuedJob { #[serde(default, skip_serializing_if = "Option::is_none")] pub aggregate_wait_time_ms: Option, @@ -4861,6 +6077,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub worker: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace_id: Option, } impl From<&QueuedJob> for QueuedJob { @@ -4977,6 +6195,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct RawScript { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub assets: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub concurrency_time_window_s: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -5003,6 +6223,209 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct RawScriptAssetsItem { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub access_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alt_access_type: Option, + pub kind: RawScriptAssetsItemKind, + pub path: String, + } + impl From<&RawScriptAssetsItem> for RawScriptAssetsItem { + fn from(value: &RawScriptAssetsItem) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum RawScriptAssetsItemAccessType { + #[serde(rename = "r")] + R, + #[serde(rename = "w")] + W, + #[serde(rename = "rw")] + Rw, + } + impl From<&RawScriptAssetsItemAccessType> for RawScriptAssetsItemAccessType { + fn from(value: &RawScriptAssetsItemAccessType) -> Self { + value.clone() + } + } + impl ToString for RawScriptAssetsItemAccessType { + fn to_string(&self) -> String { + match *self { + Self::R => "r".to_string(), + Self::W => "w".to_string(), + Self::Rw => "rw".to_string(), + } + } + } + impl std::str::FromStr for RawScriptAssetsItemAccessType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "r" => Ok(Self::R), + "w" => Ok(Self::W), + "rw" => Ok(Self::Rw), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for RawScriptAssetsItemAccessType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for RawScriptAssetsItemAccessType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for RawScriptAssetsItemAccessType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum RawScriptAssetsItemAltAccessType { + #[serde(rename = "r")] + R, + #[serde(rename = "w")] + W, + #[serde(rename = "rw")] + Rw, + } + impl From<&RawScriptAssetsItemAltAccessType> for RawScriptAssetsItemAltAccessType { + fn from(value: &RawScriptAssetsItemAltAccessType) -> Self { + value.clone() + } + } + impl ToString for RawScriptAssetsItemAltAccessType { + fn to_string(&self) -> String { + match *self { + Self::R => "r".to_string(), + Self::W => "w".to_string(), + Self::Rw => "rw".to_string(), + } + } + } + impl std::str::FromStr for RawScriptAssetsItemAltAccessType { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "r" => Ok(Self::R), + "w" => Ok(Self::W), + "rw" => Ok(Self::Rw), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for RawScriptAssetsItemAltAccessType { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for RawScriptAssetsItemAltAccessType { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for RawScriptAssetsItemAltAccessType { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum RawScriptAssetsItemKind { + #[serde(rename = "s3object")] + S3object, + #[serde(rename = "resource")] + Resource, + #[serde(rename = "ducklake")] + Ducklake, + } + impl From<&RawScriptAssetsItemKind> for RawScriptAssetsItemKind { + fn from(value: &RawScriptAssetsItemKind) -> Self { + value.clone() + } + } + impl ToString for RawScriptAssetsItemKind { + fn to_string(&self) -> String { + match *self { + Self::S3object => "s3object".to_string(), + Self::Resource => "resource".to_string(), + Self::Ducklake => "ducklake".to_string(), + } + } + } + impl std::str::FromStr for RawScriptAssetsItemKind { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "s3object" => Ok(Self::S3object), + "resource" => Ok(Self::Resource), + "ducklake" => Ok(Self::Ducklake), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for RawScriptAssetsItemKind { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for RawScriptAssetsItemKind { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for RawScriptAssetsItemKind { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct RawScriptForDependencies { pub language: ScriptLang, pub path: String, @@ -5299,6 +6722,65 @@ the execution of this script will be permissioned_as and by extension its DT_TOK PartialOrd, Serialize )] + pub enum RunnableKind { + #[serde(rename = "script")] + Script, + #[serde(rename = "flow")] + Flow, + } + impl From<&RunnableKind> for RunnableKind { + fn from(value: &RunnableKind) -> Self { + value.clone() + } + } + impl ToString for RunnableKind { + fn to_string(&self) -> String { + match *self { + Self::Script => "script".to_string(), + Self::Flow => "flow".to_string(), + } + } + } + impl std::str::FromStr for RunnableKind { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "script" => Ok(Self::Script), + "flow" => Ok(Self::Flow), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for RunnableKind { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for RunnableKind { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for RunnableKind { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] pub enum RunnableType { ScriptHash, ScriptPath, @@ -5348,6 +6830,21 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct S3Object { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filename: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub presigned: Option, + pub s3: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + } + impl From<&S3Object> for S3Object { + fn from(value: &S3Object) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct S3Resource { #[serde(rename = "accessKey", default, skip_serializing_if = "Option::is_none")] pub access_key: Option, @@ -5384,6 +6881,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub args: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub cron_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, pub edited_at: chrono::DateTime, pub edited_by: String, pub email: String, @@ -5422,8 +6921,6 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub tag: Option, pub timezone: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -5458,6 +6955,31 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct ScopeDefinition { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + pub label: String, + pub requires_resource_path: bool, + pub value: String, + } + impl From<&ScopeDefinition> for ScopeDefinition { + fn from(value: &ScopeDefinition) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct ScopeDomain { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + pub name: String, + pub scopes: Vec, + } + impl From<&ScopeDomain> for ScopeDomain { + fn from(value: &ScopeDomain) -> Self { + value.clone() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Script { pub archived: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -5687,6 +7209,12 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Csharp, #[serde(rename = "nu")] Nu, + #[serde(rename = "java")] + Java, + #[serde(rename = "ruby")] + Ruby, + #[serde(rename = "duckdb")] + Duckdb, } impl From<&ScriptLang> for ScriptLang { fn from(value: &ScriptLang) -> Self { @@ -5715,6 +7243,9 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Self::Ansible => "ansible".to_string(), Self::Csharp => "csharp".to_string(), Self::Nu => "nu".to_string(), + Self::Java => "java".to_string(), + Self::Ruby => "ruby".to_string(), + Self::Duckdb => "duckdb".to_string(), } } } @@ -5741,6 +7272,9 @@ the execution of this script will be permissioned_as and by extension its DT_TOK "ansible" => Ok(Self::Ansible), "csharp" => Ok(Self::Csharp), "nu" => Ok(Self::Nu), + "java" => Ok(Self::Java), + "ruby" => Ok(Self::Ruby), + "duckdb" => Ok(Self::Duckdb), _ => Err("invalid value"), } } @@ -5809,16 +7343,23 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct SqsTrigger { + pub aws_auth_resource_type: AwsAuthResourceType, pub aws_resource_path: String, pub enabled: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub message_attributes: Vec, pub queue_url: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub server_id: Option, } impl From<&SqsTrigger> for SqsTrigger { @@ -5830,8 +7371,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub struct StaticTransform { #[serde(rename = "type")] pub type_: StaticTransformType, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option, + pub value: serde_json::Value, } impl From<&StaticTransform> for StaticTransform { fn from(value: &StaticTransform) -> Self { @@ -5851,8 +7391,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Serialize )] pub enum StaticTransformType { - #[serde(rename = "javascript")] - Javascript, + #[serde(rename = "static")] + Static, } impl From<&StaticTransformType> for StaticTransformType { fn from(value: &StaticTransformType) -> Self { @@ -5862,7 +7402,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK impl ToString for StaticTransformType { fn to_string(&self) -> String { match *self { - Self::Javascript => "javascript".to_string(), + Self::Static => "static".to_string(), } } } @@ -5870,7 +7410,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK type Err = &'static str; fn from_str(value: &str) -> Result { match value { - "javascript" => Ok(Self::Javascript), + "static" => Ok(Self::Static), _ => Err("invalid value"), } } @@ -5894,6 +7434,79 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct StopAfterIf { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_message: Option, + pub expr: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub skip_if_stopped: Option, + } + impl From<&StopAfterIf> for StopAfterIf { + fn from(value: &StopAfterIf) -> Self { + value.clone() + } + } + ///The mode of subscription. 'existing' means using an existing GCP subscription, while 'create_update' involves creating or updating a new subscription. + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum SubscriptionMode { + #[serde(rename = "existing")] + Existing, + #[serde(rename = "create_update")] + CreateUpdate, + } + impl From<&SubscriptionMode> for SubscriptionMode { + fn from(value: &SubscriptionMode) -> Self { + value.clone() + } + } + impl ToString for SubscriptionMode { + fn to_string(&self) -> String { + match *self { + Self::Existing => "existing".to_string(), + Self::CreateUpdate => "create_update".to_string(), + } + } + } + impl std::str::FromStr for SubscriptionMode { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "existing" => Ok(Self::Existing), + "create_update" => Ok(Self::CreateUpdate), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for SubscriptionMode { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for SubscriptionMode { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for SubscriptionMode { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TableToTrack(pub Vec); impl std::ops::Deref for TableToTrack { type Target = Vec; @@ -5944,6 +7557,254 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct TeamsChannel { + ///Microsoft Teams channel ID + pub channel_id: TeamsChannelChannelId, + ///Microsoft Teams channel name + pub channel_name: TeamsChannelChannelName, + ///Microsoft Teams team ID + pub team_id: TeamsChannelTeamId, + ///Microsoft Teams team name + pub team_name: TeamsChannelTeamName, + } + impl From<&TeamsChannel> for TeamsChannel { + fn from(value: &TeamsChannel) -> Self { + value.clone() + } + } + ///Microsoft Teams channel ID + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] + pub struct TeamsChannelChannelId(String); + impl std::ops::Deref for TeamsChannelChannelId { + type Target = String; + fn deref(&self) -> &String { + &self.0 + } + } + impl From for String { + fn from(value: TeamsChannelChannelId) -> Self { + value.0 + } + } + impl From<&TeamsChannelChannelId> for TeamsChannelChannelId { + fn from(value: &TeamsChannelChannelId) -> Self { + value.clone() + } + } + impl std::str::FromStr for TeamsChannelChannelId { + type Err = &'static str; + fn from_str(value: &str) -> Result { + if value.len() < 1usize { + return Err("shorter than 1 characters"); + } + Ok(Self(value.to_string())) + } + } + impl std::convert::TryFrom<&str> for TeamsChannelChannelId { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for TeamsChannelChannelId { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for TeamsChannelChannelId { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + impl<'de> serde::Deserialize<'de> for TeamsChannelChannelId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer)? + .parse() + .map_err(|e: &'static str| { + ::custom(e.to_string()) + }) + } + } + ///Microsoft Teams channel name + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] + pub struct TeamsChannelChannelName(String); + impl std::ops::Deref for TeamsChannelChannelName { + type Target = String; + fn deref(&self) -> &String { + &self.0 + } + } + impl From for String { + fn from(value: TeamsChannelChannelName) -> Self { + value.0 + } + } + impl From<&TeamsChannelChannelName> for TeamsChannelChannelName { + fn from(value: &TeamsChannelChannelName) -> Self { + value.clone() + } + } + impl std::str::FromStr for TeamsChannelChannelName { + type Err = &'static str; + fn from_str(value: &str) -> Result { + if value.len() < 1usize { + return Err("shorter than 1 characters"); + } + Ok(Self(value.to_string())) + } + } + impl std::convert::TryFrom<&str> for TeamsChannelChannelName { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for TeamsChannelChannelName { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for TeamsChannelChannelName { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + impl<'de> serde::Deserialize<'de> for TeamsChannelChannelName { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer)? + .parse() + .map_err(|e: &'static str| { + ::custom(e.to_string()) + }) + } + } + ///Microsoft Teams team ID + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] + pub struct TeamsChannelTeamId(String); + impl std::ops::Deref for TeamsChannelTeamId { + type Target = String; + fn deref(&self) -> &String { + &self.0 + } + } + impl From for String { + fn from(value: TeamsChannelTeamId) -> Self { + value.0 + } + } + impl From<&TeamsChannelTeamId> for TeamsChannelTeamId { + fn from(value: &TeamsChannelTeamId) -> Self { + value.clone() + } + } + impl std::str::FromStr for TeamsChannelTeamId { + type Err = &'static str; + fn from_str(value: &str) -> Result { + if value.len() < 1usize { + return Err("shorter than 1 characters"); + } + Ok(Self(value.to_string())) + } + } + impl std::convert::TryFrom<&str> for TeamsChannelTeamId { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for TeamsChannelTeamId { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for TeamsChannelTeamId { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + impl<'de> serde::Deserialize<'de> for TeamsChannelTeamId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer)? + .parse() + .map_err(|e: &'static str| { + ::custom(e.to_string()) + }) + } + } + ///Microsoft Teams team name + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] + pub struct TeamsChannelTeamName(String); + impl std::ops::Deref for TeamsChannelTeamName { + type Target = String; + fn deref(&self) -> &String { + &self.0 + } + } + impl From for String { + fn from(value: TeamsChannelTeamName) -> Self { + value.0 + } + } + impl From<&TeamsChannelTeamName> for TeamsChannelTeamName { + fn from(value: &TeamsChannelTeamName) -> Self { + value.clone() + } + } + impl std::str::FromStr for TeamsChannelTeamName { + type Err = &'static str; + fn from_str(value: &str) -> Result { + if value.len() < 1usize { + return Err("shorter than 1 characters"); + } + Ok(Self(value.to_string())) + } + } + impl std::convert::TryFrom<&str> for TeamsChannelTeamName { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for TeamsChannelTeamName { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for TeamsChannelTeamName { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + impl<'de> serde::Deserialize<'de> for TeamsChannelTeamName { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer)? + .parse() + .map_err(|e: &'static str| { + ::custom(e.to_string()) + }) + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TemplateScript { pub language: Language, pub postgres_resource_path: String, @@ -5971,6 +7832,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub expires_in: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub grant_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub refresh_token: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub scope: Vec, @@ -6001,6 +7864,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Option::is_none")] pub email_count: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub gcp_count: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub http_routes_count: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub kafka_count: Option, @@ -6078,6 +7943,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct User { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub added_via: Option, pub created_at: chrono::DateTime, pub disabled: bool, pub email: String, @@ -6098,6 +7965,86 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct UserSource { + ///The domain used for auto-invite (when source is 'domain') + #[serde(default, skip_serializing_if = "Option::is_none")] + pub domain: Option, + ///The instance group name (when source is 'instance_group') + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group: Option, + ///How the user was added to the workspace + pub source: UserSourceSource, + } + impl From<&UserSource> for UserSource { + fn from(value: &UserSource) -> Self { + value.clone() + } + } + ///How the user was added to the workspace + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum UserSourceSource { + #[serde(rename = "domain")] + Domain, + #[serde(rename = "instance_group")] + InstanceGroup, + #[serde(rename = "manual")] + Manual, + } + impl From<&UserSourceSource> for UserSourceSource { + fn from(value: &UserSourceSource) -> Self { + value.clone() + } + } + impl ToString for UserSourceSource { + fn to_string(&self) -> String { + match *self { + Self::Domain => "domain".to_string(), + Self::InstanceGroup => "instance_group".to_string(), + Self::Manual => "manual".to_string(), + } + } + } + impl std::str::FromStr for UserSourceSource { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "domain" => Ok(Self::Domain), + "instance_group" => Ok(Self::InstanceGroup), + "manual" => Ok(Self::Manual), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for UserSourceSource { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for UserSourceSource { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for UserSourceSource { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UserUsage { #[serde(default, skip_serializing_if = "Option::is_none")] pub email: Option, @@ -6134,17 +8081,98 @@ the execution of this script will be permissioned_as and by extension its DT_TOK } } #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct WebhookFilters { + pub path: String, + pub runnable_kind: RunnableKind, + pub user_or_folder_regex: WebhookFiltersUserOrFolderRegex, + pub user_or_folder_regex_value: String, + } + impl From<&WebhookFilters> for WebhookFilters { + fn from(value: &WebhookFilters) -> Self { + value.clone() + } + } + #[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize + )] + pub enum WebhookFiltersUserOrFolderRegex { + #[serde(rename = "*")] + X, + #[serde(rename = "u")] + U, + #[serde(rename = "f")] + F, + } + impl From<&WebhookFiltersUserOrFolderRegex> for WebhookFiltersUserOrFolderRegex { + fn from(value: &WebhookFiltersUserOrFolderRegex) -> Self { + value.clone() + } + } + impl ToString for WebhookFiltersUserOrFolderRegex { + fn to_string(&self) -> String { + match *self { + Self::X => "*".to_string(), + Self::U => "u".to_string(), + Self::F => "f".to_string(), + } + } + } + impl std::str::FromStr for WebhookFiltersUserOrFolderRegex { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "*" => Ok(Self::X), + "u" => Ok(Self::U), + "f" => Ok(Self::F), + _ => Err("invalid value"), + } + } + } + impl std::convert::TryFrom<&str> for WebhookFiltersUserOrFolderRegex { + type Error = &'static str; + fn try_from(value: &str) -> Result { + value.parse() + } + } + impl std::convert::TryFrom<&String> for WebhookFiltersUserOrFolderRegex { + type Error = &'static str; + fn try_from(value: &String) -> Result { + value.parse() + } + } + impl std::convert::TryFrom for WebhookFiltersUserOrFolderRegex { + type Error = &'static str; + fn try_from(value: String) -> Result { + value.parse() + } + } + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct WebsocketTrigger { pub can_return_message: bool, pub enabled: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, pub filters: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub initial_messages: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_server_ping: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub server_id: Option, pub url: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -6206,8 +8234,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK Serialize )] pub enum WhileloopFlowType { - #[serde(rename = "forloopflow")] - Forloopflow, + #[serde(rename = "whileloopflow")] + Whileloopflow, } impl From<&WhileloopFlowType> for WhileloopFlowType { fn from(value: &WhileloopFlowType) -> Self { @@ -6217,7 +8245,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK impl ToString for WhileloopFlowType { fn to_string(&self) -> String { match *self { - Self::Forloopflow => "forloopflow".to_string(), + Self::Whileloopflow => "whileloopflow".to_string(), } } } @@ -6225,7 +8253,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK type Err = &'static str; fn from_str(value: &str) -> Result { match value { - "forloopflow" => Ok(Self::Forloopflow), + "whileloopflow" => Ok(Self::Whileloopflow), _ => Err("invalid value"), } } @@ -6476,99 +8504,15 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[serde(default, skip_serializing_if = "Vec::is_empty")] pub include_path: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub include_type: Vec, + pub include_type: Vec, } impl From<&WorkspaceDeployUiSettings> for WorkspaceDeployUiSettings { fn from(value: &WorkspaceDeployUiSettings) -> Self { value.clone() } } - #[derive( - Clone, - Copy, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize - )] - pub enum WorkspaceDeployUiSettingsIncludeTypeItem { - #[serde(rename = "script")] - Script, - #[serde(rename = "flow")] - Flow, - #[serde(rename = "app")] - App, - #[serde(rename = "resource")] - Resource, - #[serde(rename = "variable")] - Variable, - #[serde(rename = "secret")] - Secret, - #[serde(rename = "trigger")] - Trigger, - } - impl From<&WorkspaceDeployUiSettingsIncludeTypeItem> - for WorkspaceDeployUiSettingsIncludeTypeItem { - fn from(value: &WorkspaceDeployUiSettingsIncludeTypeItem) -> Self { - value.clone() - } - } - impl ToString for WorkspaceDeployUiSettingsIncludeTypeItem { - fn to_string(&self) -> String { - match *self { - Self::Script => "script".to_string(), - Self::Flow => "flow".to_string(), - Self::App => "app".to_string(), - Self::Resource => "resource".to_string(), - Self::Variable => "variable".to_string(), - Self::Secret => "secret".to_string(), - Self::Trigger => "trigger".to_string(), - } - } - } - impl std::str::FromStr for WorkspaceDeployUiSettingsIncludeTypeItem { - type Err = &'static str; - fn from_str(value: &str) -> Result { - match value { - "script" => Ok(Self::Script), - "flow" => Ok(Self::Flow), - "app" => Ok(Self::App), - "resource" => Ok(Self::Resource), - "variable" => Ok(Self::Variable), - "secret" => Ok(Self::Secret), - "trigger" => Ok(Self::Trigger), - _ => Err("invalid value"), - } - } - } - impl std::convert::TryFrom<&str> for WorkspaceDeployUiSettingsIncludeTypeItem { - type Error = &'static str; - fn try_from(value: &str) -> Result { - value.parse() - } - } - impl std::convert::TryFrom<&String> for WorkspaceDeployUiSettingsIncludeTypeItem { - type Error = &'static str; - fn try_from(value: &String) -> Result { - value.parse() - } - } - impl std::convert::TryFrom for WorkspaceDeployUiSettingsIncludeTypeItem { - type Error = &'static str; - fn try_from(value: String) -> Result { - value.parse() - } - } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct WorkspaceGitSyncSettings { - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub include_path: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub include_type: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub repositories: Vec, } @@ -6577,100 +8521,28 @@ the execution of this script will be permissioned_as and by extension its DT_TOK value.clone() } } - #[derive( - Clone, - Copy, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize - )] - pub enum WorkspaceGitSyncSettingsIncludeTypeItem { - #[serde(rename = "script")] - Script, - #[serde(rename = "flow")] - Flow, - #[serde(rename = "app")] - App, - #[serde(rename = "folder")] - Folder, - #[serde(rename = "resource")] - Resource, - #[serde(rename = "variable")] - Variable, - #[serde(rename = "secret")] - Secret, - #[serde(rename = "resourcetype")] - Resourcetype, - #[serde(rename = "schedule")] - Schedule, - #[serde(rename = "user")] - User, - #[serde(rename = "group")] - Group, + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct WorkspaceGithubInstallation { + pub account_id: String, + pub installation_id: f64, } - impl From<&WorkspaceGitSyncSettingsIncludeTypeItem> - for WorkspaceGitSyncSettingsIncludeTypeItem { - fn from(value: &WorkspaceGitSyncSettingsIncludeTypeItem) -> Self { + impl From<&WorkspaceGithubInstallation> for WorkspaceGithubInstallation { + fn from(value: &WorkspaceGithubInstallation) -> Self { value.clone() } } - impl ToString for WorkspaceGitSyncSettingsIncludeTypeItem { - fn to_string(&self) -> String { - match *self { - Self::Script => "script".to_string(), - Self::Flow => "flow".to_string(), - Self::App => "app".to_string(), - Self::Folder => "folder".to_string(), - Self::Resource => "resource".to_string(), - Self::Variable => "variable".to_string(), - Self::Secret => "secret".to_string(), - Self::Resourcetype => "resourcetype".to_string(), - Self::Schedule => "schedule".to_string(), - Self::User => "user".to_string(), - Self::Group => "group".to_string(), - } - } + #[derive(Clone, Debug, Deserialize, Serialize)] + pub struct WorkspaceInfo { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_name: Option, } - impl std::str::FromStr for WorkspaceGitSyncSettingsIncludeTypeItem { - type Err = &'static str; - fn from_str(value: &str) -> Result { - match value { - "script" => Ok(Self::Script), - "flow" => Ok(Self::Flow), - "app" => Ok(Self::App), - "folder" => Ok(Self::Folder), - "resource" => Ok(Self::Resource), - "variable" => Ok(Self::Variable), - "secret" => Ok(Self::Secret), - "resourcetype" => Ok(Self::Resourcetype), - "schedule" => Ok(Self::Schedule), - "user" => Ok(Self::User), - "group" => Ok(Self::Group), - _ => Err("invalid value"), - } - } - } - impl std::convert::TryFrom<&str> for WorkspaceGitSyncSettingsIncludeTypeItem { - type Error = &'static str; - fn try_from(value: &str) -> Result { - value.parse() - } - } - impl std::convert::TryFrom<&String> for WorkspaceGitSyncSettingsIncludeTypeItem { - type Error = &'static str; - fn try_from(value: &String) -> Result { - value.parse() - } - } - impl std::convert::TryFrom for WorkspaceGitSyncSettingsIncludeTypeItem { - type Error = &'static str; - fn try_from(value: String) -> Result { - value.parse() + impl From<&WorkspaceInfo> for WorkspaceInfo { + fn from(value: &WorkspaceInfo) -> Self { + value.clone() } } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -6689,7 +8561,7 @@ the execution of this script will be permissioned_as and by extension its DT_TOK #[derive(Clone, Debug)] /**Client for Windmill API -Version: 1.478.1*/ +Version: 1.526.1*/ pub struct Client { pub(crate) baseurl: String, pub(crate) client: reqwest::Client, @@ -6735,7 +8607,7 @@ impl Client { /// This string is pulled directly from the source OpenAPI /// document and may be in any format the API selects. pub fn api_version(&self) -> &'static str { - "1.478.1" + "1.526.1" } } impl Client { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7b916c85d1..132fd8bd50 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -15128,7 +15128,8 @@ components: csharp, nu, java, - duckdb, + ruby, + duckdb # for related places search: ADD_NEW_LANG ] diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index d47b1b2c12..ca2cec7ce1 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -734,6 +734,7 @@ async fn create_script_internal<'c>( || ns.language == ScriptLang::Nu || ns.language == ScriptLang::Php || ns.language == ScriptLang::Java + || ns.language == ScriptLang::Ruby // for related places search: ADD_NEW_LANG ) { Some(String::new()) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index c752367339..1b1f107be6 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -377,6 +377,7 @@ pub(crate) async fn tarball_workspace( ScriptLang::Nu => "nu", ScriptLang::OracleDB => "odb.sql", ScriptLang::Java => "java", + ScriptLang::Ruby => "rb", // for related places search: ADD_NEW_LANG }; archive diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 6418968d28..97eb6fbced 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -181,6 +181,13 @@ impl From for Error { } } +impl From for Error { + #[track_caller] + fn from(e: url::ParseError) -> Self { + Self::ArgumentErr(format!("Cannot parse provided url. \ne: {e}")) + } +} + impl Error { /// https://docs.rs/anyhow/1/anyhow/struct.Error.html#display-representations pub fn alt(&self) -> String { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 6be5acca3a..c26bdc2f7e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -13,6 +13,7 @@ pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; pub const NUGET_CONFIG_SETTING: &str = "nuget_config"; pub const MAVEN_REPOS_SETTING: &str = "maven_repos"; pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven"; +pub const RUBY_REPOS_SETTING: &str = "ruby_repos"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; @@ -66,6 +67,10 @@ pub const ENV_SETTINGS: &[&str] = &[ "DENO_PATH", "GO_PATH", "JAVA_PATH", + "RUBY_PATH", + "BUNDLE_PATH", + "GEM_PATH", + "RUBY_CONCURRENT_DOWNLOADS", // for related places search: ADD_NEW_LANG "GOPRIVATE", "GOPROXY", diff --git a/backend/windmill-common/src/schema.rs b/backend/windmill-common/src/schema.rs index 71ecd220d1..d16b86d507 100644 --- a/backend/windmill-common/src/schema.rs +++ b/backend/windmill-common/src/schema.rs @@ -348,7 +348,7 @@ pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool { use ScriptLang::*; let comment = match lang { Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//", - Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#", + Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", Rust => "//!", // for related places search: ADD_NEW_LANG diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index ff2f1865af..7633e60e25 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -55,7 +55,9 @@ pub enum ScriptLang { Ansible, CSharp, Nu, - Java, // for related places search: ADD_NEW_LANG + Java, + Ruby, + // for related places search: ADD_NEW_LANG } impl ScriptLang { @@ -83,6 +85,7 @@ impl ScriptLang { ScriptLang::CSharp => "csharp", ScriptLang::Nu => "nu", ScriptLang::Java => "java", + ScriptLang::Ruby => "ruby", // for related places search: ADD_NEW_LANG } } @@ -113,6 +116,8 @@ impl FromStr for ScriptLang { "csharp" => ScriptLang::CSharp, "nu" => ScriptLang::Nu, "java" => ScriptLang::Java, + "ruby" => ScriptLang::Ruby, + // for related places search: ADD_NEW_LANG language => { return Err(anyhow::anyhow!("{} is currently not supported", language).into()) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index c34b9959f1..50ed13ff7b 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -179,6 +179,7 @@ lazy_static::lazy_static! { "csharp".to_string(), "nu".to_string(), "java".to_string(), + "ruby".to_string(), "duckdb".to_string(), // for related places search: ADD_NEW_LANG "dependency".to_string(), @@ -622,6 +623,12 @@ fn parse_file(path: &str) -> Option { .flatten() } +#[derive(Copy, Clone)] +#[annotations("#")] +pub struct RubyAnnotations { + pub verbose: bool, +} + #[derive(Copy, Clone)] #[annotations("#")] pub struct PythonAnnotations { diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index d78d7cff27..3805171024 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -32,6 +32,7 @@ csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] +ruby = ["dep:windmill-parser-ruby"] duckdb = ["dep:duckdb"] [dependencies] @@ -46,6 +47,7 @@ windmill-parser-rust = { workspace = true, optional = true } windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } +windmill-parser-ruby = { workspace = true, optional = true } windmill-parser-py = { workspace = true, optional = true } windmill-parser-yaml.workspace = true windmill-parser-py-imports = { workspace = true, optional = true } diff --git a/backend/windmill-worker/nsjail/download.ruby.config.proto b/backend/windmill-worker/nsjail/download.ruby.config.proto new file mode 100644 index 0000000000..7b9a04f912 --- /dev/null +++ b/backend/windmill-worker/nsjail/download.ruby.config.proto @@ -0,0 +1,86 @@ +name: "gem install" + +mode: ONCE +hostname: "ruby" +log_level: ERROR +time_limit: 900 + +rlimit_as: 2048 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 64 + +envar: "HOME=/user" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: true +keep_env: true +mount_proc: true + + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + +mount { + src: "{TARGET}" + dst: "{TARGET}" + is_bind: true + mandatory: false + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +{DEV} diff --git a/backend/windmill-worker/nsjail/lock.ruby.config.proto b/backend/windmill-worker/nsjail/lock.ruby.config.proto new file mode 100644 index 0000000000..4d6035a772 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.ruby.config.proto @@ -0,0 +1,79 @@ +name: "bundle lock" + +mode: ONCE +hostname: "ruby" +log_level: ERROR +time_limit: 900 + +rlimit_as: 2048 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 64 + +envar: "HOME=/tmp" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: true +keep_env: true +mount_proc: true + + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "{JOB_DIR}" + dst: "/tmp" + is_bind: true + mandatory: false + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +{DEV} diff --git a/backend/windmill-worker/nsjail/run.ruby.config.proto b/backend/windmill-worker/nsjail/run.ruby.config.proto new file mode 100644 index 0000000000..539b87e147 --- /dev/null +++ b/backend/windmill-worker/nsjail/run.ruby.config.proto @@ -0,0 +1,113 @@ +name: "ruby run script" + +mode: ONCE +hostname: "ruby" +log_level: ERROR + +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +# mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{JOB_DIR}/main.rb" + dst: "/tmp/main.rb" + is_bind: true + mandatory: false +} + +mount { + src: "{JOB_DIR}/args.json" + dst: "/tmp/args.json" + is_bind: true +} + +mount { + src: "{JOB_DIR}/result.json" + dst: "/tmp/result.json" + rw: true + is_bind: true +} + +#mount { +# src: "{CACHE_DIR}" +# dst: "{CACHE_DIR}" +# is_bind: true +# mandatory: false +# } + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +iface_no_lo: true + +{SHARED_MOUNT} + +{SHARED_DEPENDENCIES} + + +{DEV} diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index bf5316f9eb..051101d727 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -12,7 +12,6 @@ use sha2::Digest; use sqlx::types::Json; use sqlx::{Pool, Postgres}; use tokio::process::Command; -use tokio::sync::{RwLock, Semaphore}; use tokio::{fs::File, io::AsyncReadExt}; #[cfg(feature = "parquet")] @@ -31,11 +30,10 @@ use windmill_common::{ variables::ContextualVariable, }; -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; use windmill_queue::MiniPulledJob; -use std::ops::AsyncFn; use std::path::Path; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -45,7 +43,7 @@ use windmill_common::{variables, DB}; use tokio::{io::AsyncWriteExt, time::Instant}; use crate::agent_workers::UPDATE_PING_URL; -use crate::{DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV}; +use crate::{JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV}; use windmill_common::client::AuthedClient; pub async fn build_args_map<'a>( @@ -1072,561 +1070,6 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result .map_err(|e| Error::internal_err(format!("Error building http client: {e:#}"))) } -#[derive(Clone)] -pub struct RequiredDependency { - /// Expected directory of dependency in cache - /// For example: - /// /tmp/windmill/cache/python_311/rich==0.0.0 - /// IMPORTANT!: path should not end with '/' - pub path: String, - /// Name to use for S3 tars - /// If not specified will use top level directory of path. - pub custom_name: Option, - /// Display name - /// Name that will be used for console output and logging - /// If not specified will either use custom_name or top level directory of path. - pub short_name: Option, -} - -pub enum InstallStrategy { - /// Will invoke callback to install single dependency - Single(Arc Result + Send + Sync>), - /// Will try to pull S3 first and will invoke closure to install the rest - AllAtOnce(Arc) -> Result + Send + Sync>), -} -/// # General -/// -/// Languages that compile usually include dependencies in final executable. -/// When dynamic languages do not and runtime dependencies provided separately. -/// -/// This helper implies that the language is dynamic. -/// Python, Ruby, Java are dynamic and they can use this helper. -/// -/// # Features -/// -/// This helper will install all specified dependencies in parallel and if it is EE, cache to S3 -/// It has atomic success file, allowing to distinguish failed installations from succesfull. -/// -/// Besides that it provides console output and does logging. -/// -/// # Usage -/// -/// Most important arguments in this helper are `deps` and `install_fn` -/// -/// In `deps` you specify all dependencies that are needed to be on worker in order to execute script. -/// You don't know which are actually installed and which are not. -/// -/// `deps` is a vector of RequiredDependency. Check [RequiredDependency] for more context. -/// -/// After `deps` are provided helper will check each dependency and check if it is in cache, if not it will try to pull from S3 -/// and if it does not work either, it will invoke `install_fn` closure. -/// Closure arguments has dependency name as well as it`s expected path in cache. -/// Closure should return Command that will install dependency to asked place. -pub async fn par_install_language_dependencies<'a>( - deps: Vec, - language_name: &'a str, - installer_executable_name: &'a str, - platform_agnostic: bool, - concurrent_downloads: usize, - stdout_on_err: bool, - install_fn: InstallStrategy, - postinstall_cb: impl AsyncFn(Vec) -> Result<(), error::Error>, - job_id: &'a Uuid, - w_id: &'a str, - worker_name: &'a str, - conn: &Connection, -) -> anyhow::Result<()> { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let _ = (platform_agnostic, language_name); - - let total_time = std::time::Instant::now(); - - // Total to install - let mut not_installed = vec![]; - let total_to_install; - // let mut not_installed = vec![]; - let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); - // Append logs with line like this: - // [9/21] + requests==2.32.3 << (S3) | in 57ms - #[allow(unused_assignments)] - async fn print_success( - mut s3_pull: bool, - mut s3_push: bool, - job_id: &Uuid, - w_id: &str, - req: &str, - req_tl: usize, - counter_arc: Arc>, - total_to_install: usize, - instant: std::time::Instant, - conn: &Connection, - ) { - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - (s3_pull, s3_push) = (false, false); - } - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if windmill_common::s3_helpers::OBJECT_STORE_SETTINGS - .read() - .await - .is_none() - { - (s3_pull, s3_push) = (false, false); - } - - let mut counter = counter_arc.lock().await; - *counter += 1; - - windmill_queue::append_logs( - job_id, - w_id, - format!( - "\n{}+ {}{}{}| in {}ms", - windmill_common::worker::pad_string( - &format!("[{}/{total_to_install}]", counter), - 9 - ), - // Because we want to align to max len [999/999] we take 9 - // 123456789 - windmill_common::worker::pad_string(&req, req_tl + 1), - // Margin to the right ^ - if s3_pull { "<< (S3) " } else { "" }, - if s3_push { " > (S3) " } else { "" }, - instant.elapsed().as_millis(), - ), - conn, - ) - .await; - // Drop lock, so next print success can fire - } - - let mut name_tl = 0; - struct NotInstalledDependency { - path: String, - custom_name: Option, - short_name: Option, - display_name: String, - } - { - let mut to_be_installed_is_used = false; - for RequiredDependency { - path, // - custom_name, - short_name, - } in deps.into_iter() - { - if path.ends_with("/") { - anyhow::bail!("Internal error: path should not end with '/'") - } - let display_name = short_name - .as_ref() - .or(custom_name.as_ref()) - .or(path.split("/").last().map(|e| e.to_owned()).as_ref()) - .unwrap_or_else(|| { - tracing::warn!( - workspace_id = %w_id, - job_id = %job_id, - "failed to parse top level directory name for {path}, fallback to full path.", - ); - - &path - }) - .to_owned(); - { - // Later will help us align text in log console - if display_name.len() > name_tl { - name_tl = display_name.len(); - } - } - // Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill - if tokio::fs::metadata(path.clone() + ".valid.windmill") - .await - .is_err() - { - if !to_be_installed_is_used { - windmill_queue::append_logs( - job_id, - w_id, - format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n"), - conn, - ) - .await; - to_be_installed_is_used = true; - } - windmill_queue::append_logs(job_id, w_id, format!("- {display_name}\n"), conn) - .await; - not_installed.push(NotInstalledDependency { - path, - custom_name, - short_name, - display_name, - }); - } - } - } - total_to_install = not_installed.len(); - if total_to_install == 0 { - return Ok(()); - } - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let is_not_pro = !matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Pro - ); - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if is_not_pro && matches!(install_fn, InstallStrategy::AllAtOnce(_)) { - windmill_queue::append_logs( - job_id, - w_id, - format!("\nLooking for packages on S3:\n"), - conn, - ) - .await; - } - - // Parallelism level (N) - let parallel_limit = // Semaphore will panic if value less then 1 - concurrent_downloads.clamp(1, 30); - - tracing::info!( - workspace_id = %w_id, - "Install parallel limit: {}, job: {}", - parallel_limit, - job_id - ); - - let mut handles = vec![]; - let semaphore = Arc::new(Semaphore::new(parallel_limit)); - let not_pulled = Arc::new(RwLock::new(vec![])); - // let mut handles = Vec::with_capacity(total_to_install); - for NotInstalledDependency { - // - path, - custom_name, - short_name, - display_name, - } in not_installed - { - let permit = semaphore.clone().acquire_owned().await; // Acquire a permit - - if let Err(_) = permit { - tracing::error!( - workspace_id = %w_id, - "Cannot acquire permit on semaphore, that can only mean that semaphore has been closed." - ); - break; - } - - let permit = permit.unwrap(); - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - let s3_pull_future = if is_not_pro { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - Some(crate::global_cache::pull_from_tar( - os, - path.clone(), - language_name.to_owned(), - custom_name.clone(), - platform_agnostic, - )) - } else { - None - } - } else { - None - }; - let child = { - if let InstallStrategy::Single(ref callback, ..) = install_fn { - let cmd = callback(RequiredDependency { - path: path.clone(), - custom_name: custom_name.clone(), - short_name: short_name.clone(), - })?; - tracing::debug!("{:?}", &cmd); - Some(start_child_process(cmd, &installer_executable_name, false).await?) - } else { - None - } - }; - - let ( - worker_name_2, - path_2, - display_name_2, - custom_name, - job_id_2, - w_id_2, - conn_2, - counter_arc, - language_name, - installer_executable_name, - not_pulled, - ) = ( - worker_name.to_owned(), - path.clone(), - display_name.clone(), - custom_name.clone(), - job_id.clone(), - w_id.to_owned(), - conn.clone(), - counter_arc.clone(), - language_name.to_owned(), - installer_executable_name.to_owned(), - not_pulled.clone(), - ); - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - let _ = language_name; - - let start = std::time::Instant::now(); - let handle = tokio::spawn(async move { - let _permit = permit; - - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(s3_pull_future) = s3_pull_future { - if let Err(e) = s3_pull_future.await { - tracing::info!( - workspace_id = %w_id_2, - "No tarball was found for {:?} on S3 or different problem occured {job_id_2}:\n{e}", - &custom_name.clone().unwrap_or(path) - ); - } else { - // TODO: Refactor - // Create a file to indicate that installation was successfull - let valid_path = path_2.clone() + ".valid.windmill"; - // This is atomic operation, meaning, that it either completes and dependency is valid, - // or it does not and dependency is invalid and will be reinstalled next run - if let Err(e) = File::create(&valid_path).await { - tracing::error!( - workspace_id = %w_id_2, - job_id = %job_id_2, - "Failed to create {}!\n{e}\n - This file needed for jobs to function", - valid_path - ); - }; - print_success( - true, - false, - &job_id_2, - &w_id_2, - &display_name_2, - name_tl, - counter_arc, - total_to_install, - start, - &conn_2, - ) - .await; - return; - } - } - - let Some(child) = child else { - let mut lock = not_pulled.write().await; - lock.push(RequiredDependency { - path: path_2.clone(), - custom_name: custom_name.clone(), - short_name: short_name.clone(), - }); - return; - }; - if let Err(e) = crate::handle_child::handle_child( - &job_id_2, - &conn_2, - // TODO: Return mem_peak - &mut 0, - // TODO: Return canceld_by_ref - &mut None, - child, - !*DISABLE_NSJAIL, - &worker_name_2, - &w_id_2, - &installer_executable_name, - None, - false, - &mut None, - None, - ) - .await - { - windmill_queue::append_logs( - &job_id_2, - &w_id_2, - format!("error while installing {}: {e:?}", &display_name_2), - &conn_2, - ) - .await; - } else { - // if let Some(cb) = postinstall_cb { - // if let Err(e) = cb(vec![RequiredDependency { - // path: path_2.clone(), - // custom_name: custom_name.clone(), - // short_name: short_name.clone(), - // }]) - // .await - // { - // tracing::error!( - // workspace_id = %w_id_2, - // job_id = %job_id_2, - // "Postinstall callback failed!\n{e}\n - // This might affect execution", - // ); - // } - // } - print_success( - false, - true, - &job_id_2, - &w_id_2, - &display_name_2, - name_tl, - counter_arc, - total_to_install, - start, - &conn_2, - ) - .await; - // TODO: Refactor - // Create a file to indicate that installation was successfull - let valid_path = path_2.clone() + ".valid.windmill"; - // This is atomic operation, meaning, that it either completes and dependency is valid, - // or it does not and dependency is invalid and will be reinstalled next run - if let Err(e) = File::create(&valid_path).await { - tracing::error!( - workspace_id = %w_id_2, - job_id = %job_id_2, - "Failed to create {}!\n{e}\n - This file needed for jobs to function", - valid_path - ); - }; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - tokio::spawn(async move { - if let Err(e) = crate::global_cache::build_tar_and_push( - os, - path_2, - language_name, - custom_name, - platform_agnostic, - ) - .await - { - tracing::warn!("failed to build tar and push: {e:?}"); - } - }); - } - } - } - }); - handles.push(handle); - } - - for handle in handles { - if let Err(e) = handle.await { - tracing::error!("Error joining handles: {e:?}"); - } - } - if !not_pulled.read().await.is_empty() { - if let InstallStrategy::AllAtOnce(ref callback, ..) = install_fn { - let not_pulled_copy = not_pulled.read().await.clone(); - windmill_queue::append_logs( - job_id, - w_id, - format!("\n\nFetching {} packages...\n", not_pulled_copy.len()), - &conn, - ) - .await; - let cmd = callback(not_pulled_copy.clone())?; - tracing::debug!("{:?}", &cmd); - let child = start_child_process(cmd, &installer_executable_name, false).await?; - let mut buf = "".to_owned(); - let pipe_stdout = if stdout_on_err { Some(&mut buf) } else { None }; - if let Err(e) = crate::handle_child::handle_child( - // &job_id, - &Uuid::nil(), - &conn, - // TODO: Return mem_peak - &mut 0, - // TODO: Return canceld_by_ref - &mut None, - child, - !*DISABLE_NSJAIL, - &worker_name, - &w_id, - &installer_executable_name, - None, - false, - &mut None, - pipe_stdout, - ) - .await - { - bail!(format!( - "error while installing dependencies: {e:?}\n{}", - buf - )); - } - { - postinstall_cb(not_pulled_copy.clone()).await?; - } - for RequiredDependency { path, custom_name: _custom_name, .. } in - not_pulled_copy.into_iter() - { - // TODO: Refactor - // Create a file to indicate that installation was successfull - let valid_path = path.clone() + ".valid.windmill"; - // This is atomic operation, meaning, that it either completes and dependency is valid, - // or it does not and dependency is invalid and will be reinstalled next run - if let Err(e) = File::create(&valid_path).await { - tracing::error!( - workspace_id = %w_id, - job_id = %job_id, - "Failed to create {}!\n{e}\n - This file needed for jobs to function", - valid_path - ); - }; - #[cfg(all(feature = "enterprise", feature = "parquet"))] - { - if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - let language_name = language_name.to_owned(); - tokio::spawn(async move { - if let Err(e) = crate::global_cache::build_tar_and_push( - os, - path, - language_name, - _custom_name, - platform_agnostic, - ) - .await - { - tracing::warn!("failed to build tar and push: {e:?}"); - } - }); - } - } - } - } - } - { - let total_time = total_time.elapsed().as_millis(); - windmill_queue::append_logs( - &job_id, - w_id, - format!( - "\nDone. Time spent on installation phase: {}ms\n", - total_time - ), - conn, - ) - .await; - } - Ok(()) -} - #[derive(Clone)] pub struct S3ModeWorkerData { pub client: AuthedClient, diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 0a6b13ddb9..5a779ad990 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, path::PathBuf, process::Stdio, sync::Arc}; +use std::{collections::HashMap, path::PathBuf, process::Stdio}; use anyhow::{anyhow, bail}; use async_recursion::async_recursion; @@ -21,11 +21,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, par_install_language_dependencies, - read_result, start_child_process, OccupancyMetrics, RequiredDependency, + create_args_and_out_file, get_reserved_variables, read_result, start_child_process, + OccupancyMetrics, }, - handle_child, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, - JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + handle_child, + universal_pkg_installer::{par_install_language_dependencies_all_at_once, RequiredDependency}, + COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_REPOSITORY_DIR, + MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; use windmill_common::client::AuthedClient; @@ -275,14 +277,15 @@ async fn install<'a>( ); Ok(RequiredDependency { path, - custom_name: Some(format!("{group_id}:{artifact_id}:{version}")), - short_name: Some(format!("{artifact_id}:{version}")), + _s3_handle: format!("{group_id}:{artifact_id}:{version}"), + display_name: format!("{artifact_id}:{version}"), + custom_payload: (), }) } _ => anyhow::bail!("{line} is not parsable"), } }) - .collect::>>()?; + .collect::>>>()?; let classpath = deps .clone() @@ -307,14 +310,14 @@ async fn install<'a>( let job_dir = job_dir.to_owned(); let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4()); let fetch_dir2 = fetch_dir.clone(); - par_install_language_dependencies( + par_install_language_dependencies_all_at_once( deps, "java", "java", true, *JAVA_CONCURRENT_DOWNLOADS, true, - crate::common::InstallStrategy::AllAtOnce(Arc::new(move |dependencies| { + move |dependencies| { let mut cmd = Command::new(if cfg!(windows) { "java" } else { @@ -322,12 +325,8 @@ async fn install<'a>( }); let artifacts = dependencies .into_iter() - .map(|e| { - e.custom_name.ok_or(anyhow::anyhow!( - "Internal Error: Artifact name should be Some!" - )) - }) - .collect::>>()?; + .map(|e| e._s3_handle) + .collect::>(); cmd.env_clear() .current_dir(&job_dir) .env("PATH", PATH_ENV.as_str()) @@ -385,7 +384,7 @@ async fn install<'a>( } Ok(cmd) - })), + }, async move |_| { move_to_repository(&fetch_dir2, 0).await?; remove_dir_all(&fetch_dir2).await?; @@ -419,6 +418,7 @@ async fn install<'a>( &job.id, &job.workspace_id, worker_name, + !*DISABLE_NSJAIL, conn, ) .await?; diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 3dfeba5c42..e917e32fba 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -13,6 +13,9 @@ mod bash_executor; #[cfg(feature = "java")] mod java_executor; +#[cfg(feature = "ruby")] +mod ruby_executor; + mod bun_executor; pub mod common; mod config; @@ -52,6 +55,7 @@ pub mod result_processor; mod rust_executor; mod sanitized_sql_params; mod schema; +mod universal_pkg_installer; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs new file mode 100644 index 0000000000..944ee5ee9c --- /dev/null +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -0,0 +1,899 @@ +use std::{collections::HashMap, process::Stdio}; + +use anyhow::anyhow; +use const_format::concatcp; +use itertools::Itertools; +use regex::Regex; +use tokio::{ + fs::{self, File}, + io::{AsyncReadExt, AsyncWriteExt}, + process::Command, +}; +use url::Url; +use uuid::Uuid; +use windmill_common::{ + client::AuthedClient, + error::Error, + utils::calculate_hash, + worker::{write_file, Connection, RubyAnnotations}, +}; +use windmill_parser::Arg; +use windmill_parser_ruby::parse_ruby_signature; +use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; + +use crate::{ + common::{ + create_args_and_out_file, get_reserved_variables, read_result, start_child_process, + OccupancyMetrics, + }, + handle_child::{self}, + universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency}, + DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, +}; +lazy_static::lazy_static! { + static ref RUBY_CONCURRENT_DOWNLOADS: usize = std::env::var("RUBY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); + static ref RUBY_PATH: String = std::env::var("RUBY_PATH").unwrap_or_else(|_| "/usr/bin/ruby".to_string()); + static ref BUNDLE_PATH: String = std::env::var("RUBY_BUNDLE_PATH").unwrap_or_else(|_| "/usr/bin/bundle".to_string()); + static ref GEM_PATH: String = std::env::var("RUBY_GEM_PATH").unwrap_or_else(|_| "/usr/bin/gem".to_string()); + static ref RUBY_PROXY_ENVS: Vec<(String, String)> = { + PROXY_ENVS + .clone() + .into_iter() + .map(|(k, v)| (k.to_lowercase(), v)) + .collect() + }; +} + +const NSJAIL_CONFIG_RUN_RUBY_CONTENT: &str = include_str!("../nsjail/run.ruby.config.proto"); +const NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT: &str = + include_str!("../nsjail/download.ruby.config.proto"); +const NSJAIL_CONFIG_LOCK_RUBY_CONTENT: &str = include_str!("../nsjail/lock.ruby.config.proto"); + +#[cfg(debug_assertions)] +const DEV_CONF_NSJAIL: &'static str = r#" +# Mount nix store for nixos to work properly +mount { + src: "/nix/store" + dst: "/nix/store" + is_bind: true + mandatory: false +} +"#; + +#[cfg(not(debug_assertions))] +const DEV_CONF_NSJAIL: &'static str = ""; + +#[allow(dead_code)] +pub(crate) struct JobHandlerInput<'a> { + pub base_internal_url: &'a str, + pub canceled_by: &'a mut Option, + pub client: &'a AuthedClient, + pub parent_runnable_path: Option, + pub conn: &'a Connection, + pub envs: HashMap, + pub inner_content: &'a str, + pub job: &'a MiniPulledJob, + pub job_dir: &'a str, + pub mem_peak: &'a mut i32, + pub occupancy_metrics: &'a mut OccupancyMetrics, + pub requirements_o: Option<&'a String>, + pub shared_mount: &'a str, + pub worker_name: &'a str, +} + +pub async fn handle_ruby_job<'a>( + mut args: JobHandlerInput<'a>, +) -> Result, Error> { + // --- Prepare --- + { + prepare(&args).await?; + } + // --- Gen Lockfile --- + + let lockfile = resolve( + &args.job.id, + args.inner_content, + args.mem_peak, + args.canceled_by, + args.job_dir, + args.conn, + args.worker_name, + &args.job.workspace_id, + ) + .await?; + + // --- Install --- + + let ir = install(&mut args, lockfile).await?; + + // --- Execute --- + { + run(&mut args, ir).await?; + } + // --- Retrieve results --- + { + read_result(&args.job_dir, None).await + } +} +pub async fn prepare<'a>( + JobHandlerInput { + job, + conn, + job_dir, + inner_content, + client, + // + .. + }: &JobHandlerInput<'a>, +) -> Result<(), Error> { + create_args_and_out_file(&client, job, job_dir, conn).await?; + File::create(format!("{}/main.rb", job_dir)) + .await? + .write_all(&wrap(inner_content)?.into_bytes()) + .await?; + + let mini_wm_path = format!("{RUBY_CACHE_DIR}/gems/windmill-internal/windmill"); + if !std::fs::metadata(&mini_wm_path).is_ok() { + fs::create_dir_all(&mini_wm_path).await?; + + // Create inline.rb for dependency management (bundler/inline compatibility) + File::create(format!("{}/inline.rb", &mini_wm_path)) + .await? + .write_all(&wrap( + r#" +class GemfileProxy + def initialize + @gem_calls = [] + end + + def source(arg = nil, &block) + if arg.is_a?(String) && block_given? + # Handle the case where the argument is a string and a block is given + gemfile(&block) + elsif arg.is_a?(String) + # Handle the case where the argument is a string and no block is given + # Do nothing in this case + else + raise ArgumentError, "Invalid argument or block" + end + end + + def group(*args, **kwargs) + # Handle group calls if needed + end + + def ruby(*args, **kwargs) + warn "Custom Ruby runtime specifications (engine, engine_version, patchlevel) are not supported in this environment. Using system ruby" + end + + def gem(name, version = nil, require: nil, **kwargs) + @gem_calls << { name: name, version: version, require: require, kwargs: kwargs } + + case require + when false + # Skip requiring + when nil, true + begin + Kernel.require(name) + rescue LoadError => e + warn "WARN: Gem '#{name}' failed to auto-require. Try either:" + warn " `:require => false` or specify exact require path" + warn "Warn details: #{e.class}: #{e.message}" + end + else + begin + Kernel.require(require.to_s) + rescue LoadError => e + warn "WARN: Gem '#{name}' failed to auto-require. Try either:" + warn " `:require => false` or specify exact require path" + warn "Warn details: #{e.class}: #{e.message}" + end + end + end + + + def call_block(&block) + instance_eval(&block) if block_given? + end +end + +def gemfile(&block) + proxy = GemfileProxy.new + proxy.call_block(&block) +end + +def main +end +"# + )?.into_bytes()) + .await?; + + // Create mini.rb for Windmill client methods + File::create(format!("{}/mini.rb", &mini_wm_path)) + .await? + .write_all( + &wrap( + r##" +require 'net/http' +require 'uri' +require 'json' + +# Windmill mini client methods +def get_variable(path) + base_url = ENV['BASE_INTERNAL_URL'] + workspace = ENV['WM_WORKSPACE'] + token = ENV['WM_TOKEN'] + + uri = URI("#{base_url}/api/w/#{workspace}/variables/get_value/#{path}") + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = uri.scheme == 'https' + + request = Net::HTTP::Get.new(uri) + request['Authorization'] = "Bearer #{token}" + + response = http.request(request) + + if response.code == '200' + JSON.parse(response.body) + else + raise "Failed to get variable #{path}: #{response.code} #{response.message}" + end +end + +def get_resource(path) + base_url = ENV['BASE_INTERNAL_URL'] + workspace = ENV['WM_WORKSPACE'] + token = ENV['WM_TOKEN'] + + uri = URI("#{base_url}/api/w/#{workspace}/resources/get_value_interpolated/#{path}") + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = uri.scheme == 'https' + + request = Net::HTTP::Get.new(uri) + request['Authorization'] = "Bearer #{token}" + + response = http.request(request) + + if response.code == '200' + JSON.parse(response.body) + else + raise "Failed to get resource #{path}: #{response.code} #{response.message}" + end +end + +def main +end +"##, + )? + .into_bytes(), + ) + .await?; + } + Ok(()) +} + +pub async fn resolve<'a>( + job_id: &Uuid, + inner_content: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + conn: &Connection, + worker_name: &str, + w_id: &str, +) -> Result { + lazy_static::lazy_static! { + static ref BUNDLER_RE: Regex = Regex::new(r#"(?m)^\s*require\s*['"]bundler/inline['"]"#).unwrap(); + static ref UNSUPPORTED_SOURCE_RE: Regex = Regex::new(r#"(?m)(?^\s*source\s*['"]\S+://(?\S+):(?\S+)@.*['"]\s*\n)"#).unwrap(); + static ref SOURCES_RE: Regex = Regex::new(r#"\S+://(?\S+:\S+)@(?\S*)"#).unwrap(); + } + + if BUNDLER_RE.find(&inner_content).is_some() { + return Err(anyhow!( + "Detected `require 'bundler/inline'` - please replace with `require 'windmill/inline'`. +Your Gemfile syntax will continue to work as-is." + ) + .into()); + } + + let gemfile: String = windmill_parser_ruby::parse_ruby_requirements(&inner_content)? + .lines() + .map(str::trim) + .filter(|s| !s.is_empty() && !s.starts_with('#')) + .join("\n"); + + if let Some(caps) = UNSUPPORTED_SOURCE_RE.captures(&gemfile) { + if let (Some(src_m), Some(usr_m), Some(passwd_m)) = + (caps.name("src"), caps.name("usr"), caps.name("passwd")) + { + let cause = src_m + .as_str() + .replace(usr_m.as_str(), "REDACTED") + .replace(passwd_m.as_str(), "REDACTED"); + + return Err(anyhow!("!> {cause} - inline source credentials are not supported at the moment. Please set up credentials in instance settings, if you do not have access to it, contact your administrator. If you need this feature please let us know.").into()); + } + } + + let mut file = File::create(job_dir.to_owned() + "/Gemfile").await?; + file.write_all(&gemfile.as_bytes()).await?; + + let req_hash = format!("ruby-{}", calculate_hash(&gemfile)); + if let Some(db) = conn.as_sql() { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + return Ok(cached); + } + } + let lock = { + append_logs( + job_id, + w_id, + format!("\n--- RESOLVING LOCKFILE ---\n"), + conn, + ) + .await; + + let mut cmd = if !cfg!(windows) && !*DISABLE_NSJAIL { + let nsjail_proto = format!("{}.lock.config.proto", Uuid::new_v4()); + let _ = write_file( + &job_dir, + &nsjail_proto, + &NSJAIL_CONFIG_LOCK_RUBY_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), + )?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.args(vec!["--config", &nsjail_proto, "--", BUNDLE_PATH.as_str()]); + cmd + } else if cfg!(windows) { + Command::new("bundle.bat") + } else { + Command::new(BUNDLE_PATH.as_str()) + }; + + cmd.env_clear() + .current_dir(job_dir.to_owned()) + .envs(vec![ + ("PATH".to_owned(), PATH_ENV.clone()), + // Make sure there is nothing written to actual home + // This way we keep everything clean, organized and maintable + ("HOME".to_owned(), RUBY_CACHE_DIR.to_owned()), + ]) + .envs(RUBY_PROXY_ENVS.clone()); + + for repo in RUBY_REPOS.read().await.clone().unwrap_or_default() { + if let (Some(url), usr, Some(passwd)) = + (repo.domain(), repo.username(), repo.password()) + { + cmd.env( + format!( + "BUNDLE_{}", + url + // Align with bundlers' format. Example: + // BUNDLE_GEM1__EXAMPLE__COM + .replace(".", "__") + .to_uppercase() + // Remove trailing slashes + .replace("/", "") + ), + // BUNDLE_GEM1__EXAMPLE__COM=admin:123 + format!("{usr}:{passwd}"), + ); + } + } + cmd.args(&["lock", "--retry", "2"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + let child = start_child_process(cmd, "bundle", false).await?; + + handle_child::handle_child( + job_id, + conn, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + w_id, + "bundle", + None, + false, + &mut None, + // Some(&mut stdout), + None, + ) + .await?; + + let path_lock = format!("{job_dir}/Gemfile.lock"); + let mut file = File::open(path_lock).await?; + let mut req_content = "".to_string(); + file.read_to_string(&mut req_content).await?; + req_content + }; + + if let Some(db) = conn.as_sql() { + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + req_hash, + lock.clone(), + ).fetch_optional(db).await?; + } + + append_logs(job_id, w_id, format!("\n{}", &lock), conn).await; + Ok(lock) +} + +struct InstallRes { + /// "/tmp/windmill/cache/ruby/gems/ab..xz-name-1.0.0/gems/name-1.0.0/lib:/tmp/windmill/..." + rubylib: String, + + /// ["/tmp/windmill/cache/ruby/gems/ab..xz-name-1.0.0", "/tmp/windmill/..."] + top_level_paths: Vec, +} + +async fn install<'a>( + JobHandlerInput { + worker_name, + job, + conn, + job_dir, + inner_content, + envs, + client, + parent_runnable_path, + .. + }: &mut JobHandlerInput<'a>, + lockfile: String, +) -> Result { + #[derive(Debug, Clone)] + enum CurrentSource { + GIT { remote: Option, revision: Option }, + GEM { remote: Option }, + INIT, + } + + #[derive(Debug, Clone)] + enum FinalSource { + #[allow(dead_code)] + GIT { + remote: String, + revision: String, + }, + GEM { + remote: String, + }, + } + + #[derive(Clone, Debug)] + struct CustomPayload { + source: FinalSource, + version: String, + pkg: String, + } + + let mut deps = vec![]; + let mut last_source = CurrentSource::INIT; + 'lock_lines: for line in lockfile.lines() { + // Trimed line + let tl = line.trim_start(); + if tl.starts_with("GIT") { + last_source = CurrentSource::GIT { remote: None, revision: None }; + continue 'lock_lines; + } else if tl.starts_with("GEM") { + last_source = CurrentSource::GEM { remote: None }; + continue 'lock_lines; + } + + const REMOTE: &str = "remote: "; + if line.contains(REMOTE) { + match &mut last_source { + CurrentSource::GIT { remote, .. } => remote.replace(tl.replace(REMOTE, "")), + CurrentSource::GEM { remote } => remote.replace(tl.replace(REMOTE, "")), + CurrentSource::INIT => return Err(Error::InternalErr("BUG on installation stage: detected initial enum variant at the stage it was not supposed to be at. Please report this issue.".to_owned())), + }; + continue 'lock_lines; + } + + const REVISION: &str = "revision: "; + if line.contains(REVISION) { + if let CurrentSource::GIT { revision, .. } = &mut last_source { + revision.replace(tl.replace(REVISION, "")); + } + continue 'lock_lines; + } + + // Check on untrimed line + if !line.starts_with(" ") || line.starts_with(" ") { + // "1234" "123456" + // For example: + // + // GIT + // remote: https://github.com/rails/rails.git + // revision: 8cb4f8ceffe375f64052579cd7a580bfdaf36d61 + // specs: + // actioncable (8.1.0.alpha) + // actionpack (= 8.1.0.alpha) + // activesupport (= 8.1.0.alpha) + // nio4r (~> 2.0) + // websocket-driver (>= 0.6.1) + // zeitwerk (~> 2.6) + // actionmailbox (8.1.0.alpha) + // actionpack (= 8.1.0.alpha) + // activejob (= 8.1.0.alpha) + // activerecord (= 8.1.0.alpha) + // activestorage (= 8.1.0.alpha) + // activesupport (= 8.1.0.alpha) + // mail (>= 2.8.0) + // actionmailer (8.1.0.alpha) + // ... + // + // In this snippet we only want actioncable, actionmailbox and actionmailer + // + // If it got the hit than that's not it, just skip + continue 'lock_lines; + } + + let buf = tl.replace(['(', ')'], ""); + let &[pkg, version, ..] = buf.split_whitespace().collect_vec().as_slice() else { + return Err(anyhow!("Cannot determine version and package name for: {}", tl).into()); + }; + + // Strip platform from version. E.g: + // 1.18.9-x86_64-darwin -> 1.18.9 + let version = version.split('-').next().unwrap_or(version); + + let custom_payload = CustomPayload { + version: version.into(), + pkg: pkg.into(), + source: match last_source.clone() { + CurrentSource::GIT { remote, revision } => FinalSource::GIT { + remote: remote.unwrap_or_default(), + revision: revision.unwrap_or_default(), + }, + CurrentSource::GEM { remote } => { + FinalSource::GEM { remote: remote.unwrap_or_default() } + }, + CurrentSource::INIT => return Err(Error::InternalErr("BUG on installation stage: detected initial enum variant at the stage it was not supposed to be at. Please report this issue.".to_owned())), + }, + }; + + // Calculate hash based on source + // Obfuscation is essential because remote can include credentials + let mut hash = calculate_hash(&format!("{:?}", custom_payload.clone())); + + // Use 32 characters which is 256 bits + hash.truncate(32); + + // Final format: + // 123...zx-activesupport-8.0.2 + // ^^^^^^^^ hash based on source and type (GEM or GIT) + let handle = format!("{}-{}-{}", hash, pkg, version); + let path = format!("{RUBY_CACHE_DIR}/gems/{}", &handle); + + deps.push(RequiredDependency { + path, + _s3_handle: handle, + display_name: tl.to_owned(), + custom_payload, + }); + } + + let job_dir = job_dir.to_owned(); + let jailed = !cfg!(windows) && !*DISABLE_NSJAIL; + let RubyAnnotations { verbose } = RubyAnnotations::parse(&inner_content); + let repos = RUBY_REPOS.read().await.clone().unwrap_or_default(); + let (envs, reserved_variables) = ( + envs.clone(), + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?, + ); + par_install_language_dependencies_seq( + deps.clone(), + "ruby", + "gem", + false, + *RUBY_CONCURRENT_DOWNLOADS, + // true, + // crate::common::InstallStrategy::Single(Arc::new(move |dependency| { + move |dependency| { + let mut cmd = if jailed { + std::fs::create_dir_all(&dependency.path)?; + let nsjail_proto = format!("{}.download.config.proto", Uuid::new_v4()); + let _ = write_file( + &job_dir, + &nsjail_proto, + &NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT + .replace("{TARGET}", &dependency.path) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), + )?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.args(vec!["--config", &nsjail_proto, "--", GEM_PATH.as_str()]); + cmd + } else { + Command::new(if cfg!(windows) { + "gem.cmd" + } else { + GEM_PATH.as_str() + }) + }; + cmd.env_clear() + .current_dir(&job_dir) + .envs(vec![ + ("PATH".to_owned(), PATH_ENV.clone()), + // Make sure there is nothing written to actual home + // This way we keep everything clean, organized and maintable + ("HOME".to_owned(), RUBY_CACHE_DIR.to_owned()), + ]) + .envs(RUBY_PROXY_ENVS.clone()); + + // Figure out source + let source = { + let source = match dependency.custom_payload.source { + FinalSource::GIT { .. } => return Err(anyhow!("GIT is not supported").into()), + FinalSource::GEM { remote } => remote, + } + .trim() + .to_owned(); + + let src_url = Url::parse(&source)?; + repos + .iter() + .find(|inst_url| { + inst_url.domain().is_some() + && (inst_url.domain(), inst_url.path()) + == (src_url.domain(), src_url.path()) + }) + .map(Url::to_string) + .unwrap_or(source) + }; + + cmd.args(&[ + "install", + &dependency.custom_payload.pkg, + "--version", + &dependency.custom_payload.version, + "--install-dir", + &dependency.path, + // Disable transitive dependencies! + "--ignore-dependencies", + "--no-document", + // Do not use default sources + "--clear-sources", + // Specify the one from lockfile + "--source", + source.as_str(), + ]) + .envs(envs.clone()) + .envs(reserved_variables.clone()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + if !verbose { + // Make it output minimal info + cmd.args(&["--quiet", "--silent"]); + } + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "SYSTEMDRIVE", + std::env::var("SYSTEMDRIVE").unwrap_or_else(|_| String::from("C:")), + ); + } + + Ok(cmd) + }, + // async move |_| Ok(()), + &job.id, + &job.workspace_id, + worker_name, + jailed, + conn, + ) + .await?; + + let mut res = InstallRes { + rubylib: deps + .iter() + .map(|rd| { + format!( + "{}/gems/{}-{}/lib", + rd.path, rd.custom_payload.pkg, rd.custom_payload.version + ) + }) + .join(":"), + top_level_paths: deps.iter().map(|rd| rd.path.clone()).collect_vec(), + }; + // Include builtin windmill client + { + const WM_INTERNAL: &str = concatcp!(RUBY_CACHE_DIR, "/gems/windmill-internal"); + res.top_level_paths.push(WM_INTERNAL.to_owned()); + res.rubylib += format!(":{WM_INTERNAL}").as_str(); + } + Ok(res) +} + +async fn run<'a>( + JobHandlerInput { + occupancy_metrics, + mem_peak, + canceled_by, + worker_name, + job, + conn, + job_dir, + shared_mount, + client, + envs, + base_internal_url, + parent_runnable_path, + .. + }: &mut JobHandlerInput<'a>, + InstallRes { rubylib, top_level_paths }: InstallRes, +) -> Result<(), Error> { + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; + + let child = if !cfg!(windows) && !*DISABLE_NSJAIL { + append_logs( + &job.id, + &job.workspace_id, + format!("\n--- ISOLATED RUBY CODE EXECUTION ---\n"), + conn, + ) + .await; + let shared_deps = top_level_paths + .into_iter() + .map(|pp| { + format!( + r#" +mount {{ + src: "{pp}" + dst: "{pp}" + is_bind: true + rw: false +}} + "# + ) + }) + .join("\n"); + + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_RUBY_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{SHARED_MOUNT}", &shared_mount) + .replace("{SHARED_DEPENDENCIES}", &shared_deps) + .replace("{DEV}", DEV_CONF_NSJAIL) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + )?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.env_clear() + .current_dir(job_dir) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("RUBYLIB", rubylib.as_str()) + .envs(envs) + .envs(reserved_variables) + .envs(RUBY_PROXY_ENVS.clone()) + .envs(PROXY_ENVS.clone()) + .args(vec![ + "--config", + "run.config.proto", + "--", + RUBY_PATH.as_str(), + "main.rb", + ]); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + + start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? + } else { + append_logs( + &job.id, + &job.workspace_id, + format!("\n--- RUBY CODE EXECUTION ---\n"), + conn, + ) + .await; + + let mut cmd = Command::new(if cfg!(windows) { + "ruby.exe" + } else { + RUBY_PATH.as_str() + }); + + #[cfg(windows)] + let rubylib = rubylib.replace(":", ";"); + + cmd.env_clear() + .current_dir(job_dir.to_owned()) + .env("PATH", PATH_ENV.as_str()) + .env("RUBYLIB", rubylib.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(reserved_variables) + .envs(RUBY_PROXY_ENVS.clone()) + .envs(PROXY_ENVS.clone()) + .envs(envs); + + cmd.args(&["main.rb"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + start_child_process(cmd, "ruby", false).await? + }; + handle_child::handle_child( + &job.id, + conn, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + &job.workspace_id, + "ruby", + job.timeout, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + Ok(()) +} +fn wrap(inner_content: &str) -> Result { + let sig = parse_ruby_signature(inner_content)?; + let spread = sig + .args + .clone() + .into_iter() + .map(|Arg { name, .. }| format!("a[\"{}\"]", name)) + .collect_vec() + .join(","); + Ok( + r#"INNER_CONTENT + +require 'json' +a = JSON.parse(File.read("args.json")) +res = main(SPREAD) +File.open("result.json", "w") do |file| + file.write(JSON.generate(res)) +end + "# + .replace("INNER_CONTENT", inner_content) + .replace("SPREAD", &spread), // .replace("TRANSFORM", transform) + ) +} diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs new file mode 100644 index 0000000000..1e4a8c2d39 --- /dev/null +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -0,0 +1,767 @@ +use std::sync::Arc; + +use anyhow::bail; +use itertools::Itertools; +use process_wrap::tokio::TokioChildWrapper; +use tokio::{ + fs::File, + process::Command, + sync::{broadcast::Sender, OwnedSemaphorePermit, RwLock, Semaphore}, + task::JoinHandle, +}; +use uuid::Uuid; +use windmill_common::{error, worker::Connection}; + +use crate::{common::start_child_process, DISABLE_NSJAIL}; + +#[derive(Clone, Debug)] +pub struct RequiredDependency { + /// Expected directory of dependency in cache + /// For example: + /// /tmp/windmill/cache/python_311/rich==0.0.0 + /// IMPORTANT!: path should not end with '/' + pub path: String, + /// Name to use for S3 tars + pub _s3_handle: String, + /// Display name + /// Name that will be used for console output and logging + pub display_name: String, + /// Custom payload can be used for passing information easily. If you wish not to use it just leave '()' + pub custom_payload: T, +} + +#[allow(dead_code)] +pub enum InstallStrategy { + /// Will invoke callback to install single dependency + Single(Arc) -> Result + Send + Sync>), + /// Will try to pull S3 first and will invoke closure to install the rest + AllAtOnce(Arc>>>), +} + +/// # General +/// +/// Languages that compile usually include dependencies in final executable. +/// When dynamic languages do not and runtime dependencies are provided separately. +/// +/// This helper assumes that the language is dynamic. +/// Python, Ruby, Java are dynamic and they can use this helper. +/// +/// # Features +/// +/// This helper will install all specified dependencies in parallel and if it is EE, cache to S3 +/// It has atomic success file, allowing to distinguish failed installations from succesfull. +/// +/// Besides that it provides console output and does logging. +/// +/// # Usage +/// +/// Most important arguments in this helper are `deps` and `install_fn` +/// +/// In `deps` you specify all dependencies that are needed to be on worker in order to execute script. +/// You don't know which are actually installed and which are not. +/// +/// `deps` is a vector of RequiredDependency. Check [RequiredDependency] for more context. +/// +/// After `deps` are provided helper will check each dependency and check if it is in cache, if not it will try to pull from S3 +/// and if it does not work either, it will invoke `install_fn` closure. +/// Closure arguments has dependency name as well as its expected path in cache. +/// Closure should return Command that will install dependency to asked place. +#[allow(dead_code)] +pub async fn par_install_language_dependencies_all_at_once< + 'a, + T: Clone + std::marker::Send + Sync + 'a + 'static, +>( + deps: Vec>, + _language_name: &'a str, + installer_executable_name: &'a str, + _platform_agnostic: bool, + concurrent_downloads: usize, + stdout_on_err: bool, + callback: impl Fn(Vec>) -> Result + + Send + + Sync + + 'static, + postinstall_cb: impl AsyncFn(Vec>) -> Result<(), error::Error>, + job_id: &'a Uuid, + w_id: &'a str, + worker_name: &'a str, + jailed: bool, + conn: &'a Connection, +) -> anyhow::Result<()> { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let is_not_pro = !matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ); + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if is_not_pro { + windmill_queue::append_logs( + job_id, + w_id, + format!("\nLooking for packages on S3:\n"), + conn, + ) + .await; + } + let total_time = std::time::Instant::now(); + let (missing, name_max_length) = filter_to_missing(deps, job_id, w_id, jailed, conn).await?; + if missing.is_empty() { + return Ok(()); + } + let to_batch_install = Arc::new(RwLock::new(vec![])); + let handles = spawn_wrapped_installation_threads( + missing, + name_max_length, + InstallStrategy::AllAtOnce(to_batch_install.clone()), + installer_executable_name, + concurrent_downloads, + job_id, + w_id, + worker_name, + conn, + _language_name, + _platform_agnostic, + ) + .await?; + let installation_res = process_handles(handles, w_id).await; + if !to_batch_install.read().await.is_empty() { + let for_all_at_once_copy = to_batch_install.read().await.clone(); + windmill_queue::append_logs( + job_id, + w_id, + format!("\n\nFetching {} packages...\n", for_all_at_once_copy.len()), + &conn, + ) + .await; + let cmd = callback(for_all_at_once_copy.clone())?; + let child = start_child_process(cmd, &installer_executable_name, false).await?; + let mut buf = "".to_owned(); + let pipe_stdout = if stdout_on_err { Some(&mut buf) } else { None }; + + if let Err(e) = crate::handle_child::handle_child( + &(if pipe_stdout.is_some() { + Uuid::nil() + } else { + *job_id + }), + conn, + // TODO: Return mem_peak + &mut 0, + // TODO: Return canceld_by_ref + &mut None, + child, + !*DISABLE_NSJAIL, + &worker_name, + &w_id, + &installer_executable_name, + None, + false, + &mut None, + pipe_stdout, + ) + .await + { + bail!(format!( + "\nerror while installing dependencies: {e:?}\n{}", + buf + )); + } + { + postinstall_cb(for_all_at_once_copy.clone()).await?; + } + for RequiredDependency { path, _s3_handle, .. } in for_all_at_once_copy.into_iter() { + mark_success(path.clone(), job_id, w_id).await; + #[cfg(all(feature = "enterprise", feature = "parquet"))] + { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + let language_name = _language_name.to_owned(); + tokio::spawn(async move { + if let Err(e) = crate::global_cache::build_tar_and_push( + os, + path, + language_name, + Some(_s3_handle), + _platform_agnostic, + ) + .await + { + tracing::warn!("failed to build tar and push: {e:?}"); + } + }); + } + } + } + } + finish_installation(total_time, job_id, w_id, conn).await; + installation_res +} + +/// # General +/// +/// Languages that compile usually include dependencies in final executable. +/// When dynamic languages do not and runtime dependencies are provided separately. +/// +/// This helper assumes that the language is dynamic. +/// Python, Ruby, Java are dynamic and they can use this helper. +/// +/// # Features +/// +/// This helper will install all specified dependencies in parallel and if it is EE, cache to S3 +/// It has atomic success file, allowing to distinguish failed installations from succesfull. +/// +/// Besides that it provides console output and does logging. +/// +/// # Usage +/// +/// Most important arguments in this helper are `deps` and `install_fn` +/// +/// In `deps` you specify all dependencies that are needed to be on worker in order to execute script. +/// You don't know which are actually installed and which are not. +/// +/// `deps` is a vector of RequiredDependency. Check [RequiredDependency] for more context. +/// +/// After `deps` are provided helper will check each dependency and check if it is in cache, if not it will try to pull from S3 +/// and if it does not work either, it will invoke `install_fn` closure. +/// Closure arguments has dependency name as well as it`s expected path in cache. +/// Closure should return Command that will install dependency to asked place. +#[allow(dead_code)] +pub async fn par_install_language_dependencies_seq< + 'a, + T: Clone + std::marker::Send + Sync + 'a + 'static, +>( + deps: Vec>, + _language_name: &'a str, + installer_executable_name: &'a str, + _platform_agnostic: bool, + concurrent_downloads: usize, + callback: impl Fn(RequiredDependency) -> Result + Send + Sync + 'static, + job_id: &'a Uuid, + w_id: &'a str, + worker_name: &'a str, + jailed: bool, + conn: &'a Connection, +) -> anyhow::Result<()> { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let is_not_pro = !matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ); + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if is_not_pro { + windmill_queue::append_logs( + job_id, + w_id, + format!("\nLooking for packages on S3:\n"), + conn, + ) + .await; + } + let total_time = std::time::Instant::now(); + let (missing, name_max_length) = filter_to_missing(deps, job_id, w_id, jailed, conn).await?; + if missing.is_empty() { + return Ok(()); + } + let handles = spawn_wrapped_installation_threads( + missing, + name_max_length, + InstallStrategy::Single(Arc::new(callback)), + installer_executable_name, + concurrent_downloads, + job_id, + w_id, + worker_name, + conn, + _language_name, + _platform_agnostic, + ) + .await?; + + let installation_res = process_handles(handles, w_id).await; + finish_installation(total_time, job_id, w_id, conn).await; + installation_res +} + +type NameMaxLength = usize; +async fn filter_to_missing<'a, T: Clone + std::marker::Send + Sync + 'a + 'static>( + mut deps: Vec>, + job_id: &Uuid, + w_id: &str, + jailed: bool, + conn: &Connection, +) -> anyhow::Result<(Vec>, NameMaxLength)> { + // Unique to flatten all same values + deps = deps.into_iter().unique_by(|rd| rd.path.clone()).collect(); + // Total to install + let mut missing = vec![]; + // Name max length + let mut name_ml = 0; + for rd in deps.into_iter() { + let display_name = rd.display_name.clone(); + if rd.path.ends_with("/") { + anyhow::bail!("Internal error: path should not end with '/'") + } + { + // Later will help us align text in log console + if display_name.len() > name_ml { + name_ml = rd.display_name.len(); + } + } + // Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill + if tokio::fs::metadata(rd.path.clone() + ".valid.windmill") + .await + .is_err() + { + missing.push(rd); + } + } + if !missing.is_empty() { + windmill_queue::append_logs( + job_id, + w_id, + if jailed { + format!("\n--- ISOLATED INSTALLATION ---\n\nTo be installed:\n\n") + } else { + format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n") + }, + conn, + ) + .await; + let to_log = missing + .iter() + .map(|rd| format!("- {}", &rd.display_name)) + .join("\n") + + "\n"; + + windmill_queue::append_logs(job_id, w_id, to_log, conn).await; + } + Ok((missing, name_ml)) +} + +enum Action { + Install(Box), + AddToBulk(Arc>>>), +} +struct TaskKiller(tokio::sync::broadcast::Sender); +impl Drop for TaskKiller { + fn drop(&mut self) { + // We don't care if it actually stopped anything or not. + // In some cases it will fire and actually stop + // In other cases do nothing + #[allow(unused_must_use)] + self.0.send("Installation failed".to_owned()); + } +} +// Callback will be invoked only if could not pull from S3 +async fn spawn_wrapped_installation_threads< + 'a, + T: Clone + std::marker::Send + Sync + 'a + 'static, +>( + missing: Vec>, + name_ml: usize, + strategy: InstallStrategy, + installer_executable_name: &str, + concurrent_downloads: usize, + job_id: &Uuid, + w_id: &str, + worker_name: &str, + conn: &Connection, + _language_name: &str, + _platform_agnostic: bool, +) -> anyhow::Result<( + Vec>>, + tokio::sync::broadcast::Sender<()>, +)> { + // Semaphore will panic if value less then 1 + let parallel_limit = concurrent_downloads.clamp(1, 30); + tracing::info!( + workspace_id = %w_id, + "Install parallel limit: {}, job: {}", + parallel_limit, + job_id + ); + + let (mut handles, semaphore, total_to_install, counter_arc) = ( + vec![], + Arc::new(Semaphore::new(parallel_limit)), + missing.len(), + Arc::new(tokio::sync::Mutex::new(0)), + ); + + // Pretty sensitive. Single drop will fail installation + let (kill_tx, ..) = tokio::sync::broadcast::channel::(10); + // ^^^^^^^ Original will go into listen_to_cancel + // + // But before that it will be used to populate t and r for every thread + let trxs: Vec<( + tokio::sync::broadcast::Sender, + tokio::sync::broadcast::Receiver, + )> = (0..missing.len()) + .map(|_| (kill_tx.clone(), kill_tx.subscribe())) + .collect(); + + // Listen to cancel events and stop jobs + // NOTE: + // once closer is dropped, the listener will be closed + let closer = listen_to_cancel(job_id.clone(), w_id.to_owned(), conn.clone(), kill_tx).await; + + for (dep, (kill_tx, mut kill_rx)) in missing.into_iter().zip(trxs) { + let permit = semaphore.clone().acquire_owned().await; // Acquire a permit + + if let Err(_) = permit { + tracing::error!( + workspace_id = %w_id, + "Cannot acquire permit on semaphore, that can only mean that semaphore has been closed." + ); + break; + } + + let permit = permit.unwrap(); + + let action = match strategy { + InstallStrategy::Single(ref callback) => Action::Install( + start_child_process(callback(dep.clone())?, &installer_executable_name, false) + .await?, + ), + InstallStrategy::AllAtOnce(ref rw_lock) => Action::AddToBulk(Arc::clone(rw_lock)), + }; + let task_fut = try_install_one_detached( + dep, + installer_executable_name.to_owned(), + job_id.clone(), + w_id.to_owned(), + worker_name.to_owned(), + conn.to_owned(), + counter_arc.clone(), + total_to_install, + name_ml, + action, + _language_name.to_owned(), + _platform_agnostic, + permit, + TaskKiller(kill_tx), + ); + handles.push(tokio::spawn(async move { + tokio::select! { + reason = kill_rx.recv() => return Err(anyhow::anyhow!("Installation stopped. Reason: {:?}", reason)), + r = task_fut => return r + }; + })); + } + + Ok((handles, closer)) +} +/// Do not drop the returned sender! As soon as you drop it the polling thread will stop +pub async fn listen_to_cancel( + job_id: Uuid, + w_id: String, + conn: Connection, + killpill: tokio::sync::broadcast::Sender, +) -> tokio::sync::broadcast::Sender<()> { + let (close_tx, mut close_rx) = tokio::sync::broadcast::channel::<()>(1); + tokio::spawn(async move { + 'outer: loop { + tokio::select! { + _ = tokio::time::sleep(tokio::time::Duration::from_secs(1)) => { + // TODO: Add memory usage tracking + if crate::worker_utils::ping_job_status(&conn, &job_id, None, None).await.map(|r| r.canceled_by.is_some()).unwrap_or(false){ + tracing::info!( + // If there is listener on other side, + workspace_id = %w_id, + "cancelling installations", + ); + if let Err(ref e) = killpill.send("Job was canceled".to_owned()) { + tracing::error!( + // If there is listener on other side, + workspace_id = %w_id, + "failed to send done: Probably receiving end closed too early or have not opened yet\n{}", + // If there is no listener, it will be dropped safely + e + ); + } + // Stop + break 'outer; + } + + }, + _ = close_rx.recv() => break 'outer, + } + } + }); + + // We will return the sender that can stop created thread. + // It is relatively safe, since if this handle is being dropped from the gecko the thread will stop immediately + // but it will also chainreaction all installation threads, so be carefull + close_tx +} + +async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + 'static>( + dep: RequiredDependency, + installer_executable_name: String, + job_id: Uuid, + w_id: String, + worker_name: String, + conn: Connection, + counter_arc: Arc>, + total_to_install: usize, + name_ml: usize, + action: Action, + _language_name: String, + _platform_agnostic: bool, + _permit: OwnedSemaphorePermit, + // If dropped the entire installation fails and all installation threads are being stopped + // That's why we just pass it to return so it is not being dropped + kill_all_tasks: TaskKiller, +) -> anyhow::Result { + let start = std::time::Instant::now(); + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let is_not_pro = !matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ); + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let s3_pull_future = if is_not_pro { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + Some(crate::global_cache::pull_from_tar( + os, + dep.path.clone(), + _language_name.to_owned(), + Some(dep._s3_handle.clone()), + _platform_agnostic, + )) + } else { + None + } + } else { + None + }; + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(s3_pull_future) = s3_pull_future { + if let Err(e) = s3_pull_future.await { + tracing::info!( + workspace_id = %w_id, + "No tarball was found for {:?} on S3 or different problem occured {job_id}:\n{e}", + &dep._s3_handle.clone() + ); + } else { + mark_success(dep.path, &job_id, &w_id).await; + print_success( + true, + false, + &job_id, + &w_id, + &dep.display_name, + name_ml, + counter_arc, + total_to_install, + start, + &conn, + ) + .await; + return Ok(kill_all_tasks); + } + } + let child = match action { + Action::Install(child) => child, + Action::AddToBulk(rw_lock) => { + rw_lock.write().await.push(dep.clone()); + return Ok(kill_all_tasks); + } + }; + if let Err(e) = crate::handle_child::handle_child( + &job_id, + &conn, + // TODO: Return mem_peak + &mut 0, + // TODO: Return canceld_by_ref + &mut None, + child, + !*DISABLE_NSJAIL, + &worker_name, + &w_id, + &installer_executable_name, + None, + false, + &mut None, + None, + ) + .await + { + windmill_queue::append_logs( + &job_id, + &w_id, + format!( + "\n[x] - error while installing {}: {e:?}", + &dep.display_name + ), + &conn, + ) + .await; + bail!(format!( + "\n error while installing dependencies: {e:?}\n{}", + &dep.display_name + )); + } else { + mark_success(dep.path.clone(), &job_id, &w_id).await; + print_success( + false, + true, + &job_id, + &w_id, + &dep.display_name, + name_ml, + counter_arc, + total_to_install, + start, + &conn, + ) + .await; + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { + let language_name = _language_name.to_string(); + let platform_agnostic = _platform_agnostic; + let path = dep.path.clone(); + let handle = dep._s3_handle.clone(); + tokio::spawn(async move { + if let Err(e) = crate::global_cache::build_tar_and_push( + os, + path, + language_name, + Some(handle), + platform_agnostic, + ) + .await + { + tracing::warn!("failed to build tar and push: {e:?}"); + } + }); + } + } + } + + Ok(kill_all_tasks) +} + +// Create a file to indicate that installation was successfull +async fn mark_success(path: String, job_id: &Uuid, w_id: &str) { + let valid_path = path + ".valid.windmill"; + // This is atomic operation, meaning, that it either completes and dependency is valid, + // or it does not and dependency is invalid and will be reinstalled next run + if let Err(e) = File::create(&valid_path).await { + tracing::error!( + workspace_id = %w_id, + job_id = %job_id, + "Failed to create {}!\n{e}\n + This file needed for jobs to function", + valid_path + ); + }; +} +// Append logs with line like this: +// [9/21] + requests==2.32.3 << (S3) | in 57ms +#[allow(unused_assignments)] +async fn print_success( + mut s3_pull: bool, + mut s3_push: bool, + job_id: &Uuid, + w_id: &str, + req: &str, + req_tl: usize, + counter_arc: Arc>, + total_to_install: usize, + instant: std::time::Instant, + conn: &Connection, +) { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + (s3_pull, s3_push) = (false, false); + } + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if windmill_common::s3_helpers::OBJECT_STORE_SETTINGS + .read() + .await + .is_none() + { + (s3_pull, s3_push) = (false, false); + } + + let mut counter = counter_arc.lock().await; + *counter += 1; + + windmill_queue::append_logs( + job_id, + w_id, + format!( + "\n{}+ {}{}{}| in {}ms", + windmill_common::worker::pad_string(&format!("[{}/{total_to_install}]", counter), 9), + // Because we want to align to max len [999/999] we take 9 + // 123456789 + windmill_common::worker::pad_string(&req, req_tl + 1), + // Margin to the right ^ + if s3_pull { "<< (S3) " } else { "" }, + if s3_push { " > (S3) " } else { "" }, + instant.elapsed().as_millis(), + ), + conn, + ) + .await; + // Drop lock, so next print success can fire +} + +async fn process_handles( + (handles, closer): (Vec>>, Sender<()>), + w_id: &str, +) -> anyhow::Result<()> { + let mut killers = vec![]; + for handle in handles { + match handle + .await + .unwrap_or(Err(anyhow::anyhow!("Problem by joining handle"))) + { + // __________< Cannot drop just yet. + Ok(kill_tasks) => killers.push(kill_tasks), + Err(e) => { + tracing::warn!( + workspace_id = %w_id, + "Env installation failed: {:?}", + e + ); + // We can safely return, + // it will drop `closer` + // which will drop first kill_tx for all installation threads + // which will trigger chain reaction and kill every installation + return Err(e); + } + } + } + + // Only now we can be sure there is no installation thread running + // we can safely stop poller + drop(closer); + // Drop ONLY after we processed all handles + drop(killers); + Ok(()) +} +async fn finish_installation( + total_time: std::time::Instant, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) { + let total_time = total_time.elapsed().as_millis(); + windmill_queue::append_logs( + &job_id, + w_id, + format!( + "\nDone. Time spent on installation phase: {}ms\n", + total_time + ), + conn, + ) + .await; +} diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-worker/src/windmill-client.js index 1ecf28e664..33e51577e6 100644 --- a/backend/windmill-worker/src/windmill-client.js +++ b/backend/windmill-worker/src/windmill-client.js @@ -2999,6 +2999,7 @@ var $RawScript = { "graphql", "nativets", "duckdb", + "ruby", // for related places search: ADD_NEW_LANG ], }, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 4ff474cbd2..2ed78f0066 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -134,6 +134,9 @@ use crate::nu_executor::{handle_nu_job, JobHandlerInput as JobHandlerInputNu}; #[cfg(feature = "java")] use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJava}; +#[cfg(feature = "ruby")] +use crate::ruby_executor::{handle_ruby_job, JobHandlerInput as JobHandlerInputRuby}; + #[cfg(feature = "php")] use crate::php_executor::handle_php_job; @@ -191,10 +194,14 @@ pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu"); pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); -// JAVA +// Java pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java"); pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache"); pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository"); + +// Ruby +pub const RUBY_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "ruby"); + // for related places search: ADD_NEW_LANG pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); @@ -338,6 +345,7 @@ lazy_static::lazy_static! { .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(false)); + pub static ref RUBY_REPOS: Arc>>> = Arc::new(RwLock::new(None)); pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); @@ -3439,6 +3447,33 @@ mount {{ }) .await } + Some(ScriptLang::Ruby) => { + #[cfg(not(feature = "ruby"))] + return Err(anyhow::anyhow!( + "Ruby is not available because the feature is not enabled" + ) + .into()); + + #[cfg(feature = "ruby")] + handle_ruby_job(JobHandlerInputRuby { + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + inner_content: &code, + job_dir, + requirements_o: lock.as_ref(), + shared_mount: &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + }) + .await + } + // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; tracing::info!( @@ -3511,6 +3546,10 @@ fn parse_sig_of_lang( ScriptLang::Java => Some(windmill_parser_java::parse_java_signature(code)?), #[cfg(not(feature = "java"))] ScriptLang::Java => None, + #[cfg(feature = "ruby")] + ScriptLang::Ruby => Some(windmill_parser_ruby::parse_ruby_signature(code)?), + #[cfg(not(feature = "ruby"))] + ScriptLang::Ruby => None, // for related places search: ADD_NEW_LANG } } else { diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 3d24e424d5..575d94c28c 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -42,7 +42,10 @@ use crate::common::OccupancyMetrics; use crate::csharp_executor::generate_nuget_lockfile; #[cfg(feature = "java")] -use crate::java_executor::resolve; +use crate::java_executor; + +#[cfg(feature = "ruby")] +use crate::ruby_executor; #[cfg(feature = "php")] use crate::php_executor::{composer_install, parse_php_imports}; @@ -2415,7 +2418,7 @@ async fn capture_dependency_job( )); } - resolve( + java_executor::resolve( job_id, job_raw_code, job_dir, @@ -2424,6 +2427,26 @@ async fn capture_dependency_job( ) .await } + #[cfg(feature = "ruby")] + ScriptLang::Ruby => { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for Ruby".to_string(), + )); + } + + ruby_executor::resolve( + job_id, + job_raw_code, + mem_peak, + canceled_by, + job_dir, + &Connection::Sql(db.clone()), + worker_name, + w_id, + ) + .await + } // for related places search: ADD_NEW_LANG _ => Ok("".to_owned()), } diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index e9e8a12db5..5335e1741c 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -126,6 +126,11 @@ public class Main { System.out.println("Hello World"); } } +`, + ruby: ` +def main a, b, c + puts a, b, c +end `, // for related places search: ADD_NEW_LANG }; diff --git a/cli/dnt.ts b/cli/dnt.ts index ac99fc1c4a..9f576a3a2c 100644 --- a/cli/dnt.ts +++ b/cli/dnt.ts @@ -71,6 +71,8 @@ await build({ "yaml", "csharp", "java", + "ruby", + // for related places search: ADD_NEW_LANG ]; for (const l of dirs) { diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 2becb662e4..a2240498ab 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -578,7 +578,9 @@ export function filePathExtensionFromContentType( return ".nu"; } else if (language === "java") { return ".java"; - // for related places search: ADD_NEW_LANG + } else if (language === "ruby") { + return ".rb"; + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -608,7 +610,8 @@ export const exts = [ ".nu", ".playbook.yml", ".java", - // for related places search: ADD_NEW_LANG + ".rb" + // for related places search: ADD_NEW_LANG ]; export function removeExtensionToPath(path: string): string { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index ee5c6a965c..c17861a497 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -694,7 +694,8 @@ export async function elementsToMap( "yml", "nu", "java", - // for related places search: ADD_NEW_LANG + "rb", + // for related places search: ADD_NEW_LANG ].includes(path.split(".").pop() ?? "") && !isFileResource(path) ) diff --git a/cli/src/types.ts b/cli/src/types.ts index e74d302a42..57c1c9b8e7 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -232,6 +232,7 @@ export function getTypeStrFromPath( parsed.ext == ".cs" || parsed.ext == ".nu" || parsed.ext == ".java" || + parsed.ext == ".rb" || // for related places search: ADD_NEW_LANG (parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook") ) { diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index cb5de7391a..d153fcbd8f 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -643,7 +643,10 @@ export async function inferSchema( } else if (language === "java") { const { parse_java } = await import("../../wasm/java/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_java(content)); - // for related places search: ADD_NEW_LANG + } else if (language === "ruby") { + const { parse_ruby } = await import("./wasm/ruby/windmill_parser_wasm.js"); + inferedSchema = JSON.parse(parse_ruby(content)); + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index 85efea9786..df661d25fe 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -19,6 +19,7 @@ export type ScriptLanguage = | "csharp" | "nu" | "ansible" + | "ruby" | "java"; // for related places search: ADD_NEW_LANG @@ -87,7 +88,9 @@ export function inferContentTypeFromFilePath( return "nu"; } else if (contentPath.endsWith(".java")) { return "java"; - // for related places search: ADD_NEW_LANG + } else if (contentPath.endsWith(".rb")) { + return "ruby"; + // for related places search: ADD_NEW_LANG } else { throw new Error( "Invalid language: " + contentPath.substring(contentPath.lastIndexOf(".")) diff --git a/cli/wasm/ruby/windmill_parser_wasm.d.ts b/cli/wasm/ruby/windmill_parser_wasm.d.ts new file mode 100644 index 0000000000..1d3229c2ec --- /dev/null +++ b/cli/wasm/ruby/windmill_parser_wasm.d.ts @@ -0,0 +1,3 @@ +/* tslint:disable */ +/* eslint-disable */ +export function parse_ruby(code: string): string; diff --git a/cli/wasm/ruby/windmill_parser_wasm.js b/cli/wasm/ruby/windmill_parser_wasm.js new file mode 100644 index 0000000000..2c44b756b6 --- /dev/null +++ b/cli/wasm/ruby/windmill_parser_wasm.js @@ -0,0 +1,121 @@ + + +let WASM_VECTOR_LEN = 0; + +let cachedUint8ArrayMemory0 = null; + +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } ); + +const encodeString = function (arg, view) { + return cachedTextEncoder.encodeInto(arg, view); +}; + +function passStringToWasm0(arg, malloc, realloc) { + + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = encodeString(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); + +if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; + +function getStringFromWasm0(ptr, len) { + ptr = ptr >>> 0; + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} +/** + * @param {string} code + * @returns {string} + */ +export function parse_ruby(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_ruby(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +const imports = { + __wbindgen_placeholder__: { + __wbindgen_init_externref_table: function() { + const table = wasm.__wbindgen_export_0; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); + ; + }, + }, + +}; + +const wasm_url = new URL('windmill_parser_wasm_bg.wasm', import.meta.url); +let wasmCode = ''; +switch (wasm_url.protocol) { + case 'file:': + wasmCode = await Deno.readFile(wasm_url); + break + case 'https:': + case 'http:': + wasmCode = await (await fetch(wasm_url)).arrayBuffer(); + break + default: + throw new Error(`Unsupported protocol: ${wasm_url.protocol}`); +} + +const wasmInstance = (await WebAssembly.instantiate(wasmCode, imports)).instance; +const wasm = wasmInstance.exports; +export const __wasm = wasm; + +wasm.__wbindgen_start(); + diff --git a/cli/wasm/ruby/windmill_parser_wasm_bg.wasm b/cli/wasm/ruby/windmill_parser_wasm_bg.wasm new file mode 100644 index 0000000000..e71dce2e5c Binary files /dev/null and b/cli/wasm/ruby/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/ruby/windmill_parser_wasm_bg.wasm.d.ts b/cli/wasm/ruby/windmill_parser_wasm_bg.wasm.d.ts new file mode 100644 index 0000000000..ec299d2fdc --- /dev/null +++ b/cli/wasm/ruby/windmill_parser_wasm_bg.wasm.d.ts @@ -0,0 +1,35 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory; +export const parse_ruby: (a: number, b: number) => [number, number]; +export const free: (a: number) => void; +export const abort: () => void; +export const malloc: (a: number) => number; +export const calloc: (a: number, b: number) => number; +export const realloc: (a: number, b: number) => number; +export const strncmp: (a: number, b: number, c: number) => number; +export const memchr: (a: number, b: number, c: number) => number; +export const strchr: (a: number, b: number) => number; +export const iswspace: (a: number) => number; +export const iswalnum: (a: number) => number; +export const iswdigit: (a: number) => number; +export const iswupper: (a: number) => number; +export const iswalpha: (a: number) => number; +export const iswlower: (a: number) => number; +export const clock: () => bigint; +export const isprint: (a: number) => number; +export const fprintf: (a: number, b: number, c: number) => number; +export const fputs: (a: number, b: number) => number; +export const fputc: (a: number, b: number) => number; +export const fdopen: (a: number, b: number) => number; +export const fclose: (a: number) => number; +export const fwrite: (a: number, b: number, c: number, d: number) => number; +export const vsnprintf: (a: number, b: number, c: number, d: number) => number; +export const clock_gettime: (a: number, b: number) => void; +export const snprintf: () => void; +export const __assert_fail: (a: number, b: number, c: number, d: number) => void; +export const __wbindgen_export_0: WebAssembly.Table; +export const __wbindgen_malloc: (a: number, b: number) => number; +export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_free: (a: number, b: number, c: number) => void; +export const __wbindgen_start: () => void; diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 6efb16c7bd..1d5f639113 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -22,3 +22,6 @@ RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2 && mv ./coursier /usr/bin/coursier \ && chmod +x /usr/bin/coursier RUN /usr/bin/java -jar /usr/bin/coursier about + +# Ruby +RUN apt-get install -y ruby ruby-bundler diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index e93aaaf9a0..376f994f89 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -45,3 +45,6 @@ RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2 && mv ./coursier /usr/bin/coursier \ && chmod +x /usr/bin/coursier RUN /usr/bin/java -jar /usr/bin/coursier about + +# Ruby +RUN apt-get install -y ruby ruby-bundler diff --git a/docker/DockerfileNsjail b/docker/DockerfileNsjail index 2e9b0b0580..a995c76fac 100644 --- a/docker/DockerfileNsjail +++ b/docker/DockerfileNsjail @@ -70,6 +70,9 @@ RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2 && chmod +x /usr/bin/coursier RUN /usr/bin/java -jar /usr/bin/coursier about +# Ruby +RUN apt-get install -y ruby ruby-bundler + # iptables RUN apt-get install -y iptables diff --git a/docker/dev.nu b/docker/dev.nu index a1fd303b0b..77d2641204 100755 --- a/docker/dev.nu +++ b/docker/dev.nu @@ -4,6 +4,8 @@ # run: # # docker/dev.nu --help # # ------------------------------------ # +# NOTE: +# Does not work if built from git worktree branch let branch = git branch --show-current; # cp -f Caddyfile .dev-docker/Caddyfile @@ -57,6 +59,7 @@ def patch_custom_dockerfile [ ] { open $custom_dockerfile | str replace 'ghcr.io/windmill-labs/windmill-ee:dev' (get_ident -f $features) + | str replace 'ghcr.io/windmill-labs/windmill:dev' (get_ident -f $features) | return $in } diff --git a/flake.nix b/flake.nix index 9aac3a3664..be958fd547 100644 --- a/flake.nix +++ b/flake.nix @@ -143,6 +143,7 @@ dotnet-sdk_9 oracle-instantclient ansible + ruby_3_4 # LSP/Local dev svelte-language-server @@ -214,7 +215,10 @@ GIT_PATH = "${pkgs.git}/bin/git"; NODE_ENV = "development"; NODE_OPTIONS = "--max-old-space-size=16384"; - DATABASE_URL = "postgres://postgres:changeme@127.0.0.1:5432/"; + # DATABASE_URL = "postgres://postgres:changeme@127.0.0.1:5432/"; + DATABASE_URL = + "postgres://postgres:changeme@127.0.0.1:5432/windmill?sslmode=disable"; + REMOTE = "http://127.0.0.1:8000"; REMOTE_LSP = "http://127.0.0.1:3001"; RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache"; @@ -228,7 +232,10 @@ JAVA_PATH = "${pkgs.jdk21}/bin/java"; JAVAC_PATH = "${pkgs.jdk21}/bin/javac"; COURSIER_PATH = "${coursier}/coursier"; - # for related places search: ADD_NEW_LANG + RUBY_PATH = "${pkgs.ruby}/bin/ruby"; + RUBY_BUNDLE_PATH = "${pkgs.ruby}/bin/bundle"; + RUBY_GEM_PATH = "${pkgs.ruby}/bin/gem"; + # for related places search: ADD_NEW_LANG FLOCK_PATH = "${pkgs.flock}/bin/flock"; CARGO_PATH = "${rust}/bin/cargo"; CARGO_SWEEP_PATH = "${pkgs.cargo-sweep}/bin/cargo-sweep"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index eb37230be8..324bc799da 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -81,6 +81,7 @@ "windmill-parser-wasm-php": "1.510.1", "windmill-parser-wasm-py": "1.510.1", "windmill-parser-wasm-regex": "1.512.0", + "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.510.1", "windmill-parser-wasm-ts": "1.514.1", "windmill-parser-wasm-yaml": "1.510.1", @@ -156,6 +157,11 @@ "svelte": "^5.0.0" } }, + "../backend/parsers/windmill-parser-wasm/pkg-ruby": { + "name": "windmill-parser-wasm-ruby", + "version": "1.510.0", + "extraneous": true + }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -13144,6 +13150,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.512.0.tgz", "integrity": "sha512-mOkuspfjPPhGZwmerBlFOjRKHjycrlzZUpxO7gHy5D6kwKI2bSz+VI3TsJqmK1y9PwAGgO8JnwRmNAR/YLUkzA==" }, + "node_modules/windmill-parser-wasm-ruby": { + "version": "1.526.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ruby/-/windmill-parser-wasm-ruby-1.526.1.tgz", + "integrity": "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g==" + }, "node_modules/windmill-parser-wasm-rust": { "version": "1.510.1", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.510.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index fdbfaab967..24e60adeb5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -148,6 +148,7 @@ "windmill-parser-wasm-php": "1.510.1", "windmill-parser-wasm-py": "1.510.1", "windmill-parser-wasm-regex": "1.512.0", + "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.510.1", "windmill-parser-wasm-ts": "1.514.1", "windmill-parser-wasm-yaml": "1.510.1", @@ -543,4 +544,4 @@ "@rollup/rollup-linux-x64-gnu": "^4.35.0", "fsevents": "^2.3.3" } -} +} \ No newline at end of file diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index f87322d76f..825b6f7f58 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -160,7 +160,6 @@ let editor: meditor.IStandaloneCodeEditor | null = $state(null) interface Props { - // for related places search: ADD_NEW_LANG code?: string cmdEnterAction?: (() => void) | undefined formatAction?: (() => void) | undefined diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 00b92bff41..40665e3f36 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -132,7 +132,8 @@ 'rust', 'csharp', 'nu', - 'java' + 'java', + 'ruby' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') ) @@ -151,7 +152,8 @@ 'rust', 'csharp', 'nu', - 'java' + 'java', + 'ruby' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') ) @@ -170,7 +172,8 @@ 'rust', 'csharp', 'nu', - 'java' + 'java', + 'ruby', // for related places search: ADD_NEW_LANG ].includes(lang ?? '') ) @@ -436,6 +439,8 @@ } else if (lang == 'java') { editor.insertAtCursor(`System.getenv("${name}");`) // for related places search: ADD_NEW_LANG + } else if (lang == 'ruby') { + editor.insertAtCursor(`ENV['${name}']`) } sendUserToast(`${name} inserted at cursor`) }} @@ -506,6 +511,11 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri); } else if (lang == 'java') { editor.insertAtCursor(`(Wmill.getVariable("${path}"))`) // for related places search: ADD_NEW_LANG + } else if (lang == 'ruby') { + if (!editor.getCode().includes("require 'windmill/mini'")) { + editor.insertAtBeginning("require 'windmill/mini'\n") + } + editor.insertAtCursor(`get_variable("${path}")`) } sendUserToast(`${name} inserted at cursor`) }} @@ -595,6 +605,11 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS } else if (lang == 'java') { editor.insertAtCursor(`(Wmill.getResource("${path}"))`) // for related places search: ADD_NEW_LANG + } else if (lang == 'ruby') { + if (!editor.getCode().includes("require 'windmill/mini'")) { + editor.insertAtBeginning("require 'windmill/mini'\n") + } + editor.insertAtCursor(`get_resource("${path}")`) } else if (lang == 'duckdb') { let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType] if (!t) { diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index bf196c7868..aebd2bf0f1 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -13,6 +13,7 @@ import csharp from 'svelte-highlight/languages/csharp' import yaml from 'svelte-highlight/languages/yaml' import java from 'svelte-highlight/languages/java' + import ruby from 'svelte-highlight/languages/ruby' import type { Script } from '$lib/gen' import { Button } from './common' import { copyToClipboard } from '$lib/utils' @@ -70,8 +71,10 @@ case 'ansible': return yaml case 'java': - return java - // for related places search: ADD_NEW_LANG + return java; + case 'ruby': + return ruby; + // for related places search: ADD_NEW_LANG default: return typescript } diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 28d54fd01d..7d1721a988 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -1128,7 +1128,7 @@ > {label} - {#if lang === 'nu'} + {#if lang === 'nu' || lang === 'ruby'} BETA {/if} diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 3cfb57fee5..19c8a5e991 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -23,6 +23,7 @@ import NuIcon from '$lib/components/icons/NuIcon.svelte' import JavaIcon from '$lib/components/icons/JavaIcon.svelte' import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte' + import RubyIcon from '$lib/components/icons/RubyIcon.svelte' interface Props { lang: @@ -71,8 +72,9 @@ ansible: 'Ansible', csharp: 'C#', nu: 'Nu', - java: 'Java' - // for related places search: ADD_NEW_LANG + java: 'Java', + ruby: 'Ruby' + // for related places search: ADD_NEW_LANG } const langToComponent: Record< @@ -105,6 +107,7 @@ csharp: CSharpIcon, nu: NuIcon, java: JavaIcon, + ruby: RubyIcon, duckdb: DuckDbIcon // for related places search: ADD_NEW_LANG } diff --git a/frontend/src/lib/components/icons/RubyIcon.svelte b/frontend/src/lib/components/icons/RubyIcon.svelte new file mode 100644 index 0000000000..307fde1a3d --- /dev/null +++ b/frontend/src/lib/components/icons/RubyIcon.svelte @@ -0,0 +1,661 @@ + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 1a873d532f..9bf4b6bad6 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -331,6 +331,15 @@ export const settings: Record = { fieldType: 'boolean', storage: 'setting', ee_only: '' + }, + { + label: 'Ruby Gems repositories', + description: 'Add private Ruby repositories with credentials. Should end with /', + key: 'ruby_repos', + fieldType: 'text', + placeholder: 'https://user:password@gems.foo.com/', + storage: 'setting', + ee_only: '' } ], Alerts: [ diff --git a/frontend/src/lib/components/worker_group.ts b/frontend/src/lib/components/worker_group.ts index 70277da84e..a396568836 100644 --- a/frontend/src/lib/components/worker_group.ts +++ b/frontend/src/lib/components/worker_group.ts @@ -53,8 +53,10 @@ export const defaultTags = [ 'ansible', 'csharp', 'nu', - 'java' - // KJQXZ + 'java', + 'ruby' + // for related places search: ADD_NEW_LANG + ] export const nativeTags = [ 'nativets', @@ -65,4 +67,4 @@ export const nativeTags = [ 'mssql', 'bigquery', 'oracledb' -] \ No newline at end of file +] diff --git a/frontend/src/lib/editorLangUtils.ts b/frontend/src/lib/editorLangUtils.ts index 17d75afe16..c856d1bd8b 100644 --- a/frontend/src/lib/editorLangUtils.ts +++ b/frontend/src/lib/editorLangUtils.ts @@ -97,6 +97,8 @@ export function extToLang(ext: string) { return 'nu' case 'java': return 'java' + case 'rb': + return 'ruby' // for related places search: ADD_NEW_LANG default: return 'unknown' diff --git a/frontend/src/lib/editorUtils.ts b/frontend/src/lib/editorUtils.ts index 5c86f53a60..4cb3b586ed 100644 --- a/frontend/src/lib/editorUtils.ts +++ b/frontend/src/lib/editorUtils.ts @@ -41,8 +41,6 @@ export function editorConfig( } } - - export const updateOptions = { tabSize: 2, insertSpaces: true } export function convertKind(kind: string): any { diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 702d8c7265..dbe75e6beb 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -27,6 +27,7 @@ import initYamlParser, { parse_ansible } from 'windmill-parser-wasm-yaml' import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp' import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu' import initJavaParser, { parse_java } from 'windmill-parser-wasm-java' +import initRubyParser, { parse_ruby } from 'windmill-parser-wasm-ruby' import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url' import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url' @@ -38,6 +39,7 @@ import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm? import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.wasm?url' import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url' import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url' +import wasmUrlRuby from 'windmill-parser-wasm-ruby/windmill_parser_wasm_bg.wasm?url' import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from 'windmill-utils-internal' import { type AssetWithAccessType } from './components/assets/lib.js' @@ -79,6 +81,9 @@ async function initWasmNu() { async function initWasmJava() { await initJavaParser(wasmUrlJava) } +async function initWasmRuby() { + await initRubyParser(wasmUrlRuby) +} export async function inferAssets( language: SupportedLanguage | undefined, @@ -241,7 +246,10 @@ export async function inferArgs( } else if (language == 'java') { await initWasmJava() inferedSchema = JSON.parse(parse_java(code)) - // for related places search: ADD_NEW_LANG + } else if (language == 'ruby') { + await initWasmRuby() + inferedSchema = JSON.parse(parse_ruby(code)) + // for related places search: ADD_NEW_LANG } else { return null } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 7b0797d595..7c02bd6aea 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1107,6 +1107,55 @@ public class Main { } } ` +const RUBY_INIT_CODE = `require 'windmill/inline' +require 'windmill/mini' + +# Dependency management: declare gems in gemfile block for automatic installation +# Windmill uses bundler/inline compatible syntax with automatic requiring +gemfile do + source 'https://rubygems.org' + gem 'amazing_print', '~> 1.6' +end + +def main( + no_default, + name = "Nicolas Bourbaki", + age = 42, + obj = { "even": "hashes" }, + list = ["or", "arrays!"] +) + puts "Hello World and a warm welcome especially to #{name}" + puts "and its acolytes.. #{age} #{obj} #{list}" + + # Retrieve variables using the Windmill mini client + begin + secret = get_variable("f/examples/secret") + rescue => e + secret = "No secret yet at f/examples/secret!" + end + puts "The variable at 'f/examples/secret': #{secret}" + + # Get typed resources using the mini client + # database = get_resource("u/user/my_postgresql") + + # Access environment variables provided by Windmill + user = ENV['WM_USERNAME'] + + # Pretty print results using amazing_print (automatically required from gemfile) + result = { + "splitted" => name.split, + "user" => user, + "age" => age, + "obj" => obj, + "list" => list + } + + ap result + + # Return value is automatically converted to JSON + return result +end +` // for related places search: ADD_NEW_LANG export const INITIAL_CODE = { bun: { @@ -1197,8 +1246,11 @@ export const INITIAL_CODE = { }, java: { script: JAVA_INIT_CODE - } - // for related places search: ADD_NEW_LANG + }, + ruby: { + script: RUBY_INIT_CODE + }, + // for related places search: ADD_NEW_LANG } export function isInitialCode(content: string): boolean { @@ -1306,7 +1358,9 @@ export function initialCode( return INITIAL_CODE.nu.script } else if (language == 'java') { return INITIAL_CODE.java.script - // for related places search: ADD_NEW_LANG + } else if (language == 'ruby') { + return INITIAL_CODE.ruby.script + // for related places search: ADD_NEW_LANG } else if (language == 'bun' || language == 'bunnative') { if (kind == 'trigger') { return INITIAL_CODE.bun.trigger diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 961ae72488..50884f3c26 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -143,7 +143,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['docker', 'Docker'], ['nu', 'Nu'], ['java', 'Java'], - ['duckdb', 'DuckDB'] + ['duckdb', 'DuckDB'], + ['ruby', 'Ruby'] // for related places search: ADD_NEW_LANG ] export function processLangs(selected: string | undefined, langs: string[]): string[] { @@ -153,7 +154,7 @@ export function processLangs(selected: string | undefined, langs: string[]): str let ls = langs.filter((lang) => lang !== 'nativets') //those languages are newer and may not be in the saved list - let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb'] + let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'ruby'] // for related places search: ADD_NEW_LANG nl.forEach((lang) => { if (!ls.includes(lang)) { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 26ccaea24e..8daa4d4f2a 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -791,7 +791,7 @@ priority: {job.priority} {/if} - {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'dependency'].includes(job.tag)} + {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'dependency', 'ruby'].includes(job.tag)}
Tag: {job.tag}