diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index cd1bb62cf7..dc2feb7d10 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,7 +1,11 @@ env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository || 'windmill-labs/windmill-test' }} - DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', github.event.number) }} + IMAGE_NAME: + ${{ github.event_name != 'pull_request' && github.repository || + 'windmill-labs/windmill-test' }} + DEV_SHA: + ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', + github.event.number) }} name: Build windmill:main on: @@ -495,7 +499,7 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Docker meta id: meta-ee-public uses: docker/metadata-action@v4 @@ -541,7 +545,7 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: get git hash if: github.event_name != 'pull_request' id: git_hash @@ -610,6 +614,51 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + build_slim: + if: ${{ startsWith(github.ref, 'refs/tags/') }} + needs: [build] + runs-on: ubicloud + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + - uses: depot/setup-action@v1 + + - name: Docker meta + id: meta-ee-public + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim + flavor: | + latest=false + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Login to registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push publicly ee + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/amd64 + push: true + file: "./docker/DockerfileSlim" + tags: | + ${{ steps.meta-ee-public.outputs.tags }} + labels: | + ${{ steps.meta-ee-public.outputs.labels }} + build_ee_slim: if: ${{ startsWith(github.ref, 'refs/tags/') }} needs: [build_ee] @@ -649,7 +698,98 @@ jobs: context: . platforms: linux/amd64 push: true - file: "./docker/DockerfileSlim" + file: "./docker/DockerfileSlimEe" + tags: | + ${{ steps.meta-ee-public.outputs.tags }} + labels: | + ${{ steps.meta-ee-public.outputs.labels }} + org.opencontainers.image.licenses=Windmill-Enterprise-License + + build_full: + if: ${{ startsWith(github.ref, 'refs/tags/') }} + needs: [build] + runs-on: ubicloud + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + - uses: depot/setup-action@v1 + + - name: Docker meta + id: meta-ee-public + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full + flavor: | + latest=false + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Login to registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push publicly ee + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/amd64 + push: true + file: "./docker/DockerfileRust" + tags: | + ${{ steps.meta-ee-public.outputs.tags }} + labels: | + ${{ steps.meta-ee-public.outputs.labels }} + + build_ee_full: + if: ${{ startsWith(github.ref, 'refs/tags/') }} + needs: [build_ee] + runs-on: ubicloud + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + - uses: depot/setup-action@v1 + + - name: Docker meta + id: meta-ee-public + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full + flavor: | + latest=false + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Login to registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push publicly ee + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/amd64 + push: true + file: "./docker/DockerfileRustEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/Dockerfile b/Dockerfile index 40c1789081..a6f36bd3f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -171,6 +171,9 @@ COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ +ENV RUSTUP_HOME="/usr/local/rustup" +ENV CARGO_HOME="/usr/local/cargo" + WORKDIR ${APP} RUN ln -s ${APP}/windmill /usr/local/bin/windmill diff --git a/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json b/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json index 580e8bfd01..756c432091 100644 --- a/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json +++ b/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json @@ -56,7 +56,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json b/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json index 9463b1e558..f527b2dc98 100644 --- a/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json +++ b/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json @@ -50,7 +50,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json index 790543d970..9c685da643 100644 --- a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json +++ b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json @@ -70,7 +70,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json b/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json index 3eddd1237b..349b06b505 100644 --- a/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json +++ b/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json @@ -55,7 +55,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json b/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json index 251d21f167..5d97751e42 100644 --- a/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json +++ b/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json @@ -36,7 +36,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json b/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json index 2220f81e41..f500609a9e 100644 --- a/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json +++ b/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json @@ -55,7 +55,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json index b1a706e87a..3c428971f8 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json @@ -50,7 +50,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json b/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json index 6d48bd9f44..019e2a661c 100644 --- a/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json +++ b/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json @@ -76,7 +76,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json b/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json index 9d5cfdf5af..8ad3e21f85 100644 --- a/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json +++ b/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json @@ -55,7 +55,8 @@ "powershell", "mssql", "php", - "bunnative" + "bunnative", + "rust" ] } } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e033d74127..7edc24f13a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6379,6 +6379,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" +dependencies = [ + "bitflags 2.6.0", + "getopts", + "memchr", + "unicase", +] + [[package]] name = "pulp" version = "0.18.22" @@ -10471,6 +10483,7 @@ dependencies = [ "once_cell", "pg-embed", "prometheus", + "quote", "rand 0.8.5", "reqwest 0.12.7", "rsmq_async", @@ -10772,6 +10785,23 @@ dependencies = [ "windmill-parser", ] +[[package]] +name = "windmill-parser-rust" +version = "1.387.0" +dependencies = [ + "anyhow", + "convert_case 0.6.0", + "itertools 0.10.5", + "lazy_static", + "pulldown-cmark", + "quote", + "regex", + "serde_json", + "syn 2.0.76", + "toml 0.7.8", + "windmill-parser", +] + [[package]] name = "windmill-parser-sql" version = "1.387.1" @@ -10817,6 +10847,7 @@ dependencies = [ "windmill-parser-graphql", "windmill-parser-php", "windmill-parser-py", + "windmill-parser-rust", "windmill-parser-sql", "windmill-parser-ts", ] @@ -10931,6 +10962,7 @@ dependencies = [ "windmill-parser-php", "windmill-parser-py", "windmill-parser-py-imports", + "windmill-parser-rust", "windmill-parser-sql", "windmill-parser-ts", "windmill-queue", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c7b50097be..e4b1009022 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -18,6 +18,7 @@ members = [ "./parsers/windmill-parser-ts", "./parsers/windmill-parser-wasm", "./parsers/windmill-parser-go", + "./parsers/windmill-parser-rust", "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", @@ -85,6 +86,7 @@ serde_json.workspace = true serde.workspace = true deno_core.workspace = true pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} +quote.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] @@ -114,6 +116,7 @@ windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" } windmill-parser-go = { path = "./parsers/windmill-parser-go" } +windmill-parser-rust = { path = "./parsers/windmill-parser-rust" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } @@ -266,4 +269,9 @@ tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "<0.1.12" tantivy = "0.22.0" + +pulldown-cmark = "0.9" +toml = "0.7" +syn = { version = "2.0.74", features = ["full"] } +quote = "1.0.36" regex-lite = "0.1.6" diff --git a/backend/migrations/20240807114128_add_rust_lang.down.sql b/backend/migrations/20240807114128_add_rust_lang.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20240807114128_add_rust_lang.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20240807114128_add_rust_lang.up.sql b/backend/migrations/20240807114128_add_rust_lang.up.sql new file mode 100644 index 0000000000..2a3cc1bae1 --- /dev/null +++ b/backend/migrations/20240807114128_add_rust_lang.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'rust'; +UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["rust"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php"]}'::jsonb AND NOT config->'worker_tags' @> '"rust"'::jsonb; diff --git a/backend/parsers/windmill-parser-rust/Cargo.toml b/backend/parsers/windmill-parser-rust/Cargo.toml new file mode 100644 index 0000000000..e1dcf84449 --- /dev/null +++ b/backend/parsers/windmill-parser-rust/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "windmill-parser-rust" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_rust" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +itertools.workspace = true +serde_json.workspace = true +anyhow.workspace = true +convert_case.workspace = true +lazy_static.workspace = true +regex.workspace = true +pulldown-cmark.workspace = true +toml.workspace = true +syn.workspace = true +quote.workspace = true diff --git a/backend/parsers/windmill-parser-rust/manifest/Cargo.toml.default b/backend/parsers/windmill-parser-rust/manifest/Cargo.toml.default new file mode 100644 index 0000000000..6c9e93c8a1 --- /dev/null +++ b/backend/parsers/windmill-parser-rust/manifest/Cargo.toml.default @@ -0,0 +1,16 @@ +[[bin]] +name = "main" +path = "./main.rs" + +[package] +authors = ["Anonymous"] +edition = "2021" +name = "main" +version = "0.1.0" + +[profile.release] +strip = true + +[dependencies] +serde = { version = "1.0.207", features = ["derive"] } +serde_json = "1.0.124" diff --git a/backend/parsers/windmill-parser-rust/src/lib.rs b/backend/parsers/windmill-parser-rust/src/lib.rs new file mode 100644 index 0000000000..f9a2a5b677 --- /dev/null +++ b/backend/parsers/windmill-parser-rust/src/lib.rs @@ -0,0 +1,501 @@ +use anyhow::anyhow; +use itertools::Itertools; +use quote::ToTokens; +use regex::Regex; +use windmill_parser::{Arg, MainArgSignature, Typ}; + +pub fn otyp_to_string(otyp: Option) -> String { + otyp.unwrap() +} + +pub fn parse_rust_signature(code: &str) -> anyhow::Result { + let ast: syn::File = syn::parse_file(code)?; + + if let Some(main_fn) = ast.items.iter().find_map(|item| match item { + syn::Item::Fn(f) if f.sig.ident == "main" => Some(f), + _ => None, + }) { + let args = main_fn + .sig + .inputs + .iter() + .map(|param| { + let (otyp, typ, name) = parse_rust_typ(param); + Arg { name, otyp, typ, default: None, has_default: false, oidx: None } + }) + .collect_vec(); + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: Some(false), + }) + } else { + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(true), + }) + } +} + +pub fn parse_rust_deps_into_manifest(code: &str) -> anyhow::Result { + const MODIFIABLE_MANIFEST_TABLES: &[&str] = &["dependencies"]; + + let partial_manifest = find_embedded_manifest(code).unwrap_or(Manifest::Toml("".to_string())); + let partial_manifest = partial_manifest.into_toml()?; + let mut manif = default_manifest(); + for table_name in MODIFIABLE_MANIFEST_TABLES { + match partial_manifest.get(*table_name) { + Some(toml::Value::Table(tab)) => { + // Merge. + match manif.entry(*table_name) { + toml::map::Entry::Vacant(e) => { + e.insert(toml::Value::Table(tab.to_owned())); + } + toml::map::Entry::Occupied(e) => { + let into_t = match e.into_mut() { + toml::Value::Table(t) => Some(t), + _ => None, + }; + into_t.ok_or(anyhow!(""))?.extend(tab.to_owned()); + } + } + } + Some(v) => { + // Just replace. + manif.insert(table_name.to_string(), v.to_owned()); + } + None => (), + } + } + + Ok(manif.to_string()) +} + +fn parse_pat_type(p: Box) -> Typ { + match *p { + syn::Type::Array(a) => { + let inner_typ = parse_pat_type(a.elem); + Typ::List(Box::new(inner_typ)) + } + // syn::Type::BareFn(_) => todo!(), + // syn::Type::Group(_) => todo!(), + // syn::Type::ImplTrait(_) => todo!(), + // syn::Type::Infer(_) => todo!(), + // syn::Type::Macro(_) => todo!(), + // syn::Type::Never(_) => todo!(), + syn::Type::Paren(e) => parse_pat_type(e.elem), + syn::Type::Path(e) => { + if let Some(u) = e.path.segments.last() { + match u.ident.to_string().as_str() { + "usize" | "u8" | "u16" | "u32" | "u64" | "u128" | "isize" | "i8" | "i16" + | "i32" | "i64" | "i128" => Typ::Int, + "String" | "str" => Typ::Str(None), + "bool" => Typ::Bool, + "f32" | "f64" => Typ::Float, + "Vec" => { + if let syn::PathArguments::AngleBracketed(t) = &u.arguments { + if let Some(syn::GenericArgument::Type(a)) = t.args.last() { + Typ::List(Box::new(parse_pat_type(Box::new(a.clone())))) + } else { + Typ::Unknown + } + } else { + Typ::Unknown + } + } + _ => Typ::Unknown, + } + } else { + Typ::Unknown + } + } + // syn::Type::Ptr(_) => todo!(), + syn::Type::Reference(e) => parse_pat_type(e.elem), + syn::Type::Slice(e) => Typ::List(Box::new(parse_pat_type(e.elem))), + // syn::Type::TraitObject(_) => todo!(), + // syn::Type::Tuple(_) => todo!(), + // syn::Type::Verbatim(_) => todo!(), + _ => Typ::Unknown, + } +} + +fn parse_rust_typ(param_typ: &syn::FnArg) -> (Option, Typ, String) { + match param_typ { + syn::FnArg::Receiver(_) => (None, Typ::Unknown, "self".to_string()), + syn::FnArg::Typed(s) => { + let name = match *s.pat.clone() { + syn::Pat::Ident(p) => p.ident.to_string(), + _ => "undefined".to_string(), + }; + + let otyp = Some(s.ty.to_token_stream().to_string()); + let typ = parse_pat_type(s.ty.clone()); + (otyp, typ, name) + } + } +} + +#[derive(Debug, PartialEq)] +enum Manifest { + Toml(String), + DepList(String), +} + +impl Manifest { + pub fn into_toml(self) -> anyhow::Result { + match self { + Manifest::Toml(s) => toml::from_str(&s), + Manifest::DepList(s) => Manifest::dep_list_to_toml(&s), + } + .map_err(|e| anyhow!("Could not parse embedded manifest: {}", e)) + } + + fn dep_list_to_toml(s: &str) -> ::std::result::Result { + let mut r = String::new(); + r.push_str("[dependencies]\n"); + for dep in s.trim().split(',') { + // If there's no version specified, add one. + match dep.contains('=') { + true => { + r.push_str(dep); + r.push('\n'); + } + false => { + r.push_str(dep); + r.push_str("=\"*\"\n"); + } + } + } + + toml::from_str(&r) + } +} + +fn default_manifest() -> toml::value::Table { + toml::from_str(include_str!("../manifest/Cargo.toml.default")) + .expect("Failed to parse Cargo.toml.default") +} + +/** +* Get a `Manifest` that can be made into a TOML table. The format and logic are the same as in +* the rust-script or cargo-eval projects. +*/ +fn find_embedded_manifest(s: &str) -> Option { + find_short_hand_manifest(s).or_else(|| find_code_block_manifest(s)) +} + +fn find_short_hand_manifest(s: &str) -> Option { + let re: Regex = Regex::new(r"^(?i)\s*//\s*cargo-deps\s*:(.*?)(\r\n|\n)").unwrap(); + /* + This is pretty simple: the only valid syntax for this is for the first, non-blank line to contain a single-line comment whose first token is `cargo-deps:`. That's it. + */ + if let Some(cap) = re.captures(s) { + if let Some(m) = cap.get(1) { + return Some(Manifest::DepList(m.as_str().to_string())); + } + } + None +} + +/** +Locates a "code block manifest" in Rust source. +*/ +fn find_code_block_manifest(s: &str) -> Option { + let re_crate_comment: Regex = { + Regex::new( + r"(?x) + ^\s* + (/\*!|//(!|/)) + ", + ) + .unwrap() + }; + + let start = match re_crate_comment.captures(s) { + Some(cap) => match cap.get(1) { + Some(m) => m.start(), + None => return None, + }, + None => return None, + }; + + let comment = match extract_comment(&s[start..]) { + Some(s) => s, + None => { + return None; + } + }; + + scrape_markdown_manifest(&comment).map(|s| Manifest::Toml(s)) +} + +/** +Extracts the first `Cargo` fenced code block from a chunk of Markdown. +*/ +fn scrape_markdown_manifest(content: &str) -> Option { + use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag}; + + // To match librustdoc/html/markdown.rs, opts. + let exts = Options::ENABLE_TABLES | Options::ENABLE_FOOTNOTES; + + let md = Parser::new_ext(content, exts); + + let mut found = false; + let mut output = None; + + for item in md { + match item { + Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(ref info))) + if info.to_lowercase() == "cargo" && output.is_none() => + { + found = true; + } + Event::Text(ref text) if found => { + let s = output.get_or_insert(String::new()); + s.push_str(text); + } + Event::End(Tag::CodeBlock(_)) if found => { + found = false; + } + _ => (), + } + } + + output +} + +/** +Extracts the contents of a Rust doc comment. +*/ +fn extract_comment(s: &str) -> Option { + use std::cmp::min; + + fn extract_block(s: &str) -> Option { + /* + On every line: + + - update nesting level and detect end-of-comment + - if margin is None: + - if there appears to be a margin, set margin. + - strip off margin marker + - update the leading space counter + - strip leading space + - append content + */ + let mut r = String::new(); + + let margin_re: Regex = Regex::new(r"^\s*\*( |$)").unwrap(); + let space_re: Regex = Regex::new(r"^(\s+)").unwrap(); + let nesting_re: Regex = Regex::new(r"/\*|\*/").unwrap(); + + let mut leading_space = None; + let mut margin = None; + let mut depth: u32 = 1; + + for line in s.lines() { + if depth == 0 { + break; + } + + let mut end_of_comment = None; + + for (end, marker) in nesting_re.find_iter(line).map(|m| (m.start(), m.as_str())) { + match (marker, depth) { + ("/*", _) => depth += 1, + ("*/", 1) => { + end_of_comment = Some(end); + depth = 0; + break; + } + ("*/", _) => depth -= 1, + _ => return None, + } + } + + let line = end_of_comment.map(|end| &line[..end]).unwrap_or(line); + + margin = margin.or_else(|| margin_re.find(line).map(|m| m.as_str())); + + let line = if let Some(margin) = margin { + let end = line + .char_indices() + .take(margin.len()) + .map(|(i, c)| i + c.len_utf8()) + .last() + .unwrap_or(0); + &line[end..] + } else { + line + }; + + leading_space = leading_space.or_else(|| space_re.find(line).map(|m| m.end())); + + let strip_len = min(leading_space.unwrap_or(0), line.len()); + let line = &line[strip_len..]; + + r.push_str(line); + + r.push('\n'); + } + + Some(r) + } + + fn extract_line(s: &str) -> Option { + let mut r = String::new(); + + let comment_re = Regex::new(r"^\s*//(!|/)").unwrap(); + + let space_re = Regex::new(r"^(\s+)").unwrap(); + + let mut leading_space = None; + + for line in s.lines() { + let content = match comment_re.find(line) { + Some(m) => &line[m.end()..], + None => break, + }; + + leading_space = leading_space.or_else(|| { + space_re + .captures(content) + .and_then(|c| c.get(1)) + .map(|m| m.end()) + }); + + let strip_len = min(leading_space.unwrap_or(0), content.len()); + let content = &content[strip_len..]; + + r.push_str(content); + + r.push('\n'); + } + + Some(r) + } + + if let Some(stripped) = s.strip_prefix("/*!") { + extract_block(stripped) + } else if s.starts_with("//!") || s.starts_with("///") { + extract_line(s) + } else { + None + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_parse_rust_signature() { + let code = r#" +// commenting comments + +fn main( + my_int: i8, + my_vec: Vec, + mut my_arr: [u8; 9], + my_complex: Vec>, +) -> Result { + println!("My int is {}", my_int); +}"#; + + let ret = parse_rust_signature(code).unwrap(); + + assert_eq!(ret.args.len(), 4); + + assert_eq!(ret.args[0].name, "my_int"); + assert_eq!(ret.args[0].otyp, Some("i8".to_string())); + assert_eq!(ret.args[0].typ, Typ::Int); + + assert_eq!(ret.args[1].name, "my_vec"); + assert_eq!(ret.args[1].otyp, Some("Vec < u8 >".to_string())); + assert_eq!(ret.args[1].typ, Typ::List(Box::new(Typ::Int))); + + assert_eq!(ret.args[2].name, "my_arr"); + assert_eq!(ret.args[2].otyp, Some("[u8 ; 9]".to_string())); + assert_eq!(ret.args[2].typ, Typ::List(Box::new(Typ::Int))); + + assert_eq!(ret.args[3].name, "my_complex"); + assert_eq!( + ret.args[3].otyp, + Some("Vec < Result < MyStruct , anyhow :: Error > >".to_string()) + ); + assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Unknown))); + + let code = r#" +// commenting comments + +fn main( + my_str_slice: &str, + my_String: String, + mut my_mut_ref_to_string: &mut String, + my_string_vec: Vec, +) -> Result { + println!("My int is {}", my_int); +}"#; + + let ret = parse_rust_signature(code).unwrap(); + + assert_eq!(ret.args.len(), 4); + + assert_eq!(ret.args[0].name, "my_str_slice"); + assert_eq!(ret.args[0].otyp, Some("& str".to_string())); + assert_eq!(ret.args[0].typ, Typ::Str(None)); + + assert_eq!(ret.args[1].name, "my_String"); + assert_eq!(ret.args[1].otyp, Some("String".to_string())); + assert_eq!(ret.args[1].typ, Typ::Str(None)); + + assert_eq!(ret.args[2].name, "my_mut_ref_to_string"); + assert_eq!(ret.args[2].otyp, Some("& mut String".to_string())); + assert_eq!(ret.args[2].typ, Typ::Str(None)); + + assert_eq!(ret.args[3].name, "my_string_vec"); + assert_eq!( + ret.args[3].otyp, + Some("Vec < String >".to_string()) + ); + assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None)))); + } + + #[test] + fn test_parse_rust_manifest() { + assert_eq!(find_embedded_manifest("fn main() {}"), None); + + let code = r#" + +//! Crate doc comment right here +//! We need to use the `cargo` language in the code bloc +//! +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` + +fn main() { + println!("{}", time::now().rfc822z()); +} + "#; + let manif = find_embedded_manifest(code); + + assert_eq!( + manif, + Some(Manifest::Toml( + r#"[dependencies] +time = "0.1.25" +"# + .to_string() + )) + ); + let _full_manif = parse_rust_deps_into_manifest(code).unwrap(); + } + + #[test] + fn test_default_manifest() { + default_manifest(); + } +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 7fd0e01202..8b00ca928e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -10,10 +10,6 @@ crate-type = ["cdylib"] name = "windmill_parser_wasm" path = "./src/lib.rs" -[profile.release] -lto = true -opt-level = 's' - [dev-dependencies] wasm-bindgen-test.workspace = true windmill-parser-ts.workspace = true @@ -27,6 +23,7 @@ sql-parser = [ "dep:windmill-parser-sql"] py-parser = [ "dep:windmill-parser-py"] ts-parser = [ "dep:windmill-parser-ts"] php-parser = [ "dep:windmill-parser-php"] +rust-parser = [ "dep:windmill-parser-rust"] graphql-parser = [ "dep:windmill-parser-graphql"] [dependencies] @@ -39,6 +36,7 @@ windmill-parser-py = { workspace = true, optional = true } windmill-parser-ts = { workspace = true, optional = true } windmill-parser-php = { workspace = true, optional = true } windmill-parser-graphql = { workspace = true, optional = true } +windmill-parser-rust = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true getrandom = { workspace = true, features = ["js"] } diff --git a/backend/parsers/windmill-parser-wasm/README_DEV.md b/backend/parsers/windmill-parser-wasm/README_DEV.md new file mode 100644 index 0000000000..ff3f866f4e --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/README_DEV.md @@ -0,0 +1,25 @@ +### Windmill parser wasm + +How to build + +``` +wasm-pack build --release --target web +``` + +#### Prerequisite + +Install wasm-pack + +``` +cargo install wasm-pack +``` + +#### To use it on a dev environment + +Go to frontend and run: + +``` +npm install ../backend/parsers/windmill-parser-wasm/pkg +``` + +Make sure to not reset the package.json before commiting diff --git a/backend/parsers/windmill-parser-wasm/build-pkgs.sh b/backend/parsers/windmill-parser-wasm/build-pkgs.sh index 636b5bf93b..3ac0f9b424 100755 --- a/backend/parsers/windmill-parser-wasm/build-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs.sh @@ -36,3 +36,9 @@ OUT_DIR="pkg-php" wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json + +# rust +OUT_DIR="pkg-rust" +wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \ + -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort +sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts index 6b6125640a..439090ee9a 100644 --- a/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts +++ b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts @@ -75,6 +75,11 @@ export function parse_graphql(code: string): string; * @returns {string} */ export function parse_php(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_rust(code: string): string; export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; @@ -95,6 +100,7 @@ export interface InitOutput { readonly parse_db_resource: (a: number, b: number, c: number) => void; readonly parse_graphql: (a: number, b: number, c: number) => void; readonly parse_php: (a: number, b: number, c: number) => void; + readonly parse_rust: (a: number, b: number, c: number) => void; readonly __wbindgen_malloc: (a: number, b: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; readonly __wbindgen_add_to_stack_pointer: (a: number) => number; diff --git a/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts index 1fdcb8941c..5e6938f873 100644 --- a/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts +++ b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts @@ -16,6 +16,7 @@ export function parse_mssql(a: number, b: number, c: number): void; export function parse_db_resource(a: number, b: number, c: number): void; export function parse_graphql(a: number, b: number, c: number): void; export function parse_php(a: number, b: number, c: number): void; +export function parse_rust(a: number, b: number, c: number): void; export function __wbindgen_malloc(a: number, b: number): number; export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number; export function __wbindgen_add_to_stack_pointer(a: number): number; diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index 6e4f029a87..7e2fd8beb2 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -1,10 +1,9 @@ #!/bin/bash set -eou pipefail -script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$(dirname "${BASH_SOURCE[0]}")" args=${1:-} -# bun and deno pushd "pkg-ts" && npm publish ${args} popd @@ -19,3 +18,6 @@ popd pushd "pkg-php" && npm publish ${args} popd + +pushd "pkg-rust" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index ddd555cfbf..24812f0bef 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -116,3 +116,9 @@ pub fn parse_graphql(code: &str) -> String { pub fn parse_php(code: &str) -> String { wrap_sig(windmill_parser_php::parse_php_signature(code, None)) } + +#[cfg(feature = "rust-parser")] +#[wasm_bindgen] +pub fn parse_rust(code: &str) -> String { + wrap_sig(windmill_parser_rust::parse_rust_signature(code)) +} diff --git a/backend/src/main.rs b/backend/src/main.rs index dc753f0aa4..e9ea129ab5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -64,7 +64,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, }; use crate::monitor::{ @@ -832,6 +832,7 @@ pub async fn run_workers) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +fn main(world: String) -> Result { + println!("Which world to greet today?"); + Ok(format!("Hello {}!", world)) +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Rust, + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + })) + .arg("world", json!("Hyrule")) + .run_until_complete(&db, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("Hello Hyrule!")); +} + #[sqlx::test(fixtures("base"))] async fn test_bash_job(db: Pool) { initialize_tracing().await; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0309f81a7c..ba2a259774 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9187,6 +9187,7 @@ components: nativets, bun, php, + rust, ] kind: type: string @@ -9285,6 +9286,7 @@ components: nativets, bun, php, + rust, ] kind: type: string @@ -9502,6 +9504,7 @@ components: nativets, bun, php, + rust, ] email: type: string @@ -9617,6 +9620,7 @@ components: nativets, bun, php, + rust, ] is_skipped: type: boolean @@ -10155,7 +10159,8 @@ components: graphql, nativets, bun, - php + php, + rust, ] tag: type: string @@ -11285,7 +11290,8 @@ components: graphql, nativets, bun, - php + php, + rust, ] required: - raw_code diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 662ebc06e4..61e8c553ee 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -580,6 +580,7 @@ async fn create_script_internal<'c>( || ns.language == ScriptLang::Bun || ns.language == ScriptLang::Bunnative || ns.language == ScriptLang::Deno + || ns.language == ScriptLang::Rust || ns.language == ScriptLang::Php) { Some(String::new()) diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 08509680cb..d826be203b 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2576,6 +2576,7 @@ async fn tarball_workspace( } } ScriptLang::Php => "php", + ScriptLang::Rust => "rs", }; archive .write_to_archive(&script.content, &format!("{}.{}", script.path, ext)) diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index b850f3e123..536e4bcda7 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -43,6 +43,7 @@ pub enum ScriptLang { Graphql, Mssql, Php, + Rust, } impl ScriptLang { @@ -63,6 +64,7 @@ impl ScriptLang { ScriptLang::Mssql => "mssql", ScriptLang::Graphql => "graphql", ScriptLang::Php => "php", + ScriptLang::Rust => "rust" } } } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 263f764e67..7752f4e93e 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -40,6 +40,7 @@ lazy_static::lazy_static! { "mssql".to_string(), "graphql".to_string(), "php".to_string(), + "rust".to_string(), "dependency".to_string(), "flow".to_string(), "other".to_string() diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index c4e59197fe..f526c1b00c 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -26,6 +26,7 @@ windmill-common = { workspace = true, default-features = false } windmill-parser.workspace = true windmill-parser-ts.workspace = true windmill-parser-go.workspace = true +windmill-parser-rust.workspace = true windmill-parser-py.workspace = true windmill-parser-py-imports.workspace = true windmill-parser-bash.workspace = true @@ -95,4 +96,4 @@ deno_console.workspace = true deno_url.workspace = true deno_core.workspace = true deno_net.workspace = true -zstd.workspace = true \ No newline at end of file +zstd.workspace = true diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto new file mode 100644 index 0000000000..502011049c --- /dev/null +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -0,0 +1,107 @@ +name: "rust run script" + +mode: ONCE +hostname: "rust" +log_level: ERROR + +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +keep_caps: false +keep_env: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{JOB_DIR}/main" + dst: "/tmp/main" + is_bind: true + mandatory: false +} + +mount { + src: "{JOB_DIR}/args.json" + dst: "/tmp/args.json" + is_bind: true +} + +mount { + src: "{JOB_DIR}/result.json" + dst: "/tmp/result.json" + rw: true + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +iface_no_lo: true + +mount { + src: "{CACHE_DIR}" + dst: "/tmp/.cache/rust" + is_bind: true + rw: true + mandatory: false +} + +{SHARED_MOUNT} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 336b2e5cf9..f3472613b6 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -20,6 +20,7 @@ mod mysql_executor; mod pg_executor; mod php_executor; mod python_executor; +mod rust_executor; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index c41af02dfd..cd2f7e2499 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -2,7 +2,7 @@ use convert_case::{Case, Casing}; use itertools::Itertools; use regex::Regex; use serde_json::value::RawValue; -use std::{collections::HashMap, process::Stdio}; +use std::{collections::HashMap, path::Path, process::Stdio}; use tokio::{fs::File, io::AsyncReadExt, process::Command}; use uuid::Uuid; use windmill_common::{ @@ -71,6 +71,8 @@ pub async fn composer_install( requirements: String, lock: Option, ) -> Result { + check_php_exists()?; + write_file(job_dir, "composer.json", &requirements)?; if let Some(lock) = lock.as_ref() { @@ -126,6 +128,24 @@ $args->{arg_name} = new {rt_name}($args->{arg_name});" ) } +#[cfg(not(feature = "enterprise"))] +fn check_php_exists() -> error::Result<()> { + if !Path::new(PHP_PATH.as_str()).exists() { + let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run php jobs.", PHP_PATH.as_str()); + return Err(error::Error::NotFound(msg)); + } + Ok(()) +} + +#[cfg(feature = "enterprise")] +fn check_php_exists() -> error::Result<()> { + if !Path::new(PHP_PATH.as_str()).exists() { + let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run php jobs.", PHP_PATH.as_str()); + return Err(error::Error::NotFound(msg)); + } + Ok(()) +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_php_job( requirements_o: Option, @@ -141,6 +161,8 @@ pub async fn handle_php_job( envs: HashMap, shared_mount: &str, ) -> error::Result> { + check_php_exists()?; + let (composer_json, composer_lock) = match requirements_o { Some(reqs_and_lock) if !reqs_and_lock.is_empty() => { let splitted = reqs_and_lock.split(COMPOSER_LOCK_SPLIT).collect_vec(); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs new file mode 100644 index 0000000000..3477eed51b --- /dev/null +++ b/backend/windmill-worker/src/rust_executor.rs @@ -0,0 +1,373 @@ +use serde_json::value::RawValue; +use std::{collections::HashMap, path::Path, process::Stdio}; +use uuid::Uuid; +use windmill_parser_rust::parse_rust_deps_into_manifest; + +use itertools::Itertools; +use tokio::{fs::File, io::AsyncReadExt, process::Command}; +use windmill_common::{ + error::{self, Error}, + jobs::QueuedJob, + utils::calculate_hash, + worker::{save_cache, write_file}, +}; +use windmill_queue::{append_logs, CanceledBy}; + +use crate::{ + common::{ + create_args_and_out_file, get_reserved_variables, handle_child, read_result, + start_child_process, + }, + AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, + RUST_CACHE_DIR, TZ_ENV, +}; + +const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); + +lazy_static::lazy_static! { + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string()); + static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string()); + static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string())); +} + +const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; + +fn gen_cargo_crate(code: &str, job_dir: &str) -> anyhow::Result<()> { + let manifest = parse_rust_deps_into_manifest(code)?; + write_file(job_dir, "Cargo.toml", &manifest)?; + + const WRAPPER_CONTENT: &str = r#" +use std::fs::File; +use std::io::{BufReader, Write}; +use std::error::Error; +mod inner; + +fn main() -> Result<(), Box> { + let args_file = File::open("args.json")?; + let reader = BufReader::new(args_file); + let args: inner::__WINDMILL_ARGS__ = serde_json::from_reader(reader)?; + + let result = inner::__WINDMILL_RUN__(args)?; + + let mut result_file = File::create("result.json")?; + result_file.write_all(result.into_bytes().as_ref())?; + + Ok(()) +} +"#; + write_file(job_dir, "main.rs", WRAPPER_CONTENT)?; + + let sig = windmill_parser_rust::parse_rust_signature(code)?; + let spread = &sig + .args + .clone() + .into_iter() + .map(|x| format!("_args.{}", &x.name)) + .join(", "); + let arg_struct_body = &sig + .args + .into_iter() + .map(|x| { + format!( + "{}: {},", + &x.name, + windmill_parser_rust::otyp_to_string(x.otyp), + ) + }) + .join("\n"); + let mod_content: String = format!( + r#" +{code} + +#[derive(serde::Deserialize)] +#[allow(non_camel_case_types)] +pub struct __WINDMILL_ARGS__ {{ + {arg_struct_body} +}} + + +#[allow(non_snake_case)] +pub fn __WINDMILL_RUN__(_args: __WINDMILL_ARGS__) -> Result> {{ + fn windmill_runner(f: impl core::ops::FnOnce() -> core::result::Result) -> Result + where + T: serde::Serialize, + E: std::fmt::Display, + {{ + Ok(serde_json::to_string(&f()?).unwrap()) + }} + Ok(windmill_runner(|| main({spread}))?) +}} +"# + ); + + write_file(job_dir, "inner.rs", &mod_content)?; + Ok(()) +} + +pub async fn generate_cargo_lockfile( + job_id: &Uuid, + code: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + w_id: &str, +) -> error::Result { + check_cargo_exists()?; + + gen_cargo_crate(code, job_dir)?; + + let mut gen_lockfile_cmd = Command::new(CARGO_PATH.as_str()); + gen_lockfile_cmd + .current_dir(job_dir) + .args(vec!["generate-lockfile"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + gen_lockfile_process, + false, + worker_name, + w_id, + "cargo generate-lockfile", + None, + false, + ) + .await?; + + let path_lock = format!("{job_dir}/Cargo.lock"); + let mut file = File::open(path_lock).await?; + let mut req_content = String::new(); + file.read_to_string(&mut req_content).await?; + Ok(req_content) +} + +pub async fn build_rust_crate( + job_id: &Uuid, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + w_id: &str, + base_internal_url: &str, + hash: &str, +) -> error::Result { + let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + + let mut build_rust_cmd = Command::new(CARGO_PATH.as_str()); + build_rust_cmd + .current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .args(vec!["build", "--release"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + build_rust_process, + false, + worker_name, + w_id, + "rust build", + None, + false, + ) + .await?; + append_logs(job_id, w_id, "\n\n", db).await; + + tokio::fs::copy( + &format!("{job_dir}/target/release/main"), + format! {"{job_dir}/main"}, + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "could not copy built binary from [...]/target/release/main to {job_dir}/main: {e:?}" + )) + })?; + + match save_cache( + &bin_path, + &format!("{RUST_OBJECT_STORE_PREFIX}{hash}"), + &format!("{job_dir}/main"), + ) + .await + { + Err(e) => { + let em = format!( + "could not save {bin_path} to {} to rust cache: {e:?}", + format!("{job_dir}/main"), + ); + tracing::error!(em); + Ok(em) + } + Ok(logs) => Ok(logs), + } +} + +pub fn compute_rust_hash(code: &str, requirements_o: Option<&String>) -> String { + calculate_hash(&format!( + "{}{}", + code, + requirements_o + .as_ref() + .map(|x| x.to_string()) + .unwrap_or_default() + )) +} + +#[cfg(not(feature = "enterprise"))] +fn check_cargo_exists() -> Result<(), Error> { + if !Path::new(CARGO_PATH.as_str()).exists() { + let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); + return Err(Error::NotFound(msg)); + } + Ok(()) +} + +#[cfg(feature = "enterprise")] +fn check_cargo_exists() -> Result<(), Error> { + if !Path::new(CARGO_PATH.as_str()).exists() { + let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); + return Err(Error::NotFound(msg)); + } + Ok(()) +} + +#[tracing::instrument(level = "trace", skip_all)] +pub async fn handle_rust_job( + mem_peak: &mut i32, + canceled_by: &mut Option, + job: &QueuedJob, + db: &sqlx::Pool, + client: &AuthedClientBackgroundTask, + inner_content: &str, + job_dir: &str, + requirements_o: Option, + shared_mount: &str, + base_internal_url: &str, + worker_name: &str, + envs: HashMap, +) -> Result, Error> { + check_cargo_exists()?; + + let hash = compute_rust_hash(inner_content, requirements_o.as_ref()); + let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); + + let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await; + + let cache_logs = if cache { + let target = format!("{job_dir}/main"); + std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + Error::ExecutionErr(format!( + "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + )) + })?; + + create_args_and_out_file(client, job, job_dir, db).await?; + cache_logs + } else { + let logs1 = format!("{cache_logs}\n\n--- CARGO BUILD ---\n"); + append_logs(&job.id, &job.workspace_id, logs1, db).await; + + gen_cargo_crate(inner_content, job_dir)?; + + if let Some(reqs) = requirements_o { + if !reqs.is_empty() { + write_file(job_dir, "Cargo.lock", &reqs)?; + } + } + + create_args_and_out_file(client, job, job_dir, db).await?; + + build_rust_crate( + &job.id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + &job.workspace_id, + base_internal_url, + &hash, + ) + .await? + }; + + let logs2 = format!("{cache_logs}\n\n--- RUST CODE EXECUTION ---\n"); + append_logs(&job.id, &job.workspace_id, logs2, db).await; + + let client = &client.get_authed().await; + let reserved_variables = get_reserved_variables(job, &client.token, db).await?; + + let child = if !*DISABLE_NSJAIL { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_RUST_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount), + )?; + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec!["--config", "run.config.proto", "--", "/tmp/main"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? + } else { + let compiled_executable_name = "./main"; + let mut run_rust = Command::new(compiled_executable_name); + run_rust + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + start_child_process(run_rust, compiled_executable_name).await? + }; + handle_child( + &job.id, + db, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + &job.workspace_id, + "rust run", + job.timeout, + false, + ) + .await?; + read_result(job_dir).await +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 96404535d5..92f2a2d0da 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -82,9 +82,7 @@ use rand::Rng; use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ - bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE}, - bun_executor::handle_bun_job, - common::{ + bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE}, bun_executor::handle_bun_job, common::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, read_result, save_in_cache, NO_LOGS_AT_ALL, SLOW_LOGS, }, @@ -94,14 +92,14 @@ use crate::{ js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, + rust_executor::handle_rust_job, php_executor::handle_php_job, python_executor::handle_python_job, worker_flow::{ handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, - }, - worker_lockfiles::{ + }, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, - }, + } }; #[cfg(feature = "enterprise")] @@ -234,6 +232,7 @@ pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); +pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); pub const BUN_DEPSTAR_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "buntar"); @@ -3153,6 +3152,23 @@ mount {{ &shared_mount, ) .await + }, + Some(ScriptLang::Rust) => { + handle_rust_job( + mem_peak, + canceled_by, + job, + db, + client, + &inner_content, + job_dir, + requirements_o, + &shared_mount, + base_internal_url, + worker_name, + envs, + ) + .await } _ => panic!("unreachable, language is not supported: {language:#?}"), }; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 224d254cb1..5ea3210c82 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -26,6 +26,7 @@ use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}; +use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ bun_executor::gen_lockfile, deno_executor::generate_deno_lock, @@ -1405,6 +1406,39 @@ async fn capture_dependency_job( ) .await } + ScriptLang::Rust => { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for rust".to_string(), + )); + } + + let lockfile = generate_cargo_lockfile( + job_id, + job_raw_code, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + ) + .await?; + + build_rust_crate( + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + base_internal_url, + &compute_rust_hash(&job_raw_code, Some(&lockfile)), + ) + .await?; + Ok(lockfile) + } ScriptLang::Postgresql => Ok("".to_owned()), ScriptLang::Mysql => Ok("".to_owned()), ScriptLang::Bigquery => Ok("".to_owned()), diff --git a/docker/DockerfileNsjail b/docker/DockerfileNsjail index 9aee2de39b..a6896834f5 100644 --- a/docker/DockerfileNsjail +++ b/docker/DockerfileNsjail @@ -25,4 +25,7 @@ FROM ghcr.io/windmill-labs/windmill-ee:dev RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev +COPY --from=rust:1.80.1 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.80.1 /usr/local/rustup /usr/local/rustup + COPY --from=nsjail /nsjail/nsjail /bin/nsjail diff --git a/docker/DockerfileRust b/docker/DockerfileRust new file mode 100644 index 0000000000..4ed6ed2cec --- /dev/null +++ b/docker/DockerfileRust @@ -0,0 +1,4 @@ +FROM ghcr.io/windmill-labs/windmill-ee:dev + +COPY --from=rust:1.80.1 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.80.1 /usr/local/rustup /usr/local/rustup \ No newline at end of file diff --git a/docker/DockerfileRustEe b/docker/DockerfileRustEe new file mode 100644 index 0000000000..4ed6ed2cec --- /dev/null +++ b/docker/DockerfileRustEe @@ -0,0 +1,4 @@ +FROM ghcr.io/windmill-labs/windmill-ee:dev + +COPY --from=rust:1.80.1 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.80.1 /usr/local/rustup /usr/local/rustup \ No newline at end of file diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 47d26b0470..54bfe26636 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -15,7 +15,7 @@ ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools -COPY --from=oven/bun:1.1.8 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.26 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ @@ -23,7 +23,7 @@ COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ WORKDIR ${APP} COPY --from=ghcr.io/windmill-labs/windmill-ee:dev --chmod=755 ${APP}/windmill ${APP}/windmill -COPY --from=denoland/deno:1.44.4 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:1.46.1 --chmod=755 /usr/bin/deno /usr/bin/deno RUN ln -s ${APP}/windmill /usr/local/bin/windmill diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe new file mode 100644 index 0000000000..54bfe26636 --- /dev/null +++ b/docker/DockerfileSlimEe @@ -0,0 +1,38 @@ +ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm + +FROM ${PYTHON_IMAGE} + +ARG APP=/usr/src/app + +RUN apt-get update \ + && apt-get install -y ca-certificates wget curl git jq unzip build-essential unixodbc xmlsec1 software-properties-common \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - +RUN apt-get -y update && apt-get install -y curl nodejs awscli + +ENV TZ=Etc/UTC + +RUN /usr/local/bin/python3 -m pip install pip-tools + +COPY --from=oven/bun:1.1.26 /usr/local/bin/bun /usr/bin/bun + +# add the docker client to call docker from a worker if enabled +COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ + +WORKDIR ${APP} + +COPY --from=ghcr.io/windmill-labs/windmill-ee:dev --chmod=755 ${APP}/windmill ${APP}/windmill +COPY --from=denoland/deno:1.46.1 --chmod=755 /usr/bin/deno /usr/bin/deno + +RUN ln -s ${APP}/windmill /usr/local/bin/windmill + +COPY ./frontend/src/lib/hubPaths.json ${APP}/hubPaths.json + +RUN windmill cache ${APP}/hubPaths.json + +RUN rm ${APP}/hubPaths.json + +EXPOSE 8000 + +CMD ["windmill"] diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 65dcadc623..ae6f7ea75f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -70,6 +70,7 @@ "windmill-parser-wasm-php": "^1.382.2", "windmill-parser-wasm-py": "^1.382.2", "windmill-parser-wasm-regex": "^1.382.2", + "windmill-parser-wasm-rust": "^1.387.0", "windmill-parser-wasm-ts": "^1.382.2", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -13426,6 +13427,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.385.0.tgz", "integrity": "sha512-rgqTANyiYNgdh2Hcgk8xFt/rmLCMGMnIlnZe+/9z2oggfkOKB9oAt49cyCLD9MZeXdTJuNPKn+IVMeSAOn6WYQ==" }, + "node_modules/windmill-parser-wasm-rust": { + "version": "1.387.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.387.0.tgz", + "integrity": "sha512-PX0Y/sdaaCkc9rrXB6m+AntGq/RX21fYhdpC7MFCMKb9VakkDAGnFo435xhAi74yLarGd03odUxGymokcBfU6g==" + }, "node_modules/windmill-parser-wasm-ts": { "version": "1.385.0", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.385.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 4996075c51..d1e3883252 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -139,6 +139,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.3.2", + "windmill-parser-wasm-rust": "^1.387.0", "windmill-parser-wasm-php": "^1.382.2", "windmill-parser-wasm-go": "^1.382.2", "windmill-parser-wasm-py": "^1.382.2", diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 11253e4610..73d7f08197 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -188,6 +188,7 @@ | 'php' | 'css' | 'javascript' + | 'rust' export let code: string = '' export let cmdEnterAction: (() => void) | undefined = undefined export let formatAction: (() => void) | undefined = undefined diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 662f9740cc..e89584ea44 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -90,7 +90,8 @@ 'bun', 'bunnative', 'nativets', - 'php' + 'php', + 'rust' ].includes(lang ?? '') $: showVarPicker = [ 'python3', @@ -101,7 +102,8 @@ 'bun', 'bunnative', 'nativets', - 'php' + 'php', + 'rust' ].includes(lang ?? '') $: showResourcePicker = [ 'python3', @@ -112,7 +114,8 @@ 'bun', 'bunnative', 'nativets', - 'php' + 'php', + 'rust' ].includes(lang ?? '') $: showResourceTypePicker = ['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) || diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index 382e6cc553..04048e72d5 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -9,6 +9,7 @@ import sql from 'svelte-highlight/languages/sql' import powershell from 'svelte-highlight/languages/powershell' import php from 'svelte-highlight/languages/php' + import rust from 'svelte-highlight/languages/rust' import type { Script } from '$lib/gen' import { Button } from './common' import { copyToClipboard } from '$lib/utils' @@ -51,7 +52,8 @@ return powershell case 'php': return php - + case 'rust': + return rust default: return typescript } diff --git a/frontend/src/lib/components/WorkspaceGroup.svelte b/frontend/src/lib/components/WorkspaceGroup.svelte index 217ba4eb65..83d2c24bfe 100644 --- a/frontend/src/lib/components/WorkspaceGroup.svelte +++ b/frontend/src/lib/components/WorkspaceGroup.svelte @@ -112,7 +112,8 @@ 'flow', 'other', 'bun', - 'php' + 'php', + 'rust' ] const nativeTags = [ 'nativets', diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 1ef98ce443..fcdb23bfec 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -16,6 +16,7 @@ import DenoIcon from '$lib/components/icons/DenoIcon.svelte' import type { Script } from '$lib/gen' import PHPIcon from '$lib/components/icons/PHPIcon.svelte' + import RustIcon from '$lib/components/icons/RustIcon.svelte' export let lang: | SupportedLanguage @@ -46,7 +47,8 @@ mysql: 'MySQL', mssql: 'MS SQL Server', bun: 'TypeScript', - php: 'PHP' + php: 'PHP', + rust: 'Rust' } const langToComponent: Record< @@ -72,7 +74,8 @@ postgresql: PostgresIcon, nativets: RestIcon, graphql: GraphqlIcon, - php: PHPIcon + php: PHPIcon, + rust: RustIcon } let subIconScale = width === 30 ? 0.6 : 0.8 diff --git a/frontend/src/lib/components/icons/RustIcon.svelte b/frontend/src/lib/components/icons/RustIcon.svelte new file mode 100644 index 0000000000..36b3ed30c1 --- /dev/null +++ b/frontend/src/lib/components/icons/RustIcon.svelte @@ -0,0 +1,70 @@ + + + + + + diff --git a/frontend/src/lib/components/icons/rust-logo.svg b/frontend/src/lib/components/icons/rust-logo.svg new file mode 100644 index 0000000000..1a6c762d4e --- /dev/null +++ b/frontend/src/lib/components/icons/rust-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/lib/editorUtils.ts b/frontend/src/lib/editorUtils.ts index 8425bd2005..b4684f914b 100644 --- a/frontend/src/lib/editorUtils.ts +++ b/frontend/src/lib/editorUtils.ts @@ -67,6 +67,8 @@ export function langToExt(lang: string): string { return 'ps1' case 'php': return 'php' + case 'rust': + return 'rs' case 'deno': return 'ts' case 'nativets': diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index e0777caa98..edd8774c55 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -29,12 +29,16 @@ import initGoParser, { import initPhpParser, { parse_php, } from 'windmill-parser-wasm-php' +import initRustParser, { + parse_rust, +} from 'windmill-parser-wasm-rust' import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url' import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url' import wasmUrlPy from 'windmill-parser-wasm-py/windmill_parser_wasm_bg.wasm?url' import wasmUrlGo from 'windmill-parser-wasm-go/windmill_parser_wasm_bg.wasm?url' import wasmUrlPhp from 'windmill-parser-wasm-php/windmill_parser_wasm_bg.wasm?url' +import wasmUrlRust from 'windmill-parser-wasm-rust/windmill_parser_wasm_bg.wasm?url' import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from './inferArgSig.js' @@ -56,6 +60,9 @@ async function initWasmPython() { async function initWasmPhp() { await initPhpParser(wasmUrlPhp) } +async function initWasmRust() { + await initRustParser(wasmUrlRust) +} async function initWasmGo() { await initGoParser(wasmUrlGo) } @@ -160,6 +167,9 @@ export async function inferArgs( } else if (language == 'php') { await initWasmPhp() inferedSchema = JSON.parse(parse_php(code)) + } else if (language == 'rust') { + await initWasmRust() + inferedSchema = JSON.parse(parse_rust(code)) } else { return null } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 326cea6d83..01f023ad92 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -252,6 +252,43 @@ function main( } ` +export const RUST_INIT_CODE = `//! Add dependencies in the following partial Cargo.toml manifest +//! +//! \`\`\`cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! rand = "0.7.2" +//! \`\`\` +//! +//! Note that serde is used by default with the \`derive\` feature. +//! You can still reimport it if you need additional features. + +use anyhow::anyhow; +use rand::seq::SliceRandom; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct Ret { + msg: String, + number: i8, +} + +fn main(who_to_greet: String, numbers: Vec) -> anyhow::Result { + println!( + "Person to greet: {} - numbers to choose: {:?}", + who_to_greet, numbers + ); + Ok(Ret { + msg: format!("Greetings {}!", who_to_greet), + number: *numbers + .choose(&mut rand::thread_rng()) + .ok_or(anyhow!("There should be some numbers to choose from"))?, + }) +} +` + + + export const FETCH_INIT_CODE = `export async function main( url: string | undefined, method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'OPTIONS' = 'GET', @@ -465,7 +502,8 @@ const ALL_INITIAL_CODE = [ BUN_INIT_CODE_APPROVAL, BASH_INIT_CODE, POWERSHELL_INIT_CODE, - PHP_INIT_CODE + PHP_INIT_CODE, + RUST_INIT_CODE, ] export function isInitialCode(content: string): boolean { @@ -552,6 +590,8 @@ export function initialCode( return GRAPHQL_INIT_CODE } else if (language == 'php') { return PHP_INIT_CODE + } else if (language == 'rust') { + return RUST_INIT_CODE } else if (language == 'bun' || language == 'bunnative') { if (language == 'bunnative' || subkind === 'bunnative') { return BUNNATIVE_INIT_CODE diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 9c4153fb58..c46fc7ae9c 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -33,6 +33,8 @@ export function scriptLangToEditorLang( return 'powershell' } else if (lang == 'php') { return 'php' + } else if (lang == 'rust') { + return 'rust' } else if (lang == 'graphql') { return 'graphql' } else if (lang == undefined) { @@ -110,7 +112,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['graphql', 'GraphQL'], ['powershell', 'PowerShell'], ['php', 'PHP'], - ['docker', 'Docker'] + ['rust', 'Rust'], + ['docker', 'Docker'], ] export function processLangs(selected: string | undefined, langs: string[]): string[] { if (selected === 'nativets') { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 01ffcd5b40..67dc95a065 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -647,7 +647,7 @@ priority: {job.priority} {/if} - {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'nativets', 'bash', 'powershell', 'php', 'other', 'dependency'].includes(job.tag)} + {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'dependency'].includes(job.tag)}
Tag: {job.tag}