From 6263e907356aaefc4c7b8fa9f2858ef43b196ae8 Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Sat, 29 Oct 2022 11:58:06 +0200 Subject: [PATCH] restructure the entire backend layout using workspaces (#815) --- .github/DockerfileBackendTests | 7 +- .github/change-versions.sh | 2 +- .github/workflows/backend-test.yml | 6 +- Dockerfile | 65 +- backend/Cargo.lock | 1407 +++--- backend/Cargo.toml | 94 +- backend/README.md | 15 + backend/parsers/windmill-parser-go/Cargo.toml | 17 + .../windmill-parser-go/src/lib.rs} | 21 +- .../windmill-parser-go}/src/parser_go_ast.rs | 0 .../src/parser_go_scanner.rs | 0 .../src/parser_go_token.rs | 0 backend/parsers/windmill-parser-py/Cargo.toml | 19 + .../windmill-parser-py/src/lib.rs} | 165 +- backend/parsers/windmill-parser-ts/Cargo.toml | 19 + .../windmill-parser-ts/src/lib.rs} | 28 +- backend/parsers/windmill-parser/Cargo.toml | 13 + .../windmill-parser/src/lib.rs} | 0 backend/src/client.rs | 47 - backend/src/main.rs | 111 +- backend/src/users.rs | 1549 ------- backend/src/worker.rs | 3934 ----------------- backend/{src => tests}/fixtures/base.sql | 0 backend/tests/worker.rs | 1905 ++++++++ backend/windmill-api-client/.gitignore | 1 + backend/windmill-api-client/Cargo.toml | 25 + backend/windmill-api-client/README.md | 8 + backend/windmill-api-client/build.rs | 22 + backend/windmill-api-client/bundle.sh | 3 + backend/windmill-api-client/src/lib.rs | 14 + backend/windmill-api/Cargo.toml | 64 + backend/windmill-api/README.md | 5 + backend/{ => windmill-api}/openapi.yaml | 62 +- backend/windmill-api/src/audit.rs | 44 + backend/{ => windmill-api}/src/capture.rs | 14 +- backend/{ => windmill-api}/src/db.rs | 18 +- backend/{ => windmill-api}/src/flows.rs | 240 +- .../{ => windmill-api}/src/granular_acls.rs | 17 +- backend/{ => windmill-api}/src/groups.rs | 27 +- backend/{ => windmill-api}/src/jobs.rs | 1968 +++------ backend/{ => windmill-api}/src/lib.rs | 165 +- backend/windmill-api/src/main.rs | 70 + backend/{ => windmill-api}/src/oauth2.rs | 40 +- backend/{ => windmill-api}/src/resources.rs | 18 +- backend/windmill-api/src/schedule.rs | 126 + backend/{ => windmill-api}/src/scripts.rs | 265 +- .../{ => windmill-api}/src/static_assets.rs | 2 +- backend/windmill-api/src/tracing_init.rs | 45 + backend/windmill-api/src/utils.rs | 30 + backend/{ => windmill-api}/src/variables.rs | 149 +- backend/{ => windmill-api}/src/worker_ping.rs | 8 +- backend/{ => windmill-api}/src/workspaces.rs | 20 +- backend/windmill-audit/Cargo.toml | 17 + .../audit.rs => windmill-audit/src/lib.rs} | 47 +- backend/windmill-common/Cargo.toml | 46 + backend/{ => windmill-common}/src/error.rs | 22 +- .../{ => windmill-common}/src/external_ip.rs | 8 + backend/windmill-common/src/flows.rs | 220 + backend/windmill-common/src/lib.rs | 132 + .../{ => windmill-common}/src/more_serde.rs | 8 + backend/windmill-common/src/oauth2.rs | 12 + backend/windmill-common/src/scripts.rs | 257 ++ .../{ => windmill-common}/src/tracing_init.rs | 48 +- backend/windmill-common/src/users.rs | 12 + backend/{ => windmill-common}/src/utils.rs | 51 +- backend/windmill-common/src/variables.rs | 118 + backend/windmill-common/src/worker_flow.rs | 162 + backend/windmill-queue/Cargo.toml | 28 + backend/windmill-queue/src/jobs.rs | 572 +++ backend/windmill-queue/src/lib.rs | 12 + backend/{ => windmill-queue}/src/schedule.rs | 151 +- backend/windmill-worker/Cargo.toml | 42 + backend/windmill-worker/README.md | 5 + .../nsjail}/download.py.config.proto | 0 .../nsjail}/download_deps.py.sh | 0 .../nsjail}/run.deno.config.proto | 0 .../nsjail}/run.go.config.proto | 0 .../nsjail}/run.python3.config.proto | 0 backend/windmill-worker/src/jobs.rs | 175 + backend/{ => windmill-worker}/src/js_eval.rs | 70 +- backend/windmill-worker/src/lib.rs | 6 + backend/windmill-worker/src/main.rs | 133 + backend/windmill-worker/src/worker.rs | 2221 ++++++++++ .../{ => windmill-worker}/src/worker_flow.rs | 505 +-- deno-client/build.sh | 2 +- frontend/package.json | 2 +- go-client/build.sh | 2 +- openflow.openapi.yaml | 2 + python-client/build.sh | 2 +- 89 files changed, 9009 insertions(+), 8975 deletions(-) create mode 100644 backend/README.md create mode 100644 backend/parsers/windmill-parser-go/Cargo.toml rename backend/{src/parser_go.rs => parsers/windmill-parser-go/src/lib.rs} (98%) rename backend/{ => parsers/windmill-parser-go}/src/parser_go_ast.rs (100%) rename backend/{ => parsers/windmill-parser-go}/src/parser_go_scanner.rs (100%) rename backend/{ => parsers/windmill-parser-go}/src/parser_go_token.rs (100%) create mode 100644 backend/parsers/windmill-parser-py/Cargo.toml rename backend/{src/parser_py.rs => parsers/windmill-parser-py/src/lib.rs} (78%) create mode 100644 backend/parsers/windmill-parser-ts/Cargo.toml rename backend/{src/parser_ts.rs => parsers/windmill-parser-ts/src/lib.rs} (95%) create mode 100644 backend/parsers/windmill-parser/Cargo.toml rename backend/{src/parser.rs => parsers/windmill-parser/src/lib.rs} (100%) delete mode 100644 backend/src/client.rs delete mode 100644 backend/src/users.rs delete mode 100644 backend/src/worker.rs rename backend/{src => tests}/fixtures/base.sql (100%) create mode 100644 backend/tests/worker.rs create mode 100644 backend/windmill-api-client/.gitignore create mode 100644 backend/windmill-api-client/Cargo.toml create mode 100644 backend/windmill-api-client/README.md create mode 100644 backend/windmill-api-client/build.rs create mode 100644 backend/windmill-api-client/bundle.sh create mode 100644 backend/windmill-api-client/src/lib.rs create mode 100644 backend/windmill-api/Cargo.toml create mode 100644 backend/windmill-api/README.md rename backend/{ => windmill-api}/openapi.yaml (98%) create mode 100644 backend/windmill-api/src/audit.rs rename backend/{ => windmill-api}/src/capture.rs (91%) rename backend/{ => windmill-api}/src/db.rs (75%) rename backend/{ => windmill-api}/src/flows.rs (71%) rename backend/{ => windmill-api}/src/granular_acls.rs (92%) rename backend/{ => windmill-api}/src/groups.rs (91%) rename backend/{ => windmill-api}/src/jobs.rs (61%) rename backend/{ => windmill-api}/src/lib.rs (58%) create mode 100644 backend/windmill-api/src/main.rs rename backend/{ => windmill-api}/src/oauth2.rs (97%) rename backend/{ => windmill-api}/src/resources.rs (96%) create mode 100644 backend/windmill-api/src/schedule.rs rename backend/{ => windmill-api}/src/scripts.rs (73%) rename backend/{ => windmill-api}/src/static_assets.rs (98%) create mode 100644 backend/windmill-api/src/tracing_init.rs create mode 100644 backend/windmill-api/src/utils.rs rename backend/{ => windmill-api}/src/variables.rs (71%) rename backend/{ => windmill-api}/src/worker_ping.rs (87%) rename backend/{ => windmill-api}/src/workspaces.rs (98%) create mode 100644 backend/windmill-audit/Cargo.toml rename backend/{src/audit.rs => windmill-audit/src/lib.rs} (77%) create mode 100644 backend/windmill-common/Cargo.toml rename backend/{ => windmill-common}/src/error.rs (78%) rename backend/{ => windmill-common}/src/external_ip.rs (64%) create mode 100644 backend/windmill-common/src/flows.rs create mode 100644 backend/windmill-common/src/lib.rs rename backend/{ => windmill-common}/src/more_serde.rs (52%) create mode 100644 backend/windmill-common/src/oauth2.rs create mode 100644 backend/windmill-common/src/scripts.rs rename backend/{ => windmill-common}/src/tracing_init.rs (65%) create mode 100644 backend/windmill-common/src/users.rs rename backend/{ => windmill-common}/src/utils.rs (79%) create mode 100644 backend/windmill-common/src/variables.rs create mode 100644 backend/windmill-common/src/worker_flow.rs create mode 100644 backend/windmill-queue/Cargo.toml create mode 100644 backend/windmill-queue/src/jobs.rs create mode 100644 backend/windmill-queue/src/lib.rs rename backend/{ => windmill-queue}/src/schedule.rs (74%) create mode 100644 backend/windmill-worker/Cargo.toml create mode 100644 backend/windmill-worker/README.md rename {nsjail => backend/windmill-worker/nsjail}/download.py.config.proto (100%) rename {nsjail => backend/windmill-worker/nsjail}/download_deps.py.sh (100%) rename {nsjail => backend/windmill-worker/nsjail}/run.deno.config.proto (100%) rename {nsjail => backend/windmill-worker/nsjail}/run.go.config.proto (100%) rename {nsjail => backend/windmill-worker/nsjail}/run.python3.config.proto (100%) create mode 100644 backend/windmill-worker/src/jobs.rs rename backend/{ => windmill-worker}/src/js_eval.rs (82%) create mode 100644 backend/windmill-worker/src/lib.rs create mode 100644 backend/windmill-worker/src/main.rs create mode 100644 backend/windmill-worker/src/worker.rs rename backend/{ => windmill-worker}/src/worker_flow.rs (80%) diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 0a6ae25b14..7eefa861f4 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -30,7 +30,6 @@ RUN apt-get -y update \ ENV SQLX_OFFLINE=true -COPY ./nsjail /nsjail RUN mkdir -p /frontend/build RUN apt-get update \ @@ -38,7 +37,7 @@ RUN apt-get update \ make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \ libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \ - libv8-dev tesseract-ocr golang-go \ + libv8-dev tesseract-ocr nodejs npm\ && rm -rf /var/lib/apt/lists/* RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz @@ -62,4 +61,6 @@ COPY --from=nsjail /nsjail/nsjail /bin/nsjail COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno RUN apt-get update \ - && apt-get install -y postgresql-client + && apt-get install -y postgresql-client --allow-unauthenticated + +RUN rustup component add rustfmt \ No newline at end of file diff --git a/.github/change-versions.sh b/.github/change-versions.sh index a6df296c44..c5c22a05f8 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -4,7 +4,7 @@ VERSION=$1 echo "Updating versions to: $VERSION" sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml -sed -i -e "/version: /s/: .*/: $VERSION/" backend/openapi.yaml +sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 6e482a76ff..6b74939695 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -35,7 +35,9 @@ jobs: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 with: - workspaces: backend -> target + workspaces: | + backend + backend -> target - name: cargo test timeout-minutes: 5 - run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test -- --nocapture + run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture diff --git a/Dockerfile b/Dockerfile index f2ce3a8fc6..69a838932e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN npm ci # Copy all local files into the image. COPY frontend . RUN mkdir /backend -COPY /backend/openapi.yaml /backend/openapi.yaml +COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml COPY /openflow.openapi.yaml /openflow.openapi.yaml RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" @@ -38,29 +38,68 @@ RUN npm run check FROM rust:slim-buster as builder -RUN apt-get update && apt-get install -y git libssl-dev pkg-config +RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm + +RUN apt-get -y update \ + && apt-get install -y \ + curl lld nodejs npm + +RUN rustup component add rustfmt RUN USER=root cargo new --bin windmill WORKDIR /windmill -COPY ./backend/Cargo.toml . -COPY ./backend/Cargo.lock . -COPY ./backend/.cargo/ .cargo/ +COPY ./openflow.openapi.yaml /openflow.openapi.yaml -RUN apt-get -y update \ - && apt-get install -y \ - curl lld +RUN USER=root cargo new --bin windmill +RUN USER=root cargo new --lib windmill-api +RUN USER=root cargo new --lib windmill-audit +RUN USER=root cargo new --lib windmill-queue +RUN USER=root cargo new --lib windmill-worker +WORKDIR /windmill/parsers +RUN USER=root cargo new --lib windmill-parser +RUN USER=root cargo new --lib windmill-parser-go +RUN USER=root cargo new --lib windmill-parser-py +RUN USER=root cargo new --lib windmill-parser-ts +WORKDIR /windmill + +ENV SQLX_OFFLINE=true + +# COPY ./backend/Cargo.toml . +# COPY ./backend/windmill-api/Cargo.toml ./windmill-api/ +# COPY ./backend/windmill-audit/Cargo.toml ./windmill-audit/ +# COPY ./backend/sqlx-data.json ./ +# COPY ./backend/windmill-common ./windmill-common +# COPY ./backend/windmill-queue/Cargo.toml ./windmill-common/ +# COPY ./backend/windmill-queue/Cargo.toml ./windmill-queue/ +# COPY ./backend/windmill-worker/Cargo.toml ./windmill-worker/ +# COPY ./backend/parsers/windmill-parser/Cargo.toml ./parsers/windmill-parser/ +# COPY ./backend/parsers/windmill-parser-go/Cargo.toml ./parsers/windmill-parser-go/ +# COPY ./backend/parsers/windmill-parser-py/Cargo.toml ./parsers/windmill-parser-py/ +# COPY ./backend/parsers/windmill-parser-ts/Cargo.toml ./parsers/windmill-parser-ts/ +# COPY ./backend/.cargo/ .cargo/ + +# COPY ./backend/windmill-api-client/ ./windmill-api-client/ +# COPY ./backend/windmill-api/openapi.yaml ./windmill-api/openapi.yaml ENV CARGO_INCREMENTAL=1 -RUN cargo build --release -RUN rm src/*.rs +# RUN cargo build --release +# RUN rm ./src/*.rs +# RUN rm ./windmill-api/src/*.rs +# RUN rm ./windmill-api-client/src/*.rs +# RUN rm ./windmill-audit/src/*.rs +# RUN rm ./windmill-common/src/*.rs +# RUN rm ./windmill-queue/src/*.rs +# RUN rm ./windmill-worker/src/*.rs +# RUN rm ./parsers/windmill-parser/src/*.rs +# RUN rm ./parsers/windmill-parser-go/src/*.rs +# RUN rm ./parsers/windmill-parser-py/src/*.rs +# RUN rm ./parsers/windmill-parser-ts/src/*.rs -RUN rm ./target/release/deps/windmill* -ENV SQLX_OFFLINE=true +# RUN rm -r ./target/release/deps/windmill* COPY ./backend ./ -COPY ./nsjail /nsjail COPY --from=frontend /frontend /frontend COPY .git/ .git/ diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 085e8ecf17..aab9862080 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -27,7 +27,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] @@ -36,7 +36,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.8", + "getrandom", "once_cell", "version_check", ] @@ -77,22 +77,16 @@ dependencies = [ ] [[package]] -name = "arrayref" -version = "0.3.6" +name = "ascii" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" - -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "ascii-canvas" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8eb72df928aafb99fe5d37b383f2fe25bd2a765e3e5f7c365916b6f2463a29" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" dependencies = [ "term", ] @@ -113,11 +107,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" dependencies = [ "event-listener", + "futures-lite", ] [[package]] @@ -129,7 +124,7 @@ dependencies = [ "base64", "bytes", "http", - "rand 0.8.5", + "rand", "reqwest", "serde", "serde-aux", @@ -213,15 +208,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.1.0", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -297,6 +283,15 @@ dependencies = [ "scoped-tls", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -327,37 +322,14 @@ dependencies = [ "digest 0.10.5", ] -[[package]] -name = "blake2b_simd" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding 0.1.5", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "block-padding 0.2.1", - "generic-array 0.14.6", + "block-padding", + "generic-array", ] [[package]] @@ -366,7 +338,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ - "generic-array 0.14.6", + "generic-array", ] [[package]] @@ -375,37 +347,33 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" dependencies = [ - "block-padding 0.2.1", + "block-padding", "cipher", ] -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", -] - [[package]] name = "block-padding" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", +] + [[package]] name = "bumpalo" version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "byteorder" version = "1.4.3" @@ -446,22 +414,56 @@ dependencies = [ "winapi", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "cipher" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" dependencies = [ - "generic-array 0.14.6", + "generic-array", ] [[package]] -name = "cloudabi" -version = "0.0.3" +name = "clap" +version = "4.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +checksum = "335867764ed2de42325fafe6d18b8af74ba97ee0c590fa016f157535b42ab04b" dependencies = [ + "atty", "bitflags", + "clap_derive", + "clap_lex", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1b0f6422af32d5da0c58e2703320f379216ee70198241c84173a8c5ac28f3" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", ] [[package]] @@ -510,12 +512,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "convert_case" version = "0.4.0" @@ -631,13 +627,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.6", + "generic-array", "typenum", ] @@ -726,20 +728,11 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e" -[[package]] -name = "debug_unreachable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a032eac705ca39214d169f83e3d3da290af06d8d1d344d1baad2fd002dca4b3" -dependencies = [ - "unreachable", -] - [[package]] name = "deno_core" -version = "0.155.0" +version = "0.156.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95d779187cc23328025662dd9cd5e7871867ca047486bff86f7be0ab46fa468c" +checksum = "9157c30bd82f4e4feca1b19786554dace4ebdaf0f61e8ee25bce8c966c7d97c7" dependencies = [ "anyhow", "bytes", @@ -761,9 +754,9 @@ dependencies = [ [[package]] name = "deno_ops" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c198aa2122ca174a7e13398125d5a29291c7c38bad98c9ea32cd8a86b15302" +checksum = "2b29fb7131fc319533e89b3fd3100409927295c15d05bfd212cbbbbc18182dc0" dependencies = [ "once_cell", "proc-macro-crate", @@ -794,7 +787,7 @@ checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d" dependencies = [ "byteorder", "cipher", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] @@ -803,22 +796,13 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.6", + "generic-array", ] [[package]] @@ -832,17 +816,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" -dependencies = [ - "libc", - "redox_users 0.3.5", - "winapi", -] - [[package]] name = "dirs" version = "4.0.0" @@ -852,6 +825,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.7" @@ -859,20 +842,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", - "redox_users 0.4.3", + "redox_users", "winapi", ] [[package]] -name = "docopt" -version = "1.1.1" +name = "dirs-sys-next" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ - "lazy_static", - "regex", - "serde", - "strsim", + "libc", + "redox_users", + "winapi", ] [[package]] @@ -887,6 +869,12 @@ version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03d8c417d7a8cb362e0c37e5d815f5eb7c37f79ff93707329d5a194e42e54ca0" +[[package]] +name = "dyn-clone" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f94fa09c2aeea5b8839e414b7b841bf429fd25b9c522116ac97ee87856d88b2" + [[package]] name = "either" version = "1.8.0" @@ -896,27 +884,11 @@ dependencies = [ "serde", ] -[[package]] -name = "email-encoding" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34dd14c63662e0206599796cd5e1ad0268ab2b9d19b868d6050d688eba2bbf98" -dependencies = [ - "base64", - "memchr", -] - -[[package]] -name = "email_address" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b32a7a2580c4473f10f66b512c34bdd7d33c5e3473227ca833abdb5afe4809" - [[package]] name = "ena" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36" +checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3" dependencies = [ "log", ] @@ -948,12 +920,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "fastrand" version = "1.8.0" @@ -971,15 +937,15 @@ checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "windows-sys 0.42.0", ] [[package]] name = "fixedbitset" -version = "0.1.9" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" @@ -1043,12 +1009,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futures" version = "0.3.25" @@ -1108,6 +1068,21 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.25" @@ -1149,15 +1124,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.6" @@ -1169,14 +1135,12 @@ dependencies = [ ] [[package]] -name = "getrandom" -version = "0.1.16" +name = "getopts" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", + "unicode-width", ] [[package]] @@ -1329,6 +1293,15 @@ dependencies = [ "digest 0.10.5", ] +[[package]] +name = "home" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" +dependencies = [ + "winapi", +] + [[package]] name = "http" version = "0.2.8" @@ -1426,9 +1399,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.51" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1454,17 +1427,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.3.0" @@ -1487,8 +1449,9 @@ version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ - "autocfg 1.1.0", + "autocfg", "hashbrown", + "serde", ] [[package]] @@ -1519,15 +1482,6 @@ dependencies = [ "syn", ] -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.10.5" @@ -1552,45 +1506,37 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json-pointer" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe841b94e719a482213cee19dd04927cf412f26d8dc84c5a446c081e49c2997" -dependencies = [ - "serde_json", -] - [[package]] name = "lalrpop" -version = "0.17.2" +version = "0.19.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64dc3698e75d452867d9bd86f4a723f452ce9d01fe1d55990b79f0c790aa67db" +checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823" dependencies = [ "ascii-canvas", "atty", "bit-set", "diff", - "docopt", "ena", - "itertools 0.8.2", + "itertools", "lalrpop-util", "petgraph", + "pico-args", "regex", "regex-syntax", - "serde", - "serde_derive", - "sha2 0.8.2", - "string_cache 0.7.5", + "string_cache", "term", + "tiny-keccak", "unicode-xid", ] [[package]] name = "lalrpop-util" -version = "0.17.2" +version = "0.19.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c277d18683b36349ab5cd030158b54856fca6bb2d5dc5263b06288f486958b7c" +checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4" +dependencies = [ + "regex", +] [[package]] name = "lazy_static" @@ -1598,33 +1544,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lettre" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eabca5e0b4d0e98e7f2243fb5b7520b6af2b65d8f87bcc86f2c75185a6ff243" -dependencies = [ - "async-trait", - "base64", - "email-encoding", - "email_address", - "fastrand", - "futures-io", - "futures-util", - "httpdate", - "idna 0.2.3", - "mime", - "nom", - "once_cell", - "quoted_printable", - "rustls", - "rustls-pemfile", - "socket2", - "tokio", - "tokio-rustls", - "webpki-roots", -] - [[package]] name = "lexical" version = "6.1.1" @@ -1700,9 +1619,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.136" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55edcf6c0bb319052dea84732cf99db461780fd5e8d3eb46ab6ff312ab31f197" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "link-cplusplus" @@ -1719,7 +1638,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ - "autocfg 1.1.0", + "autocfg", "scopeguard", ] @@ -1732,6 +1651,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lz4_flex" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3" +dependencies = [ + "twox-hash", +] + [[package]] name = "magic-crypt" version = "3.1.12" @@ -1758,12 +1686,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matches" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" - [[package]] name = "matchit" version = "0.5.0" @@ -1778,7 +1700,7 @@ checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" dependencies = [ "block-buffer 0.9.0", "digest 0.9.0", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] @@ -1883,36 +1805,35 @@ dependencies = [ "winapi", ] -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.1.0", - "num-integer", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", "serde", ] +[[package]] +name = "num-complex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "num-integer" version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -1922,7 +1843,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -1946,15 +1867,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" - -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "opaque-debug" @@ -1962,6 +1877,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openapiv3" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b4689110fd71f196934fbdf1ad0f0a4b49ea41fbc6f19008c00dba735b544c" +dependencies = [ + "indexmap", + "serde", + "serde_json", +] + [[package]] name = "openssl" version = "0.10.42" @@ -2000,7 +1926,7 @@ version = "0.9.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cc", "libc", "pkg-config", @@ -2008,10 +1934,10 @@ dependencies = [ ] [[package]] -name = "ordermap" -version = "0.3.5" +name = "os_str_bytes" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" +checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" [[package]] name = "overload" @@ -2019,6 +1945,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2037,7 +1969,7 @@ checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", "windows-sys 0.42.0", ] @@ -2049,7 +1981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" dependencies = [ "base64ct", - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -2066,13 +1998,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] -name = "petgraph" -version = "0.4.13" +name = "pest" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" dependencies = [ "fixedbitset", - "ordermap", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", ] [[package]] @@ -2086,13 +2037,13 @@ dependencies = [ ] [[package]] -name = "phf_generator" -version = "0.7.24" +name = "phf_codegen" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" dependencies = [ - "phf_shared 0.7.24", - "rand 0.6.5", + "phf_generator 0.10.0", + "phf_shared 0.10.0", ] [[package]] @@ -2102,7 +2053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ "phf_shared 0.10.0", - "rand 0.8.5", + "rand", ] [[package]] @@ -2112,7 +2063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" dependencies = [ "phf_shared 0.11.1", - "rand 0.8.5", + "rand", ] [[package]] @@ -2128,22 +2079,13 @@ dependencies = [ "syn", ] -[[package]] -name = "phf_shared" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" -dependencies = [ - "siphasher 0.2.3", -] - [[package]] name = "phf_shared" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "siphasher 0.3.10", + "siphasher", ] [[package]] @@ -2152,9 +2094,15 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" dependencies = [ - "siphasher 0.3.10", + "siphasher", ] +[[package]] +name = "pico-args" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" + [[package]] name = "pin-project" version = "1.0.12" @@ -2189,9 +2137,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "pmutil" @@ -2227,6 +2175,30 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -2242,6 +2214,72 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "progenitor" +version = "0.2.1-dev" +source = "git+https://github.com/oxidecomputer/progenitor#f9708ef56c3a0b88dc88fc0a0fbf0d8885fdd3e8" +dependencies = [ + "anyhow", + "clap", + "openapiv3", + "progenitor-client", + "progenitor-impl", + "progenitor-macro", + "serde", + "serde_json", +] + +[[package]] +name = "progenitor-client" +version = "0.2.1-dev" +source = "git+https://github.com/oxidecomputer/progenitor#f9708ef56c3a0b88dc88fc0a0fbf0d8885fdd3e8" +dependencies = [ + "bytes", + "futures-core", + "percent-encoding", + "reqwest", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "progenitor-impl" +version = "0.2.1-dev" +source = "git+https://github.com/oxidecomputer/progenitor#f9708ef56c3a0b88dc88fc0a0fbf0d8885fdd3e8" +dependencies = [ + "getopts", + "heck", + "indexmap", + "openapiv3", + "proc-macro2", + "quote", + "regex", + "rustfmt-wrapper", + "schemars", + "serde", + "serde_json", + "syn", + "thiserror", + "typify", + "unicode-ident", +] + +[[package]] +name = "progenitor-macro" +version = "0.2.1-dev" +source = "git+https://github.com/oxidecomputer/progenitor#f9708ef56c3a0b88dc88fc0a0fbf0d8885fdd3e8" +dependencies = [ + "openapiv3", + "proc-macro2", + "progenitor-impl", + "quote", + "serde", + "serde_json", + "serde_tokenstream", + "syn", +] + [[package]] name = "prometheus" version = "0.13.3" @@ -2273,7 +2311,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn", @@ -2298,31 +2336,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "quoted_printable" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fee2dce59f7a43418e3382c766554c614e06a552d53a8f07ef499ea4b332c0f" - -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.8.5" @@ -2330,18 +2343,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -2351,110 +2354,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.8", + "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.16" @@ -2464,25 +2375,14 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" -dependencies = [ - "getrandom 0.1.16", - "redox_syscall 0.1.57", - "rust-argon2", -] - [[package]] name = "redox_users" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.8", - "redox_syscall 0.2.16", + "getrandom", + "redox_syscall", "thiserror", ] @@ -2512,6 +2412,15 @@ version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +[[package]] +name = "regress" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a92ff21fe8026ce3f2627faaf43606f0b67b014dbc9ccf027181a804f75d92e" +dependencies = [ + "memchr", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -2550,6 +2459,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -2567,7 +2477,7 @@ dependencies = [ "async-lock", "async-timer", "log", - "rand 0.8.5", + "rand", ] [[package]] @@ -2585,18 +2495,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rust-argon2" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" -dependencies = [ - "base64", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", -] - [[package]] name = "rust-embed" version = "6.4.2" @@ -2655,6 +2553,19 @@ dependencies = [ "semver 1.0.14", ] +[[package]] +name = "rustfmt-wrapper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed729e3bee08ec2befd593c27e90ca9fdd25efdc83c94c3b82eaef16e4f7406e" +dependencies = [ + "serde", + "tempfile", + "thiserror", + "toml", + "toolchain_find", +] + [[package]] name = "rustls" version = "0.20.7" @@ -2676,22 +2587,62 @@ dependencies = [ "base64", ] +[[package]] +name = "rustpython-ast" +version = "0.1.0" +source = "git+https://github.com/RustPython/RustPython#835771b7ea2903bf641f00db1cc3c88b74e2a08f" +dependencies = [ + "num-bigint", + "rustpython-compiler-core", +] + +[[package]] +name = "rustpython-compiler-core" +version = "0.1.2" +source = "git+https://github.com/RustPython/RustPython#835771b7ea2903bf641f00db1cc3c88b74e2a08f" +dependencies = [ + "bincode", + "bitflags", + "bstr", + "itertools", + "lz4_flex", + "num-bigint", + "num-complex", + "serde", + "static_assertions", + "thiserror", +] + [[package]] name = "rustpython-parser" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85b1038ecd8791bdae455ae784b48f522ebeca1b3323d0af2b251c5c8ff1c68" +source = "git+https://github.com/RustPython/RustPython#835771b7ea2903bf641f00db1cc3c88b74e2a08f" dependencies = [ + "ahash", + "anyhow", + "itertools", "lalrpop", "lalrpop-util", "log", - "num-bigint 0.2.6", + "num-bigint", "num-traits", + "phf 0.10.1", + "phf_codegen", + "rustpython-ast", + "rustpython-compiler-core", + "thiserror", + "tiny-keccak", "unic-emoji-char", "unic-ucd-ident", "unicode_names2", ] +[[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + [[package]] name = "ryu" version = "1.0.11" @@ -2717,6 +2668,32 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "schemars" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a5fb6c61f29e723026dc8e923d94c694313212abbecbbe5f55a7748eec5b307" +dependencies = [ + "chrono", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "schemars_derive" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f188d036977451159430f3b8dc82ec76364a42b7e289c2b18a9a18f4470058e9" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -2774,7 +2751,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -2789,6 +2775,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "serde" version = "1.0.147" @@ -2829,6 +2824,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_json" version = "1.0.87" @@ -2841,6 +2847,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_tokenstream" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6deb15c3a535e81438110111d90168d91721652f502abb147f31cde129f683d" +dependencies = [ + "proc-macro2", + "serde", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2855,9 +2872,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.66.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2b0ef50e4ea3b5a74bcddb063c132176b2433d873e126db983667ac951f98e" +checksum = "225ac8039b46a4094ce9a30985820af6a602d27bdc7c0ce2728c10ad67aa03dd" dependencies = [ "bytes", "derive_more", @@ -2878,18 +2895,6 @@ dependencies = [ "digest 0.10.5", ] -[[package]] -name = "sha2" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - [[package]] name = "sha2" version = "0.9.9" @@ -2900,7 +2905,7 @@ dependencies = [ "cfg-if", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] @@ -2932,12 +2937,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" - [[package]] name = "siphasher" version = "0.3.10" @@ -2950,7 +2949,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -3007,7 +3006,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87e292b4291f154971a43c3774364e2cbcaec599d3f5bf6fa9d122885dbc38a" dependencies = [ - "itertools 0.10.5", + "itertools", "nom", "unicode_categories", ] @@ -3037,7 +3036,7 @@ dependencies = [ "chrono", "crc", "crossbeam-queue", - "dirs 4.0.0", + "dirs", "dotenvy", "either", "event-listener", @@ -3058,7 +3057,7 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rand 0.8.5", + "rand", "rustls", "rustls-pemfile", "serde", @@ -3122,21 +3121,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "string_cache" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67" -dependencies = [ - "lazy_static", - "new_debug_unreachable", - "phf_shared 0.7.24", - "precomputed-hash", - "serde", - "string_cache_codegen 0.4.4", - "string_cache_shared", -] - [[package]] name = "string_cache" version = "0.8.4" @@ -3151,19 +3135,6 @@ dependencies = [ "serde", ] -[[package]] -name = "string_cache_codegen" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" -dependencies = [ - "phf_generator 0.7.24", - "phf_shared 0.7.24", - "proc-macro2", - "quote", - "string_cache_shared", -] - [[package]] name = "string_cache_codegen" version = "0.5.2" @@ -3176,12 +3147,6 @@ dependencies = [ "quote", ] -[[package]] -name = "string_cache_shared" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" - [[package]] name = "string_enum" version = "0.3.2" @@ -3219,37 +3184,37 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "swc_atoms" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b878052680dcec3421ab50384279443dbf93651b05da38e5133e0894a18096" +checksum = "79642938ff437f2217718abf30a3450b014f600847c8f4bd60fa44f88a5210ea" dependencies = [ "once_cell", "rustc-hash", "serde", - "string_cache 0.8.4", - "string_cache_codegen 0.5.2", + "string_cache", + "string_cache_codegen", "triomphe", ] [[package]] name = "swc_common" -version = "0.29.10" +version = "0.29.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd844dfbd9969a9ef8430e954661de43edde353d65e987f935a328619698883" +checksum = "953e1f014688eadbbd3e9131a525e8922c552540bb02b0bb6d9fdcb1375bccc4" dependencies = [ "ahash", "ast_node", "better_scoped_tls", "cfg-if", - "debug_unreachable", "either", "from_variant", - "num-bigint 0.4.3", + "new_debug_unreachable", + "num-bigint", "once_cell", "rustc-hash", "serde", - "siphasher 0.3.10", - "string_cache 0.8.4", + "siphasher", + "string_cache", "swc_atoms", "swc_eq_ignore_macros", "swc_visit", @@ -3260,13 +3225,13 @@ dependencies = [ [[package]] name = "swc_ecma_ast" -version = "0.94.14" +version = "0.94.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c3303de79adce1137e6514e5939686173e7d26c71d91c3067056caa45183547" +checksum = "bc39246540303a9058283e6ef691a276c34afd8331e6873fb3e6fb7803eb77eb" dependencies = [ "bitflags", "is-macro", - "num-bigint 0.4.3", + "num-bigint", "scoped-tls", "serde", "string_enum", @@ -3277,14 +3242,14 @@ dependencies = [ [[package]] name = "swc_ecma_parser" -version = "0.122.20" +version = "0.122.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e09f503e4e509cde4deb310950b645f7e3189328fa8a912bcb5497ea632bac" +checksum = "6472a516d3e7f30277650353f5bdce431c273e43ba25ee918e8d0a5a0a9868eb" dependencies = [ "either", "enum_kind", "lexical", - "num-bigint 0.4.3", + "num-bigint", "serde", "smallvec", "swc_atoms", @@ -3368,19 +3333,19 @@ dependencies = [ "cfg-if", "fastrand", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "remove_dir_all", "winapi", ] [[package]] name = "term" -version = "0.5.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ - "byteorder", - "dirs 1.0.5", + "dirs-next", + "rustversion", "winapi", ] @@ -3473,6 +3438,27 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -3494,7 +3480,7 @@ version = "1.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" dependencies = [ - "autocfg 1.1.0", + "autocfg", "bytes", "libc", "memchr", @@ -3582,7 +3568,7 @@ dependencies = [ "filetime", "futures-core", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "tokio", "tokio-stream", "xattr", @@ -3643,6 +3629,19 @@ dependencies = [ "tracing-futures", ] +[[package]] +name = "toolchain_find" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413" +dependencies = [ + "home", + "lazy_static", + "regex", + "semver 0.11.0", + "walkdir", +] + [[package]] name = "tower" version = "0.4.13" @@ -3654,7 +3653,7 @@ dependencies = [ "indexmap", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand", "slab", "tokio", "tokio-util", @@ -3814,6 +3813,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typed-arena" version = "2.0.1" @@ -3826,13 +3835,64 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "typify" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8486352f3c946e69f983558cfc09b295250b01e01b381ec67a05a812d01d63" +dependencies = [ + "typify-impl", + "typify-macro", +] + +[[package]] +name = "typify-impl" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7624d0b911df6e2bbf34a236f76281f93b294cdde1d4df1dbdb748e5a7fefa5" +dependencies = [ + "heck", + "log", + "proc-macro2", + "quote", + "regress", + "rustfmt-wrapper", + "schemars", + "serde_json", + "syn", + "thiserror", + "unicode-ident", +] + +[[package]] +name = "typify-macro" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c42802aa033cee7650a4e1509ba7d5848a56f84be7c4b31e4385ee12445e942" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "serde", + "serde_json", + "serde_tokenstream", + "syn", + "typify-impl", +] + +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + [[package]] name = "ulid" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13a3aaa69b04e5b66cc27309710a569ea23593612387d67daaf102e73aa974fd" dependencies = [ - "rand 0.8.5", + "rand", "uuid", ] @@ -3944,9 +4004,9 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.1.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unicode_categories" @@ -3956,18 +4016,9 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "unicode_names2" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d6678d7916394abad0d4b19df4d3802e1fd84abd7d701f39b75ee71b9e8cf1" - -[[package]] -name = "unreachable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" -dependencies = [ - "void", -] +checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301" [[package]] name = "untrusted" @@ -3982,32 +4033,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna 0.3.0", + "idna", "percent-encoding", "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" - [[package]] name = "uuid" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83" dependencies = [ - "getrandom 0.2.8", + "getrandom", "serde", ] [[package]] name = "v8" -version = "0.53.1" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e952e936bcb610c9f22997f50dc7f65887afe76e1fedd37daf532a20211335ca" +checksum = "3b63103bd7caa4c3571e8baafe58f3e04818df70505304ed814737e655d1d8d6" dependencies = [ "bitflags", "fslock", @@ -4035,10 +4080,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "void" -version = "1.0.2" +name = "waker-fn" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" @@ -4061,12 +4106,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -4230,53 +4269,57 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" version = "1.41.0" +dependencies = [ + "anyhow", + "axum", + "chrono", + "dotenv", + "futures", + "rand", + "reqwest", + "serde_json", + "sqlx", + "tokio", + "tokio-metrics", + "tracing", + "windmill-api", + "windmill-api-client", + "windmill-common", + "windmill-queue", + "windmill-worker", +] + +[[package]] +name = "windmill-api" +version = "1.41.0" dependencies = [ "anyhow", "argon2", "async-oauth2", - "async-recursion", "axum", "base64", "chrono", - "console-subscriber", "cron", - "deno_core", - "dotenv", "futures", "git-version", - "headers", "hex", "hmac", "hyper", - "itertools 0.10.5", - "json-pointer", - "lazy_static", - "lettre", + "itertools", "magic-crypt", "mime_guess", - "phf", - "prometheus", - "rand 0.8.5", - "rand_core 0.6.4", - "regex", + "rand", "reqwest", "retainer", "rust-embed", - "rustpython-parser", "serde", "serde_json", "serde_urlencoded", - "sha2 0.10.6", "sql-builder", "sqlx", - "swc_common", - "swc_ecma_ast", - "swc_ecma_parser", "tempfile", - "thiserror", "time 0.3.16", "tokio", - "tokio-metrics", "tokio-tar", "tokio-util", "tower", @@ -4284,13 +4327,171 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", - "ulid", - "unicode-general-category", - "url", - "urlencoding", + "windmill-audit", + "windmill-common", + "windmill-parser", + "windmill-parser-go", + "windmill-parser-py", + "windmill-parser-ts", + "windmill-queue", +] + +[[package]] +name = "windmill-api-client" +version = "1.41.0" +dependencies = [ + "base64", + "chrono", + "progenitor", + "progenitor-client", + "rand", + "reqwest", + "serde", + "serde_json", "uuid", ] +[[package]] +name = "windmill-audit" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tracing", + "windmill-common", +] + +[[package]] +name = "windmill-common" +version = "1.41.0" +dependencies = [ + "anyhow", + "axum", + "chrono", + "console-subscriber", + "hex", + "hmac", + "hyper", + "prometheus", + "rand", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.6", + "sqlx", + "thiserror", + "tiny_http", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "windmill-parser" +version = "1.41.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "windmill-parser-go" +version = "1.41.0" +dependencies = [ + "anyhow", + "itertools", + "phf 0.11.1", + "unicode-general-category", + "windmill-common", + "windmill-parser", +] + +[[package]] +name = "windmill-parser-py" +version = "1.41.0" +dependencies = [ + "anyhow", + "itertools", + "phf 0.11.1", + "regex", + "rustpython-parser", + "serde_json", + "windmill-common", + "windmill-parser", +] + +[[package]] +name = "windmill-parser-ts" +version = "1.41.0" +dependencies = [ + "anyhow", + "deno_core", + "serde_json", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "windmill-common", + "windmill-parser", +] + +[[package]] +name = "windmill-queue" +version = "1.41.0" +dependencies = [ + "anyhow", + "chrono", + "cron", + "hex", + "hmac", + "lazy_static", + "prometheus", + "reqwest", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tracing", + "ulid", + "uuid", + "windmill-audit", + "windmill-common", +] + +[[package]] +name = "windmill-worker" +version = "1.41.0" +dependencies = [ + "anyhow", + "async-recursion", + "chrono", + "deno_core", + "dotenv", + "futures", + "itertools", + "lazy_static", + "prometheus", + "rand", + "regex", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-api-client", + "windmill-audit", + "windmill-common", + "windmill-parser", + "windmill-parser-go", + "windmill-parser-py", + "windmill-parser-ts", + "windmill-queue", +] + [[package]] name = "windows-sys" version = "0.36.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 3fc1115b64..3c6b4508e0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,13 +1,65 @@ [package] name = "windmill" version = "1.41.0" +authors.workspace = true +edition.workspace = true + +[workspace] +members = [ + "./windmill-api", + "./windmill-queue", + "./windmill-worker", + "./windmill-common", + "./windmill-audit", + "./windmill-api-client", + "./parsers/windmill-parser", + "./parsers/windmill-parser-ts", + "./parsers/windmill-parser-go", + "./parsers/windmill-parser-py", +] + +[workspace.package] +version = "1.41.0" authors = ["Ruben Fiszel "] edition = "2021" -[build-dependencies] -deno_core = "^0" +[[bin]] +name = "windmill" +path = "./src/main.rs" [dependencies] +anyhow.workspace = true +tokio.workspace = true +dotenv.workspace = true +windmill-common = { workspace = true, features = ["tracing_init"] } +windmill-api.workspace = true +windmill-api-client.workspace = true +windmill-worker.workspace = true +futures.workspace = true +tracing.workspace = true +sqlx.workspace = true +tokio-metrics.workspace = true +rand.workspace = true +chrono.workspace = true + +[dev-dependencies] +serde_json.workspace = true +reqwest.workspace = true +windmill-queue.workspace = true +axum.workspace = true + +[workspace.dependencies] +windmill-api = { path = "./windmill-api" } +windmill-api-client = { path = "./windmill-api-client" } +windmill-queue = { path = "./windmill-queue" } +windmill-worker = { path = "./windmill-worker" } +windmill-common = { path = "./windmill-common" } +windmill-audit = { path = "./windmill-audit" } +windmill-parser = { path = "./parsers/windmill-parser" } +windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } +windmill-parser-py = { path = "./parsers/windmill-parser-py" } +windmill-parser-go = { path = "./parsers/windmill-parser-go" } +tiny_http = "0.12.0" axum = { version = "^0", features = ["headers"] } headers = "^0" hyper = { version = "^0", features = ["full"] } @@ -20,31 +72,36 @@ serde_json = { version = "^1", features = ["preserve_order"] } uuid = { version = "^1", features = ["serde", "v4"] } thiserror = "^1" anyhow = "^1" -chrono = { version = "^0", features = ["serde"]} +chrono = { version = "^0", features = ["serde"] } tracing = "^0" -tracing-subscriber = { version = "^0", features = ["env-filter", "json"]} +tracing-subscriber = { version = "^0", features = ["env-filter", "json"] } console-subscriber = "^0" prometheus = { version = "^0", default-features = false } phf = { version = "0.11", features = ["macros"] } - rust-embed = "^6" mime_guess = "^2" hex = "^0" sql-builder = "^3" argon2 = "^0" retainer = "^0" -rand = "^0" +rand = "0.8.5" rand_core = { version = "^0", features = ["std"] } magic-crypt = "^3" git-version = "^0" -rustpython-parser = "^0" +rustpython-parser = { git = "https://github.com/RustPython/RustPython" } cron = "^0" -lettre = { version = "^0", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"], default-features = false} +lettre = { version = "^0", features = [ + "rustls-tls", + "tokio1", + "tokio1-rustls-tls", + "builder", + "smtp-transport", +], default-features = false } urlencoding = "^2" url = "^2" async-oauth2 = "^0" reqwest = { version = "^0", features = ["json"] } -time = "^0" +time = "0.3.16" serde_urlencoded = "^0" tokio-tar = "^0" tempfile = "^3" @@ -57,14 +114,23 @@ async-recursion = "^1" swc_common = "^0" swc_ecma_parser = "^0" swc_ecma_ast = "^0" -base64 = "^0" +base64 = "^0" unicode-general-category = "^0" -hmac = "^0" -sha2 = "^0" - -sqlx = { version = "^0", features = ["offline", "macros", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]} +hmac = "0.12.1" +sha2 = "0.10.6" +sqlx = { version = "^0", features = [ + "offline", + "macros", + "migrate", + "uuid", + "json", + "chrono", + "postgres", + "runtime-tokio-rustls", +] } dotenv = "^0" ulid = { version = "^1", features = ["uuid"] } futures = "^0" tokio-metrics = "0.1.0" lazy_static = "1.4.0" +serde_derive = "1.0.147" diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000000..2d3d35b1a1 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,15 @@ +# Windmill Backend + +This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary. + +## Components + +| name | description | +| --------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| [windmill-api](./windmill-api/) | The API server, exposing functionality to other components and the frontend | +| [windmill-api-client](./windmill-api-client/) | An autogenerated Rust API client, used by other components to talk to the API | +| [windmill-audit](./windmill-audit/) | Contains audit functionality, allowing different components to record important actions | +| [windmill-common](./windmill-common/) | Common code shared by all crates | +| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers | +| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. | +| [parsers](./parsers/) | Contains code to parse signatures in different langauges. | diff --git a/backend/parsers/windmill-parser-go/Cargo.toml b/backend/parsers/windmill-parser-go/Cargo.toml new file mode 100644 index 0000000000..729b08f1ee --- /dev/null +++ b/backend/parsers/windmill-parser-go/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "windmill-parser-go" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_go" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +windmill-common.workspace = true +phf.workspace = true +unicode-general-category.workspace = true +itertools.workspace = true +anyhow.workspace = true diff --git a/backend/src/parser_go.rs b/backend/parsers/windmill-parser-go/src/lib.rs similarity index 98% rename from backend/src/parser_go.rs rename to backend/parsers/windmill-parser-go/src/lib.rs index 6ae38b420a..40c762ff7d 100644 --- a/backend/src/parser_go.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -1,16 +1,19 @@ #![allow(non_snake_case)] // TODO: switch to parse_* function naming +mod parser_go_ast; +mod parser_go_scanner; +mod parser_go_token; + use itertools::Itertools; -use crate::error::to_anyhow; -use crate::parser::{Arg, MainArgSignature, ObjectProperty, Typ}; -use crate::parser_go_ast::{self, FieldList, Ident, StructType}; -use crate::parser_go_ast::{Decl, Expr}; -use crate::parser_go_scanner; -use crate::parser_go_token::{Position, Token}; +use parser_go_ast::{Decl, Expr}; +use parser_go_ast::{FieldList, Ident, StructType}; +use parser_go_token::{Position, Token}; use std::fmt; +use windmill_common::error::to_anyhow; +use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; -pub fn parse_go_sig(code: &str) -> crate::error::Result { +pub fn parse_go_sig(code: &str) -> windmill_common::error::Result { let filtered_code = filter_non_main(code); let file = parse_file("main.go", &filtered_code).map_err(to_anyhow)?; if let Some(Decl::FuncDecl(func)) = file.decls.first() { @@ -26,7 +29,7 @@ pub fn parse_go_sig(code: &str) -> crate::error::Result { .collect_vec(); Ok(MainArgSignature { star_args: false, star_kwargs: false, args }) } else { - Err(crate::error::Error::BadRequest( + Err(windmill_common::error::Error::BadRequest( "no main function found".to_string(), )) } @@ -107,7 +110,7 @@ pub fn otyp_to_string(otyp: Option) -> String { #[cfg(test)] mod tests { - use crate::parser::{Arg, MainArgSignature, ObjectProperty, Typ}; + use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; use super::*; diff --git a/backend/src/parser_go_ast.rs b/backend/parsers/windmill-parser-go/src/parser_go_ast.rs similarity index 100% rename from backend/src/parser_go_ast.rs rename to backend/parsers/windmill-parser-go/src/parser_go_ast.rs diff --git a/backend/src/parser_go_scanner.rs b/backend/parsers/windmill-parser-go/src/parser_go_scanner.rs similarity index 100% rename from backend/src/parser_go_scanner.rs rename to backend/parsers/windmill-parser-go/src/parser_go_scanner.rs diff --git a/backend/src/parser_go_token.rs b/backend/parsers/windmill-parser-go/src/parser_go_token.rs similarity index 100% rename from backend/src/parser_go_token.rs rename to backend/parsers/windmill-parser-go/src/parser_go_token.rs diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml new file mode 100644 index 0000000000..996fc4a17c --- /dev/null +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "windmill-parser-py" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_py" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +windmill-common.workspace = true +rustpython-parser.workspace = true +phf.workspace = true +itertools.workspace = true +regex.workspace = true +serde_json.workspace = true +anyhow.workspace = true diff --git a/backend/src/parser_py.rs b/backend/parsers/windmill-parser-py/src/lib.rs similarity index 78% rename from backend/src/parser_py.rs rename to backend/parsers/windmill-parser-py/src/lib.rs index 0dc270462b..8a980b6735 100644 --- a/backend/src/parser_py.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -11,21 +11,19 @@ use std::collections::HashMap; use itertools::Itertools; use phf::phf_map; use regex::Regex; -use serde_json::json; -use crate::{ - error, - parser::{Arg, MainArgSignature, Typ}, -}; +use serde_json::json; +use windmill_common::error; +use windmill_parser::{Arg, MainArgSignature, Typ}; use rustpython_parser::{ - ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs}, + ast::{Constant, ExprKind, Located, StmtKind}, parser, }; -fn filter_non_main(code: &str) -> String { - const DEF_MAIN: &str = "def main("; +const DEF_MAIN: &str = "def main("; +fn filter_non_main(code: &str) -> String { let mut filtered_code = String::new(); let mut code_iter = code.split("\n"); let mut remaining: String = String::new(); @@ -66,35 +64,27 @@ pub fn parse_python_signature(code: &str) -> error::Result { "No main function found".to_string(), )); } - let ast = parser::parse_program(&filtered_code) - .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))? - .statements; + let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| { + error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) + })?; let param = ast.into_iter().find_map(|x| match x { - Located { - location: _, - node: - StatementType::FunctionDef { - is_async: _, - name, - args, - body: _, - decorator_list: _, - returns: _, - }, - } if &name == "main" => Some(*args), + Located { node: StmtKind::FunctionDef { name, args, .. }, .. } if &name == "main" => { + Some(*args) + } _ => None, }); if let Some(params) = param { //println!("{:?}", params); let def_arg_start = params.args.len() - params.defaults.len(); Ok(MainArgSignature { - star_args: params.vararg != Varargs::None, - star_kwargs: params.vararg != Varargs::None, + star_args: params.vararg.is_some(), + star_kwargs: params.vararg.is_some(), args: params .args .into_iter() .enumerate() .map(|(i, x)| { + let x = x.node; let default = if i >= def_arg_start { to_value(¶ms.defaults[i - def_arg_start].node) } else { @@ -104,20 +94,18 @@ pub fn parse_python_signature(code: &str) -> error::Result { otyp: None, name: x.arg, typ: x.annotation.map_or(Typ::Unknown, |e| match *e { - Located { location: _, node: ExpressionType::Identifier { name } } => { - match name.as_ref() { - "str" => Typ::Str(None), - "float" => Typ::Float, - "int" => Typ::Int, - "bool" => Typ::Bool, - "dict" => Typ::Object(vec![]), - "list" => Typ::List(Box::new(Typ::Str(None))), - "bytes" => Typ::Bytes, - "datetime" => Typ::Datetime, - "datetime.datetime" => Typ::Datetime, - _ => Typ::Unknown, - } - } + Located { node: ExprKind::Name { id, .. }, .. } => match id.as_ref() { + "str" => Typ::Str(None), + "float" => Typ::Float, + "int" => Typ::Int, + "bool" => Typ::Bool, + "dict" => Typ::Object(vec![]), + "list" => Typ::List(Box::new(Typ::Str(None))), + "bytes" => Typ::Bytes, + "datetime" => Typ::Datetime, + "datetime.datetime" => Typ::Datetime, + _ => Typ::Unknown, + }, _ => Typ::Unknown, }), has_default: default.is_some(), @@ -133,24 +121,15 @@ pub fn parse_python_signature(code: &str) -> error::Result { } } -fn to_value(et: &ExpressionType) -> Option { +fn to_value(et: &ExprKind) -> Option { match et { - ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)), - ExpressionType::Number { value } => match value { - Number::Integer { value } => Some(json!(value.to_string().parse::().unwrap())), - Number::Float { value } => Some(json!(value)), - _ => None, - }, - ExpressionType::True => Some(json!(true)), - ExpressionType::False => Some(json!(false)), - - ExpressionType::Dict { elements } => { - let v = elements + ExprKind::Constant { value, .. } => Some(constant_to_value(value)), + ExprKind::Dict { keys, values } => { + let v = keys .into_iter() + .zip(values) .map(|(k, v)| { - let key = k - .as_ref() - .and_then(|x| to_value(&x.node)) + let key = to_value(&k.node) .and_then(|x| match x { serde_json::Value::String(s) => Some(s), _ => None, @@ -161,23 +140,32 @@ fn to_value(et: &ExpressionType) -> Option { .collect::>(); Some(json!(v)) } - ExpressionType::List { elements } => { - let v = elements + ExprKind::List { elts, .. } => { + let v = elts .into_iter() .map(|x| to_value(&x.node)) .collect::>(); Some(json!(v)) } - ExpressionType::None => Some(json!(null)), - - ExpressionType::Call { function: _, args: _, keywords: _ } => { - Some(json!("")) - } - + ExprKind::Call { .. } => Some(json!("")), _ => None, } } +fn constant_to_value(c: &Constant) -> serde_json::Value { + match c { + Constant::None => json!(null), + Constant::Bool(b) => json!(b), + Constant::Str(s) => json!(s), + Constant::Bytes(b) => json!(b), + Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")), + Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::>()), + Constant::Float(f) => json!(f), + Constant::Complex { real, imag } => json!([real, imag]), + Constant::Ellipsis => json!("..."), + } +} + static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! { "psycopg2" => "psycopg2-binary" }; @@ -206,25 +194,22 @@ pub fn parse_python_imports(code: &str) -> error::Result> { .collect(); Ok(lines) } else { - let code = &&code; - let ast = parser::parse_program(code) - .map_err(|e| { - error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) - })? - .statements; - + let code = code.split(DEF_MAIN).next().unwrap_or(""); + let ast = parser::parse_program(code, "main.py").map_err(|e| { + error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) + })?; let imports = ast .into_iter() .filter_map(|x| match x { - Located { location: _, node } => match node { - StatementType::Import { names } => Some( + Located { node, .. } => match node { + StmtKind::Import { names } => Some( names .into_iter() - .map(|x| x.symbol.split('.').next().unwrap_or("").to_string()) + .map(|x| x.node.name.split('.').next().unwrap_or("").to_string()) .map(replace_import) .collect::>(), ), - StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => { + StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => { let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-"); Some(vec![replace_import(imprt)]) @@ -244,6 +229,8 @@ pub fn parse_python_imports(code: &str) -> error::Result> { #[cfg(test)] mod tests { + use serde_json::json; + use super::*; #[test] @@ -252,7 +239,7 @@ mod tests { import os -def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)): +def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1), f = \"wewe\", g = 21, h = [1,2], i = True): print(f\"Hello World and a warm welcome especially to {name}\") print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\")) @@ -286,7 +273,35 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt typ: Typ::Bytes, default: Some(json!("")), has_default: true - } + }, + Arg { + otyp: None, + name: "f".to_string(), + typ: Typ::Unknown, + default: Some(json!("wewe")), + has_default: true + }, + Arg { + otyp: None, + name: "g".to_string(), + typ: Typ::Unknown, + default: Some(json!(21)), + has_default: true + }, + Arg { + otyp: None, + name: "h".to_string(), + typ: Typ::Unknown, + default: Some(json!([1, 2])), + has_default: true + }, + Arg { + otyp: None, + name: "i".to_string(), + typ: Typ::Unknown, + default: Some(json!(true)), + has_default: true + }, ] } ); diff --git a/backend/parsers/windmill-parser-ts/Cargo.toml b/backend/parsers/windmill-parser-ts/Cargo.toml new file mode 100644 index 0000000000..d7042b9b9c --- /dev/null +++ b/backend/parsers/windmill-parser-ts/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "windmill-parser-ts" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_ts" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +windmill-common.workspace = true +deno_core.workspace = true +swc_common.workspace = true +swc_ecma_parser.workspace = true +swc_ecma_ast.workspace = true +serde_json.workspace = true +anyhow.workspace = true diff --git a/backend/src/parser_ts.rs b/backend/parsers/windmill-parser-ts/src/lib.rs similarity index 95% rename from backend/src/parser_ts.rs rename to backend/parsers/windmill-parser-ts/src/lib.rs index 67522d4010..1da385ac1c 100644 --- a/backend/src/parser_ts.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -5,12 +5,9 @@ * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ - -use crate::{ - error, - js_eval::eval_sync, - parser::{Arg, MainArgSignature, ObjectProperty, Typ}, -}; +use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions}; +use windmill_common::error; +use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; use serde_json::Value; use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned}; @@ -263,6 +260,25 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) { } } +pub fn eval_sync(code: &str) -> Result { + let mut context = JsRuntime::new(RuntimeOptions::default()); + let code = format!("let x = {}; x", code); + let res = context.execute_script("", &code); + match res { + Ok(global) => { + let scope = &mut context.handle_scope(); + let local = v8::Local::new(scope, global); + let deserialized_value = serde_v8::from_v8::(scope, local); + + match deserialized_value { + Ok(value) => Ok(value), + Err(err) => Err(format!("Cannot deserialize value: {:?}", err)), + } + } + Err(err) => Err(format!("Evaling error: {:?}", err)), + } +} + #[cfg(test)] mod tests { diff --git a/backend/parsers/windmill-parser/Cargo.toml b/backend/parsers/windmill-parser/Cargo.toml new file mode 100644 index 0000000000..a56721468a --- /dev/null +++ b/backend/parsers/windmill-parser/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "windmill-parser" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser" +path = "./src/lib.rs" + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/backend/src/parser.rs b/backend/parsers/windmill-parser/src/lib.rs similarity index 100% rename from backend/src/parser.rs rename to backend/parsers/windmill-parser/src/lib.rs diff --git a/backend/src/client.rs b/backend/src/client.rs deleted file mode 100644 index e8fb4ae0a2..0000000000 --- a/backend/src/client.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::{error::Error, variables::ListableVariable}; - -pub async fn get_variable( - workspace: &str, - path: &str, - token: &str, - base_url: &str, -) -> Result { - let client = reqwest::Client::new(); - let res = client - .get(format!("{base_url}/api/w/{workspace}/variables/get/{path}")) - .bearer_auth(token) - .send() - .await?; - if res.status().is_success() { - let value = res - .json::() - .await? - .value - .unwrap_or_else(|| "".to_string()); - Ok(value) - } else { - Err(Error::NotFound(format!("Variable not found at {path}")))? - } -} - -pub async fn get_resource( - workspace: &str, - path: &str, - token: &str, - base_url: &str, -) -> Result, anyhow::Error> { - let client = reqwest::Client::new(); - let res = client - .get(format!( - "{base_url}/api/w/{workspace}/resources/get_value/{path}" - )) - .bearer_auth(token) - .send() - .await?; - if res.status().is_success() { - let value = res.json::>().await?; - Ok(value) - } else { - Err(Error::NotFound(format!("Resource not found at {path}")))? - } -} diff --git a/backend/src/main.rs b/backend/src/main.rs index f77d8de6be..9a31d656c7 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,21 +8,22 @@ use std::net::SocketAddr; -use dotenv::dotenv; -use windmill::WorkerConfig; +use sqlx::{Pool, Postgres}; +use windmill_common::utils::rd_string; +use windmill_worker::WorkerConfig; #[tokio::main] async fn main() -> anyhow::Result<()> { - dotenv().ok(); + dotenv::dotenv().ok(); - windmill::initialize_tracing(); + windmill_common::tracing_init::initialize_tracing(); - let db = windmill::connect_db().await?; + let db = windmill_common::connect_db().await?; let num_workers = std::env::var("NUM_WORKERS") .ok() .and_then(|x| x.parse::().ok()) - .unwrap_or(windmill::DEFAULT_NUM_WORKERS as i32); + .unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32); let metrics_addr: Option = std::env::var("METRICS_ADDR") .ok() @@ -40,39 +41,40 @@ async fn main() -> anyhow::Result<()> { .unwrap_or(false); if server_mode { - windmill::migrate_db(&db).await?; + windmill_api::migrate_db(&db).await?; } let (tx, rx) = tokio::sync::broadcast::channel::<()>(3); - let shutdown_signal = windmill::shutdown_signal(tx); + let shutdown_signal = windmill_common::shutdown_signal(tx); let base_internal_url = std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string()); let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()); + let timeout = std::env::var("TIMEOUT") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(windmill_common::DEFAULT_TIMEOUT); + if server_mode || num_workers > 0 { let addr = SocketAddr::from(([0, 0, 0, 0], 8000)); - let timeout = std::env::var("TIMEOUT") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(windmill::DEFAULT_TIMEOUT); - - let base_url_2 = base_url.clone(); + let base_url2 = base_url.clone(); let server_f = async { if server_mode { - windmill::run_server(db.clone(), addr, base_url_2, rx.resubscribe()).await?; + windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?; } Ok(()) as anyhow::Result<()> }; + let base_url = base_url2.clone(); let workers_f = async { if num_workers > 0 { let sleep_queue = std::env::var("SLEEP_QUEUE") .ok() .and_then(|x| x.parse::().ok()) - .unwrap_or(windmill::DEFAULT_SLEEP_QUEUE); + .unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE); let disable_nuser = std::env::var("DISABLE_NUSER") .ok() .and_then(|x| x.parse::().ok()) @@ -91,7 +93,7 @@ async fn main() -> anyhow::Result<()> { {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \ {timeout}, KEEP_JOB_DIR: {keep_job_dir}" ); - windmill::run_workers( + run_workers( db.clone(), addr, timeout, @@ -111,24 +113,91 @@ async fn main() -> anyhow::Result<()> { Ok(()) as anyhow::Result<()> }; + let base_url = base_url2; let monitor_f = async { if server_mode { - windmill::monitor_db(&db, timeout, rx.resubscribe()); + monitor_db(&db, timeout, base_url, rx.resubscribe()); } Ok(()) as anyhow::Result<()> }; let metrics_f = async { match metrics_addr { - Some(addr) => windmill::serve_metrics(addr, rx.resubscribe()) + Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe()) .await .map_err(anyhow::Error::from), None => Ok(()), } }; - futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f, metrics_f)?; + futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?; } - + Ok(()) +} + +pub fn monitor_db( + db: &Pool, + timeout: i32, + base_url: String, + rx: tokio::sync::broadcast::Receiver<()>, +) { + let db1 = db.clone(); + let db2 = db.clone(); + + let rx2 = rx.resubscribe(); + + tokio::spawn(async move { + windmill_worker::handle_zombie_jobs_periodically(&db1, timeout, &base_url, rx).await + }); + tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await }); +} + +pub async fn run_workers( + db: Pool, + addr: SocketAddr, + timeout: i32, + num_workers: i32, + sleep_queue: u64, + worker_config: WorkerConfig, + rx: tokio::sync::broadcast::Receiver<()>, +) -> anyhow::Result<()> { + let instance_name = rd_string(5); + let monitor = tokio_metrics::TaskMonitor::new(); + + let ip = windmill_common::external_ip::get_ip() + .await + .unwrap_or_else(|e| { + tracing::warn!(error = e.to_string(), "failed to get external IP"); + "unretrievable IP".to_string() + }); + + let mut handles = Vec::with_capacity(num_workers as usize); + + for i in 1..(num_workers + 1) { + let db1 = db.clone(); + let instance_name = instance_name.clone(); + let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5)); + let ip = ip.clone(); + let rx = rx.resubscribe(); + let worker_config = worker_config.clone(); + handles.push(tokio::spawn(monitor.instrument(async move { + tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker"); + windmill_worker::run_worker( + &db1, + timeout, + &instance_name, + worker_name, + i as u64, + num_workers as u64, + &ip, + sleep_queue, + worker_config, + rx, + ) + .await + }))); + } + + futures::future::try_join_all(handles).await?; Ok(()) } diff --git a/backend/src/users.rs b/backend/src/users.rs deleted file mode 100644 index 516d07eb7b..0000000000 --- a/backend/src/users.rs +++ /dev/null @@ -1,1549 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use std::{sync::Arc, time::Duration}; - -use crate::{ - audit::{audit_log, ActionKind}, - db::{UserDB, DB}, - error::{self, Error, JsonResult, Result}, - utils::{require_admin, require_super_admin, Pagination}, - IsSecure, -}; -use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier}; -use axum::{ - async_trait, - extract::{Extension, FromRequest, Path, Query, RequestParts}, - http, - routing::{delete, get, post}, - Json, Router, -}; -use hyper::StatusCode; -use rand::rngs::OsRng; -use retainer::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use time::OffsetDateTime; -use tower_cookies::{Cookie, Cookies}; -use tracing::Span; - -const TTL_TOKEN_CACHE_S: u64 = 60 * 5; // 5 minutes -pub const TTL_TOKEN_DB_H: u32 = 72; - -const COOKIE_NAME: &str = "token"; -const COOKIE_PATH: &str = "/"; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_users)) - .route("/list_usernames", get(list_usernames)) - .route("/exists", post(exists_username)) - .route("/update/:user", post(update_workspace_user)) - .route("/delete/:user", delete(delete_user)) - .route("/whois/:email", get(whois)) - .route("/whoami", get(whoami)) - .route("/leave", post(leave_workspace)) -} - -pub fn global_service() -> Router { - Router::new() - .route("/email", get(get_email)) - .route("/whoami", get(global_whoami)) - .route("/list_invites", get(list_invites)) - .route("/decline_invite", post(decline_invite)) - .route("/accept_invite", post(accept_invite)) - .route("/list_as_super_admin", get(list_users_as_super_admin)) - .route("/setpassword", post(set_password)) - .route("/create", post(create_user)) - .route("/update/:user", post(update_user)) - .route("/logout", post(logout)) - .route("/tokens/create", post(create_token)) - .route("/tokens/delete/:token_prefix", delete(delete_token)) - .route("/tokens/list", get(list_tokens)) - // .route("/list_invite_codes", get(list_invite_codes)) - // .route("/create_invite_code", post(create_invite_code)) - // .route("/signup", post(signup)) - // .route("/lost_password", post(lost_password)) - // .route("/use_magic_link", get(use_magic_link)) -} - -pub fn make_unauthed_service() -> Router { - Router::new().route("/login", post(login)) -} - -pub struct AuthCache { - cache: Cache<(String, String), Authed>, - db: DB, -} - -impl AuthCache { - pub fn new(db: DB) -> Self { - AuthCache { cache: Cache::new(), db } - } - - pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { - let key = ( - w_id.as_ref().unwrap_or(&"".to_string()).to_string(), - token.to_string(), - ); - let s = self.cache.get(&key).await.map(|c| c.to_owned()); - match s { - a @ Some(_) => a, - None => { - let user_o = sqlx::query_as::<_, (Option, Option, bool)>( - "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \ - OR expiration IS NULL) RETURNING owner, email, super_admin", - ) - .bind(token) - .fetch_optional(&self.db) - .await - .ok() - .flatten(); - - if let Some(user) = user_o { - let authed_o = { - match user { - (_, Some(email), super_admin) => { - if w_id.is_some() { - let row_o = sqlx::query_as::<_, (String, bool)>( - "SELECT username, is_admin FROM usr where email = $1 AND \ - workspace_id = $2", - ) - .bind(&email) - .bind(&w_id.as_ref().unwrap()) - .fetch_optional(&self.db) - .await - .unwrap_or(Some(("error".to_string(), false))); - - match row_o { - Some((username, is_admin)) => { - let groups = get_groups_for_user( - &w_id.as_ref().unwrap(), - &username, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - - Some(Authed { - email: Some(email), - username, - is_admin: is_admin || super_admin, - groups, - }) - } - None if super_admin || w_id.unwrap() == "starter" => { - Some(Authed { - email: Some(email.to_string()), - username: email, - is_admin: super_admin, - groups: vec![], - }) - } - None => None, - } - } else { - Some(Authed { - email: Some(email.to_string()), - username: email, - is_admin: super_admin, - groups: Vec::new(), - }) - } - } - (Some(owner), _, super_admin) if w_id.is_some() => { - if let Some((prefix, name)) = owner.split_once('/') { - if prefix == "u" { - let is_admin = super_admin - || sqlx::query_scalar!( - "SELECT is_admin FROM usr where username = $1 AND \ - workspace_id = $2", - name, - &w_id.as_ref().unwrap() - ) - .fetch_one(&self.db) - .await - .ok() - .unwrap_or(false); - - let groups = - get_groups_for_user(&w_id.unwrap(), &name, &self.db) - .await - .ok() - .unwrap_or_default(); - - Some(Authed { - email: None, - username: name.to_string(), - is_admin, - groups, - }) - } else { - Some(Authed { - email: None, - username: format!("group-{name}"), - is_admin: false, - groups: vec![name.to_string()], - }) - } - } else { - None - } - } - _ => None, - } - }; - if let Some(authed) = authed_o.as_ref() { - self.cache - .insert(key, authed.clone(), Duration::from_secs(TTL_TOKEN_CACHE_S)) - .await; - } - authed_o - } else { - None - } - } - } - } - - pub async fn monitor(&self) { - self.cache.monitor(20, 0.25, Duration::from_secs(10)).await; - } -} - -async fn extract_token(req: &mut RequestParts) -> Option { - let auth_header = req - .headers() - .get(http::header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|s| s.strip_prefix("Bearer ")); - - let from_cookie = match auth_header { - Some(x) => Some(x.to_owned()), - None => Extension::::from_request(req) - .await - .ok() - .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), - }; - - #[derive(Deserialize)] - struct Token { - token: Option, - } - match from_cookie { - Some(token) => Some(token), - None => Query::::from_request(req) - .await - .ok() - .and_then(|token| token.token.clone()), - } -} - -#[derive(Clone, Debug)] -pub struct Tokened { - pub token: String, -} - -#[async_trait] -impl FromRequest for Tokened -where - B: Send, -{ - type Rejection = (StatusCode, String); - - async fn from_request(req: &mut RequestParts) -> std::result::Result { - let already_tokened = req.extensions().get::(); - if let Some(tokened) = already_tokened { - Ok(tokened.clone()) - } else { - let token_o = extract_token(req).await; - if let Some(token) = token_o { - let tokened = Self { token }; - req.extensions_mut().insert(tokened.clone()); - Ok(tokened) - } else { - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) - } - } - } -} - -#[derive(Clone, Debug)] -pub struct Authed { - pub email: Option, - pub username: String, - pub is_admin: bool, - pub groups: Vec, -} - -#[async_trait] -impl FromRequest for Authed -where - B: Send, -{ - type Rejection = (StatusCode, String); - - async fn from_request(req: &mut RequestParts) -> std::result::Result { - let already_authed = req.extensions().get::(); - if let Some(authed) = already_authed { - Ok(authed.clone()) - } else { - let already_tokened = req.extensions().get::(); - let token_o = if let Some(token) = already_tokened { - Some(token.token.clone()) - } else { - extract_token(req).await - }; - let path_vec: Vec<&str> = req.uri().path().split("/").collect(); - let workspace_id = if path_vec[0] == "" && path_vec[1] == "w" { - Some(path_vec[2].to_owned()) - } else { - None - }; - if let Some(token) = token_o { - if let Ok(Extension(cache)) = Extension::>::from_request(req).await { - if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await { - req.extensions_mut().insert(authed.clone()); - Span::current().record("username", &authed.username.as_str()); - if let Some(email) = authed.email.clone() { - Span::current().record("email", &email.as_str()); - } - if let Some(workspace_id) = workspace_id { - Span::current().record("workspace_id", &workspace_id); - } - return Ok(authed); - } - } - } - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) - } - } -} - -#[derive(FromRow, Serialize)] -pub struct User { - pub workspace_id: String, - pub email: String, - pub username: String, - pub is_admin: bool, - pub created_at: chrono::DateTime, - pub operator: bool, - pub disabled: bool, - pub role: Option, -} - -#[derive(FromRow, Serialize)] -pub struct Usage { - pub duration_ms: i64, - pub jobs: i64, - pub flows: i64, -} - -#[derive(Serialize)] -pub struct UserWithUsage { - #[serde(flatten)] - pub user: User, - pub usage: Usage, -} - -#[derive(FromRow, Serialize)] -pub struct GlobalUserInfo { - email: String, - login_type: Option, - super_admin: bool, - verified: bool, - name: Option, - company: Option, -} - -#[derive(Serialize)] -pub struct UserInfo { - pub workspace_id: String, - pub email: String, - pub username: String, - pub is_admin: bool, - pub is_super_admin: bool, - pub created_at: chrono::DateTime, - pub groups: Vec, - pub operator: bool, - pub disabled: bool, - pub role: Option, -} - -#[derive(FromRow, Serialize)] -pub struct WorkspaceInvite { - pub workspace_id: String, - pub email: String, - pub is_admin: bool, -} - -#[derive(FromRow, Serialize)] -pub struct InviteCode { - pub code: String, - pub seats_left: i32, - pub seats_given: i32, -} - -#[derive(Deserialize)] -pub struct NewInviteCode { - pub code: String, - pub seats: i32, -} - -#[derive(FromRow)] -pub struct MagicLink { - pub email: String, - pub token: String, - pub expiration: chrono::DateTime, -} - -#[derive(Deserialize)] -pub struct UseMagicLink { - pub email: String, - pub token: String, -} - -#[derive(Deserialize)] -pub struct NewUser { - pub email: String, - pub password: String, - pub super_admin: bool, - pub name: Option, - pub company: Option, -} - -#[derive(Deserialize)] -pub struct AcceptInvite { - pub workspace_id: String, - pub username: String, -} - -#[derive(Deserialize)] -pub struct DeclineInvite { - pub workspace_id: String, -} - -#[derive(Deserialize)] -pub struct EditUser { - pub is_super_admin: Option, -} - -#[derive(Deserialize)] -pub struct EditWorkspaceUser { - pub is_admin: Option, - pub enabled: Option, -} - -#[derive(Deserialize)] -pub struct EditPassword { - pub password: String, -} - -#[derive(FromRow, Serialize)] -pub struct TruncatedToken { - pub label: Option, - pub token_prefix: Option, - pub expiration: Option>, - pub created_at: chrono::DateTime, - pub last_used_at: chrono::DateTime, -} - -#[derive(Deserialize)] -pub struct NewToken { - pub label: Option, - pub expiration: Option>, -} - -#[derive(Deserialize)] -pub struct Login { - pub email: String, - pub password: String, -} - -#[derive(Deserialize)] -pub struct Signup { - pub email: String, - pub password: String, - pub name: Option, - pub company: Option, -} - -#[derive(Deserialize)] -pub struct LostPassword { - pub email: String, -} - -#[derive(Deserialize)] -struct WorkspaceUsername { - pub username: String, -} - -async fn exists_username( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(WorkspaceUsername { username }): Json, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", - &w_id, - &username - ) - .fetch_one(&mut tx) - .await? - .unwrap_or(false); - tx.commit().await?; - Ok(Json(exists)) -} - -async fn list_users( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query( - " - SELECT usr.*, usage.* - FROM usr - , LATERAL ( - SELECT COALESCE(SUM(duration_ms), 0) duration_ms - , COALESCE(SUM(job_kind IN ('flow', 'flowpreview') ::int), 0) flows - , COALESCE(SUM(job_kind NOT IN ('flow', 'flowpreview') ::int), 0) jobs - FROM completed_job - WHERE workspace_id = usr.workspace_id - AND created_by = usr.username - AND parent_job IS NULL - AND now() - '2 week'::interval < created_at - ) usage - WHERE workspace_id = $1 - ", - ) - .bind(&w_id) - .try_map(|row| { - // flatten not released yet https://github.com/launchbadge/sqlx/pull/1959 - Ok(UserWithUsage { user: FromRow::from_row(&row)?, usage: FromRow::from_row(&row)? }) - }) - .fetch_all(&mut tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_users_as_super_admin( - authed: Authed, - Extension(db): Extension, - Query(pagination): Query, -) -> JsonResult> { - let mut tx = db.begin().await?; - require_super_admin(&mut tx, authed.email).await?; - let (per_page, offset) = crate::utils::paginate(pagination); - - let rows = sqlx::query_as!( - GlobalUserInfo, - "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT \ - $1 OFFSET $2", - per_page as i32, - offset as i32 - ) - .fetch_all(&mut tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -// async fn list_invite_codes( -// authed: Authed, -// Extension(db): Extension, -// Query(pagination): Query -// ) -> JsonResult> { -// let mut tx = db.begin().await?; -// require_super_admin(&mut tx, authed.email).await?; -// let (per_page, offset) = crate::utils::paginate(pagination); - -// let rows = sqlx::query_as!(InviteCode, "SELECT * from invite_code LIMIT $1 OFFSET $2", per_page as i32, offset as i32) -// .fetch_all(&mut tx) -// .await?; -// tx.commit().await?; -// Ok(Json(rows)) -// } - -async fn list_usernames( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id) - .fetch_all(&mut tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_invites( - authed: Authed, - Extension(db): Extension, -) -> JsonResult> { - let mut tx = db.begin().await?; - let rows = sqlx::query_as!( - WorkspaceInvite, - "SELECT * from workspace_invite WHERE email = $1", - authed.email - ) - .fetch_all(&mut tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn logout( - Tokened { token }: Tokened, - cookies: Cookies, - Extension(db): Extension, - Authed { username, .. }: Authed, -) -> Result { - let mut cookie = Cookie::new(COOKIE_NAME, ""); - cookie.set_path(COOKIE_PATH); - cookies.remove(cookie); - let mut tx = db.begin().await?; - audit_log( - &mut tx, - &username, - "users.logout", - ActionKind::Delete, - "global", - Some(&truncate_token(&token)), - None, - ) - .await?; - tx.commit().await?; - - Ok(username) -} - -async fn whoami( - Extension(db): Extension, - Path(w_id): Path, - Authed { username, email, is_admin, groups }: Authed, -) -> JsonResult { - let user = get_user(&w_id, &username, &db).await?; - if let Some(user) = user { - Ok(Json(user)) - } else { - let email = email.unwrap_or_else(|| "noemail".to_string()); - Ok(Json(UserInfo { - workspace_id: w_id, - email: email.clone(), - username: email, - is_admin, - is_super_admin: is_admin, - created_at: chrono::Utc::now(), - groups: groups, - operator: false, - disabled: false, - role: Some("superadmin".to_string()), - })) - } -} - -async fn global_whoami( - Extension(db): Extension, - Authed { email, .. }: Authed, -) -> JsonResult { - let user: GlobalUserInfo = sqlx::query_as!( - GlobalUserInfo, - "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE \ - email = $1", - email - ) - .fetch_one(&db) - .await - .map_err(|e| Error::InternalErr(format!("fetching global identity: {e}")))?; - - Ok(Json(user)) -} - -async fn get_email(Authed { email, .. }: Authed) -> Result { - let email = email.ok_or(Error::BadRequest( - "current session does not correspond to an user with email".to_string(), - ))?; - Ok(email) -} - -async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { - let user = sqlx::query_as!( - User, - "SELECT * FROM usr where username = $1 AND workspace_id = $2", - username, - w_id - ) - .fetch_optional(db) - .await?; - let is_super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - user.as_ref().map(|x| &x.email) - ) - .fetch_optional(db) - .await? - .unwrap_or(false); - let groups = get_groups_for_user(&w_id, username, db).await?; - Ok(user.map(|usr| UserInfo { - groups, - workspace_id: usr.workspace_id, - email: usr.email, - username: usr.username, - is_admin: usr.is_admin, - is_super_admin, - created_at: usr.created_at, - operator: usr.operator, - disabled: usr.disabled, - role: usr.role, - })) -} - -async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result> { - let groups = sqlx::query_scalar!( - "SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2", - username, - w_id - ) - .fetch_all(db) - .await?; - Ok(groups) -} - -async fn whois( - Extension(db): Extension, - Path((w_id, username)): Path<(String, String)>, -) -> JsonResult { - let user_o = get_user(&w_id, &username, &db).await?; - let user = crate::utils::not_found_if_none(user_o, "User", username)?; - Ok(Json(user)) -} - -// async fn create_invite_code( -// Authed { email, .. }: Authed, -// Extension(db): Extension, -// Json(nu): Json, -// ) -> Result<(StatusCode, String)> { - -// let mut tx = db.begin().await?; -// require_super_admin(&mut tx, email).await?; - -// sqlx::query!( -// "INSERT INTO invite_code -// (code, seats_left) -// VALUES ($1, $2)", -// nu.code, -// nu.seats -// ) -// .execute(&mut tx) -// .await?; - -// tx.commit().await?; - -// Ok(( -// StatusCode::CREATED, -// format!("new invite code {}", nu.code), -// )) -// } - -async fn decline_invite( - Authed { email, .. }: Authed, - Extension(db): Extension, - Json(nu): Json, -) -> Result<(StatusCode, String)> { - let mut tx = db.begin().await?; - - let email = email.unwrap_or("".to_string()); - let is_admin = sqlx::query_scalar!( - "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin", - nu.workspace_id, - email, - ) - .fetch_optional(&mut tx) - .await?; - - audit_log( - &mut tx, - &email, - "users.decline_invite", - ActionKind::Delete, - &nu.workspace_id, - Some(&email), - None, - ) - .await?; - tx.commit().await?; - - if is_admin.is_some() { - Ok(( - StatusCode::OK, - format!( - "user {} declined invite to workspace {}", - &email, nu.workspace_id - ), - )) - } else { - Err(Error::NotFound(format!("invite for {email} not found"))) - } -} - -async fn accept_invite( - Authed { email, .. }: Authed, - Extension(db): Extension, - Json(nu): Json, -) -> Result<(StatusCode, String)> { - if &nu.username == "bot" { - return Err(Error::BadRequest("bot is a reserved username".to_string())); - } - let mut tx = db.begin().await?; - - let email = email.unwrap_or("".to_string()); - let is_admin = sqlx::query_scalar!( - "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin", - nu.workspace_id, - email, - ) - .fetch_optional(&mut tx) - .await?; - - if let Some(is_admin) = is_admin { - tx = add_user_to_workspace(&nu.workspace_id, &email, &nu.username, is_admin, tx).await?; - } - - audit_log( - &mut tx, - &nu.username, - "users.accept_invite", - ActionKind::Create, - &nu.workspace_id, - Some(&email), - None, - ) - .await?; - tx.commit().await?; - - if is_admin.is_some() { - Ok(( - StatusCode::CREATED, - format!( - "user {} accepted invite to workspace {}", - &email, nu.workspace_id - ), - )) - } else { - Err(Error::NotFound(format!("invite for {email} not found"))) - } -} - -async fn add_user_to_workspace<'c>( - w_id: &str, - email: &str, - username: &str, - is_admin: bool, - mut tx: sqlx::Transaction<'c, sqlx::Postgres>, -) -> error::Result> { - sqlx::query!( - "INSERT INTO usr - (workspace_id, email, username, is_admin) - VALUES ($1, $2, $3, $4)", - &w_id, - email, - username, - is_admin - ) - .execute(&mut tx) - .await?; - sqlx::query_as!( - Group, - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)", - &w_id, - username, - "all", - ) - .execute(&mut tx) - .await?; - audit_log( - &mut tx, - username, - "users.add_to_workspace", - ActionKind::Create, - &w_id, - Some(email), - None, - ) - .await?; - Ok(tx) -} - -async fn update_workspace_user( - Authed { username, is_admin, .. }: Authed, - Extension(db): Extension, - Path((w_id, username_to_update)): Path<(String, String)>, - Json(eu): Json, -) -> Result { - let mut tx = db.begin().await?; - - require_admin(is_admin, &username)?; - - if let Some(a) = eu.is_admin { - sqlx::query_scalar!( - "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3", - a, - &username_to_update, - &w_id - ) - .execute(&mut tx) - .await?; - } - - audit_log( - &mut tx, - &username, - "users.update", - ActionKind::Update, - &w_id, - Some(&username_to_update), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("user {username} updated")) -} - -async fn update_user( - Authed { email, .. }: Authed, - Path(email_to_update): Path, - Extension(db): Extension, - Json(eu): Json, -) -> Result { - let mut tx = db.begin().await?; - - require_super_admin(&mut tx, email.clone()).await?; - - if let Some(sa) = eu.is_super_admin { - sqlx::query_scalar!( - "UPDATE password SET super_admin = $1 WHERE email = $2", - sa, - &email_to_update - ) - .execute(&mut tx) - .await?; - } - - audit_log( - &mut tx, - &email.unwrap(), - "users.update", - ActionKind::Update, - "global", - Some(&email_to_update), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("email {} updated", &email_to_update)) -} - -async fn create_user( - Authed { email, .. }: Authed, - Extension(db): Extension, - Extension(argon2): Extension>>, - Json(nu): Json, -) -> Result<(StatusCode, String)> { - let mut tx = db.begin().await?; - - require_super_admin(&mut tx, email.clone()).await?; - - sqlx::query!( - "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, \ - company) - VALUES ($1, $2, $3, 'password', $4, $5, $6)", - &nu.email, - true, - &hash_password(argon2, nu.password)?, - &nu.super_admin, - nu.name, - nu.company - ) - .execute(&mut tx) - .await?; - - audit_log( - &mut tx, - &email.unwrap(), - "users.update", - ActionKind::Update, - "global", - Some(&nu.email), - None, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, format!("email {} created", nu.email))) -} - -pub fn owner_to_token_owner(user: &str, is_group: bool) -> String { - let prefix = if is_group { 'g' } else { 'u' }; - format!("{}/{}", prefix, user) -} - -async fn delete_user( - Authed { username, is_admin, .. }: Authed, - Extension(db): Extension, - Path((w_id, username_to_delete)): Path<(String, String)>, -) -> Result { - let mut tx = db.begin().await?; - - require_admin(is_admin, &username)?; - - let email_to_delete_o = sqlx::query_scalar!( - "SELECT email FROM usr where username = $1 AND workspace_id = $2", - username_to_delete, - &w_id, - ) - .fetch_optional(&db) - .await?; - - let email_to_delete = - crate::utils::not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; - - sqlx::query!("DELETE FROM usr WHERE email = $1", email_to_delete) - .execute(&mut tx) - .await?; - - audit_log( - &mut tx, - &username, - "users.delete", - ActionKind::Delete, - &w_id, - Some(&username_to_delete), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("username {} deleted", username_to_delete)) -} - -async fn set_password( - Extension(db): Extension, - Extension(argon2): Extension>>, - Authed { username, email, .. }: Authed, - Json(EditPassword { password }): Json, -) -> Result { - let mut tx = db.begin().await?; - let email = email - .ok_or("no_email") - .map_err(|e| Error::NotAuthorized(e.to_string()))?; - - let custom_type = sqlx::query_scalar!( - "SELECT login_type::TEXT FROM password WHERE email = $1", - &email - ) - .fetch_one(&mut tx) - .await - .map_err(|e| Error::InternalErr(format!("setting password: {e}")))? - .unwrap_or("".to_string()); - - if custom_type != "password".to_string() { - return Err(Error::BadRequest(format!( - "login type for {email} is of type {custom_type}. Cannot set password." - ))); - } - - sqlx::query!( - "UPDATE password SET password_hash = $1 WHERE email = $2", - &hash_password(argon2, password)?, - &email, - ) - .execute(&mut tx) - .await?; - - audit_log( - &mut tx, - &username, - "users.setpassword", - ActionKind::Update, - "global", - Some(&email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("password of {} updated", email)) -} - -pub async fn get_email_from_username(username: &String, db: &DB) -> Result> { - let email = sqlx::query_scalar!("SELECT email FROM usr WHERE username = $1", username) - .fetch_optional(db) - .await?; - Ok(email) -} - -pub fn hash_password(argon2: Arc, password: String) -> Result { - let salt = SaltString::generate(&mut OsRng); - let password_hash = argon2 - .hash_password(password.as_bytes(), &salt) - .map_err(|e| Error::InternalErr(e.to_string()))? - .to_string(); - Ok(password_hash) -} - -// async fn lost_password( -// Extension(db): Extension, -// Extension(es): Extension>, -// TypedHeader(host): TypedHeader, -// Json(LostPassword { -// email -// }): Json, -// ) -> Result { -// let mut tx = db.begin().await?; - -// let exists = sqlx::query_scalar!( -// "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", -// &email) -// .fetch_one(&mut tx) -// .await? -// .unwrap_or(false); - -// if !exists { -// return Err(Error::NotFound(format!("no user found at email {email}"))) -// } - -// let already = sqlx::query_scalar!( -// "SELECT EXISTS(SELECT 1 FROM magic_link WHERE email = $1)", -// &email) -// .fetch_one(&mut tx) -// .await? -// .unwrap_or(false); - -// if already { -// return Err(Error::BadRequest(format!("a magic link was already sent at {email}"))) -// } - -// let tx = create_magic_link(&host.hostname(), &email, &es, tx).await?; -// tx.commit().await?; - -// Ok(format!("Magic link sent to {email}")) -// } - -// async fn use_magic_link( -// cookies: Cookies, -// Extension(db): Extension, -// Query(UseMagicLink { -// email, -// token -// }): Query, -// ) -> Result { -// let mut tx = db.begin().await?; - -// let email_o = sqlx::query_scalar!( -// "DELETE FROM magic_link WHERE email = $1 AND token = $2 -// RETURNING email", email, token -// ) -// .fetch_optional(&mut tx) -// .await?; - -// if let Some(email) = email_o { -// let is_super_admin = sqlx::query_scalar!("UPDATE password SET verified = true WHERE email = $1 RETURNING super_admin", email) -// .fetch_optional(&mut tx) -// .await? -// .unwrap_or(false); - -// let token = create_session_token(&email, is_super_admin, &mut tx, cookies).await?; -// tx.commit().await?; -// Ok(token) -// } else { -// Err(Error::NotFound(format!("magic link for {email} not found"))) -// } -// } - -// async fn signup( -// TypedHeader(host): TypedHeader, -// Extension(db): Extension, -// Extension(argon2): Extension>>, -// Extension(es): Extension>, -// Json(Signup { -// email, -// password, -// name, -// company -// }): Json, -// ) -> Result<(StatusCode, String)> { -// let mut tx = db.begin().await?; - -// let email = sqlx::query_scalar!( -// "INSERT INTO password (email, password_hash, name, company) VALUES ($1, $2, $3, $4) RETURNING email", -// &email, &hash_password(argon2, password)?, name, company) -// .fetch_optional(&mut tx) -// .await?; - -// if let Some(email) = email { -// let tx = create_magic_link(&host.hostname(), &email, &es, tx).await?; -// tx.commit().await?; - -// Ok(( -// StatusCode::CREATED, -// format!("user with email {} created", email), -// )) -// } else { -// Err(Error::BadRequest("Invalid login".to_string())) -// } -// } - -// async fn create_magic_link<'c>(host: &str, email: &str, es: &EmailSender, mut tx: sqlx::Transaction<'c, sqlx::Postgres>) -> error::Result> { -// let token = gen_token(); - -// sqlx::query!( -// "INSERT INTO magic_link -// (email, token) -// VALUES ($1, $2)", -// email, -// &token -// ) -// .execute(&mut tx) -// .await?; - -// let encoded_token = urlencoding::encode(&token); -// let encoded_email = urlencoding::encode(email); -// es.send_email(Message::builder() -// .to(email.parse().unwrap()) -// .subject("New magic link") -// .body(format!("Magic link: https://{host}/magic_link?token={encoded_token}&email={encoded_email}")) -// .unwrap()).await?; - -// audit_log( -// &mut tx, -// email, -// "users.magic_link", -// ActionKind::Create, -// "global", -// Some(email), -// None, -// ) -// .await?; -// Ok(tx) -// } - -async fn login( - cookies: Cookies, - Extension(db): Extension, - Extension(argon2): Extension>>, - Extension(is_secure): Extension>, - Json(Login { email, password }): Json, -) -> Result { - let mut tx = db.begin().await?; - - let email_w_h: Option<(String, String, bool)> = sqlx::query_as( - "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \ - 'password'", - ) - .bind(&email) - .fetch_optional(&mut tx) - .await?; - - if let Some((email, hash, super_admin)) = email_w_h { - let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::InternalErr(e.to_string()))?; - if argon2 - .verify_password(password.as_bytes(), &parsed_hash) - .is_err() - { - Err(Error::BadRequest("Invalid login".to_string())) - } else { - let token = - create_session_token(&email, super_admin, &mut tx, cookies, is_secure.0).await?; - tx.commit().await?; - Ok(token) - } - } else { - Err(Error::BadRequest("Invalid login".to_string())) - } -} - -pub async fn create_session_token<'c>( - email: &str, - super_admin: bool, - tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, - cookies: Cookies, - is_secure: bool, -) -> Result { - let token = gen_token(); - sqlx::query!( - "INSERT INTO token - (token, email, label, expiration, super_admin) - VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)", - token, - email, - "session", - TTL_TOKEN_DB_H.to_string(), - super_admin - ) - .execute(tx) - .await?; - let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); - cookie.set_secure(is_secure); - cookie.set_path(COOKIE_PATH); - let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); - expire += time::Duration::days(3); - cookie.set_expires(expire); - cookies.add(cookie); - Ok(token) -} - -fn gen_token() -> String { - use rand::prelude::*; - let token: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(30) - .map(char::from) - .collect(); - token -} - -#[tracing::instrument(level = "trace", skip_all)] -pub async fn create_token_for_owner( - db: &DB, - w_id: &str, - owner: &str, - label: &str, - expires_in: i32, - username: &str, -) -> Result { - use rand::prelude::*; - let token: String = tracing::trace_span!("generate_token").in_scope(|| { - rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(30) - .map(char::from) - .collect() - }); - let mut tx = db.begin().await?; - let is_super_admin = username.contains('@') - && sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - owner.split_once('/').map(|x| x.1).unwrap_or("") - ) - .fetch_optional(&mut tx) - .await? - .unwrap_or(false); - - let expiration = sqlx::query_scalar!( - "INSERT INTO token - (workspace_id, token, owner, label, expiration, super_admin) - VALUES ($1, $2, $3, $4, now() + ($5 || ' seconds')::interval, $6) RETURNING expiration", - &w_id, - token, - owner, - label, - expires_in.to_string(), - is_super_admin - ) - .fetch_one(&mut tx) - .await?; - audit_log( - &mut tx, - &username, - "users.token.create", - ActionKind::Create, - w_id, - Some(&truncate_token(&token)), - Some( - [ - Some(("label", label)), - expiration - .map(|x| x.to_string()) - .as_ref() - .map(|exp| ("expiration", &exp[..])), - ] - .into_iter() - .flatten() - .collect(), - ), - ) - .await?; - tx.commit().await?; - Ok(token) -} - -async fn create_token( - Extension(db): Extension, - Authed { email, .. }: Authed, - Json(new_token): Json, -) -> Result<(StatusCode, String)> { - let token = gen_token(); - let mut tx = db.begin().await?; - let email = email.ok_or_else(|| { - error::Error::BadRequest(format!("Only users with email can create tokens")) - })?; - let is_super_admin = - sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) - .fetch_optional(&mut tx) - .await? - .unwrap_or(false); - sqlx::query!( - "INSERT INTO token - (token, email, label, expiration, super_admin) - VALUES ($1, $2, $3, $4, $5)", - token, - email, - new_token.label, - new_token.expiration, - is_super_admin - ) - .execute(&mut tx) - .await?; - - audit_log( - &mut tx, - &email, - "users.token.create", - ActionKind::Delete, - &"global", - Some(&token[0..10]), - None, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, token)) -} - -async fn list_tokens( - Extension(db): Extension, - Authed { email, .. }: Authed, -) -> JsonResult> { - let rows = sqlx::query_as!( - TruncatedToken, - "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ - last_used_at FROM token WHERE email = $1", - email, - ) - .fetch_all(&db) - .await?; - Ok(Json(rows)) -} - -async fn delete_token( - Extension(db): Extension, - Authed { email, .. }: Authed, - Path(token_prefix): Path, -) -> Result { - let mut tx = db.begin().await?; - let email = email.ok_or_else(|| { - error::Error::BadRequest(format!("Only users with email can create tokens")) - })?; - let tokens_deleted: Vec = sqlx::query_scalar( - "DELETE FROM token - WHERE email = $1 - AND token LIKE concat($2::text, '%') - RETURNING concat(substring(token for 10), '*****')", - ) - .bind(&email) - .bind(&token_prefix) - .fetch_all(&mut tx) - .await?; - - audit_log( - &mut tx, - &email, - "users.token.delete", - ActionKind::Delete, - &"global", - Some(&token_prefix), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!( - "deleted {} tokens {:?} with prefix {}", - tokens_deleted.len(), - tokens_deleted, - token_prefix - )) -} - -async fn leave_workspace( - Extension(db): Extension, - Path(w_id): Path, - Authed { username, .. }: Authed, -) -> Result { - let mut tx = db.begin().await?; - sqlx::query!( - "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", - &w_id, - username - ) - .execute(&mut tx) - .await?; - - audit_log( - &mut tx, - &username, - "users.leave_workspace", - ActionKind::Delete, - &w_id, - None, - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("left workspace {w_id}")) -} - -pub async fn delete_expired_items_perdiodically( - db: &DB, - mut rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - loop { - let tokens_deleted_r: std::result::Result, _> = sqlx::query_scalar( - "DELETE FROM token WHERE expiration <= now() - RETURNING concat(substring(token for 10), '*****')", - ) - .fetch_all(db) - .await; - - match tokens_deleted_r { - Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens), - Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), - } - - let magic_links_deleted_r: std::result::Result, _> = sqlx::query_scalar( - "DELETE FROM magic_link WHERE expiration <= now() - RETURNING concat(substring(token for 10), '*****')", - ) - .fetch_all(db) - .await; - - match magic_links_deleted_r { - Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens), - Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), - } - - tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(600)) => (), - _ = rx. recv() => { - println!("received killpill for delete expired tokens"); - break; - } - } - } -} - -pub fn truncate_token(token: &str) -> String { - let mut s = token[..10].to_owned(); - s.push_str("*****"); - s -} diff --git a/backend/src/worker.rs b/backend/src/worker.rs deleted file mode 100644 index d915b012e9..0000000000 --- a/backend/src/worker.rs +++ /dev/null @@ -1,3934 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use itertools::Itertools; -use std::{borrow::Borrow, collections::HashMap, io, panic, process::Stdio, time::Duration}; -use tracing::{trace_span, Instrument}; -use uuid::Uuid; - -use crate::{ - db::DB, - error::{self, Error}, - jobs::{ - add_completed_job, add_completed_job_error, canceled_job_to_result, get_hub_script, - get_queued_job, pull, JobKind, QueuedJob, - }, - parser::Typ, - parser_go::otyp_to_string, - parser_py, - scripts::{ScriptHash, ScriptLang}, - users::{create_token_for_owner, get_email_from_username}, - variables, - worker_flow::{ - handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, - }, -}; - -use serde_json::{json, Map, Value}; - -use tokio::{ - fs::{metadata, symlink, DirBuilder, File}, - io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, - process::{Child, Command}, - sync::{ - mpsc::{self, Sender}, - oneshot, watch, - }, - time::{interval, sleep, Instant, MissedTickBehavior}, -}; - -use futures::{ - future::{self, ready, FutureExt}, - stream::{self, StreamExt}, -}; - -use async_recursion::async_recursion; - -const TMP_DIR: &str = "/tmp/windmill"; -const PIP_SUPERCACHE_DIR: &str = "/tmp/windmill/cache/pip_permanent"; -const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip"; -const DENO_CACHE_DIR: &str = "/tmp/windmill/cache/deno"; -const GO_CACHE_DIR: &str = "/tmp/windmill/cache/go"; -const NUM_SECS_ENV_CHECK: u64 = 15; -const DEFAULT_HEAVY_DEPS: [&str; 18] = [ - "numpy", - "pandas", - "anyio", - "attrs", - "certifi", - "h11", - "httpcore", - "httpx", - "idna", - "python-dateutil", - "rfc3986", - "six", - "sniffio", - "windmill-api", - "wmill", - "psycopg2-binary", - "matplotlib", - "seaborn", -]; - -const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../../nsjail/download_deps.py.sh"); -const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = - include_str!("../../nsjail/download.py.config.proto"); -const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = - include_str!("../../nsjail/run.python3.config.proto"); - -const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../../nsjail/run.go.config.proto"); - -const NSJAIL_CONFIG_RUN_DENO_CONTENT: &str = include_str!("../../nsjail/run.deno.config.proto"); -const MAX_LOG_SIZE: u32 = 200000; -const GO_REQ_SPLITTER: &str = "//go.sum"; - -#[derive(Clone)] -pub struct Metrics { - pub worker_execution_failed: prometheus::IntCounter, -} - -#[derive(Clone, Debug)] -pub struct WorkerConfig { - pub base_internal_url: String, - pub base_url: String, - pub disable_nuser: bool, - pub disable_nsjail: bool, - pub keep_job_dir: bool, -} - -lazy_static::lazy_static! { - static ref WORKER_STARTED: prometheus::IntGauge = prometheus::register_int_gauge!( - "worker_started", - "Total number of workers started." - ) - .unwrap(); - static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( - "queue_zombie_restart_count", - "Total number of jobs restarted due to ping timeout." - ) - .unwrap(); - static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( - "queue_zombie_delete_count", - "Total number of jobs deleted due to their ping timing out in an unrecoverable state." - ) - .unwrap(); - static ref WORKER_UPTIME_OPTS: prometheus::Opts = prometheus::opts!( - "worker_uptime", - "Total number of milliseconds since the worker has started" - ); -} - -#[tracing::instrument(level = "trace")] -pub async fn run_worker( - db: &DB, - timeout: i32, - worker_instance: &str, - worker_name: String, - i_worker: u64, - num_workers: u64, - ip: &str, - sleep_queue: u64, - worker_config: WorkerConfig, - mut rx: tokio::sync::broadcast::Receiver<()>, -) { - let start_time = Instant::now(); - - let worker_dir = format!("{TMP_DIR}/{worker_name}"); - tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir"); - - for x in [ - &worker_dir, - PIP_SUPERCACHE_DIR, - PIP_CACHE_DIR, - DENO_CACHE_DIR, - GO_CACHE_DIR, - ] { - DirBuilder::new() - .recursive(true) - .create(x) - .await - .expect("could not create initial worker dir"); - } - - let _ = write_file( - &worker_dir, - "download_deps.py.sh", - INCLUDE_DEPS_PY_SH_CONTENT, - ) - .await; - - let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_ENV_CHECK + 1); - - insert_initial_ping(worker_instance, &worker_name, ip, db).await; - - let uptime_metric = prometheus::register_int_counter!(WORKER_UPTIME_OPTS - .clone() - .const_label("name", &worker_name)) - .unwrap(); - uptime_metric.inc_by( - ((Instant::now() - start_time).as_millis() - uptime_metric.get() as u128) - .try_into() - .unwrap(), - ); - - let worker_execution_duration = prometheus::register_histogram_vec!( - prometheus::HistogramOpts::new( - "worker_execution_duration", - "Duration between receiving a job and completing it", - ) - .const_label("name", &worker_name), - &["workspace_id", "language"], - ) - .expect("register prometheus metric"); - - let worker_execution_failed = prometheus::register_int_counter_vec!( - prometheus::Opts::new("worker_execution_failed", "Number of failed jobs",) - .const_label("name", &worker_name), - &["workspace_id", "language"], - ) - .expect("register prometheus metric"); - - let worker_execution_count = prometheus::register_int_counter_vec!( - prometheus::Opts::new("worker_execution_count", "Number of executed jobs",) - .const_label("name", &worker_name), - &["workspace_id", "language"], - ) - .expect("register prometheus metric"); - - let mut jobs_executed = 0; - - let deno_path = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); - let go_path = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string()); - let python_path = - std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); - let python_heavy_deps = std::env::var("PYTHON_HEAVY_DEPS") - .map(|x| x.split(',').map(|x| x.to_string()).collect::>()) - .unwrap_or_else(|_| vec![]); - let nsjail_path = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); - let path_env = std::env::var("PATH").unwrap_or_else(|_| String::new()); - let gopath_env = std::env::var("GOPATH").unwrap_or_else(|_| String::new()); - let home_env = std::env::var("HOME").unwrap_or_else(|_| String::new()); - let pip_index_url = std::env::var("PIP_INDEX_URL").ok(); - let pip_extra_index_url = std::env::var("PIP_EXTRA_INDEX_URL").ok(); - let pip_trusted_host = std::env::var("PIP_TRUSTED_HOST").ok(); - let envs = Envs { - deno_path, - go_path, - python_path, - python_heavy_deps, - nsjail_path, - path_env, - gopath_env, - home_env, - pip_index_url, - pip_extra_index_url, - pip_trusted_host, - }; - WORKER_STARTED.inc(); - - let (same_worker_tx, mut same_worker_rx) = mpsc::channel::(5); - - loop { - uptime_metric.inc_by( - ((Instant::now() - start_time).as_millis() - uptime_metric.get() as u128) - .try_into() - .unwrap(), - ); - let do_break = async { - if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK { - sqlx::query!( - "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2", - jobs_executed, - &worker_name - ) - .execute(db) - .await - .expect("update worker ping"); - - last_ping = Instant::now(); - } - - let (do_break, next_job) = async { - tokio::select! { - biased; - _ = rx.recv() => { - println!("received killpill for worker {}", i_worker); - (true, Ok(None)) - }, - Some(job_id) = same_worker_rx.recv() => { - (false, sqlx::query_as::<_, QueuedJob>("SELECT * FROM queue WHERE id = $1") - .bind(job_id) - .fetch_optional(db) - .await - .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()))) - }, - job = pull(&db) => (false, job), - } - }.instrument(trace_span!("worker_get_next_job")).await; - if do_break { - return true; - } - - match next_job { - Ok(Some(job)) => { - let label_values = [ - &job.workspace_id, - job.language.as_ref().map(|l| l.as_str()).unwrap_or(""), - ]; - - let _timer = worker_execution_duration - .with_label_values(label_values.as_slice()) - .start_timer(); - - jobs_executed += 1; - worker_execution_count - .with_label_values(label_values.as_slice()) - .inc(); - - let metrics = Metrics { - worker_execution_failed: worker_execution_failed - .with_label_values(label_values.as_slice()), - }; - - tracing::info!(worker = %worker_name, id = %job.id, "fetched job {}", job.id); - - let job_dir = format!("{worker_dir}/{}", job.id); - - DirBuilder::new() - .create(&job_dir) - .await - .expect("could not create job dir"); - - let same_worker = job.same_worker; - let is_flow = job.job_kind == JobKind::Flow || job.job_kind == JobKind::FlowPreview; - - if is_flow && same_worker { - let target = &format!("{job_dir}/shared"); - if let Some(parent_flow) = job.parent_job { - let parent_shared_dir = format!("{worker_dir}/{parent_flow}/shared"); - if metadata(&parent_shared_dir).await.is_err() { - DirBuilder::new() - .recursive(true) - .create(&parent_shared_dir) - .await - .expect("could not create parent shared dir"); - } - symlink(&parent_shared_dir, target) - .await - .expect("could not symlink target"); - } else { - DirBuilder::new() - .create(target) - .await - .expect("could not create shared dir"); - } - } - - if let Some(err) = handle_queued_job( - job.clone(), - db, - timeout, - &worker_name, - &worker_dir, - &job_dir, - &worker_config, - metrics.clone(), - &envs, - same_worker_tx.clone(), - &worker_config.base_internal_url, - ) - .await - .err() - { - handle_job_error( - db, - job, - err, - Some(metrics), - false, - same_worker_tx.clone(), - &worker_dir, - !worker_config.keep_job_dir, - &worker_config.base_internal_url, - ) - .await; - }; - - if !worker_config.keep_job_dir && !(is_flow && same_worker) { - let _ = tokio::fs::remove_dir_all(job_dir).await; - } - } - Ok(None) => { - tokio::time::sleep(Duration::from_millis(sleep_queue * num_workers)).await - } - Err(err) => { - tracing::error!(worker = %worker_name, "run_worker: pulling jobs: {}", err); - } - }; - - false - } - .instrument(trace_span!("worker_loop_iteration")) - .await; - if do_break { - break; - } - } -} - -async fn handle_job_error( - db: &DB, - job: QueuedJob, - err: Error, - metrics: Option, - unrecoverable: bool, - same_worker_tx: Sender, - worker_dir: &str, - keep_job_dir: bool, - base_internal_url: &str, -) { - let m = add_completed_job_error( - db, - &job, - "Unexpected error during job execution:\n".to_string(), - &err, - metrics.clone(), - ) - .await - .map(|(_, m)| m) - .unwrap_or_else(|_| Map::new()); - - if let Some(parent_job_id) = job.parent_job { - let updated_flow = update_flow_status_after_job_completion( - db, - &job, - false, - serde_json::Value::Object(m), - metrics.clone(), - unrecoverable, - same_worker_tx, - worker_dir, - keep_job_dir, - base_internal_url, - ) - .await; - if let Err(err) = updated_flow { - if let Ok(mut tx) = db.begin().await { - if let Ok(Some(parent_job)) = - get_queued_job(parent_job_id, &job.workspace_id, &mut tx).await - { - let _ = add_completed_job_error( - db, - &parent_job, - format!("Unexpected error during flow job error handling:\n{err}"), - err, - metrics, - ) - .await; - } - } - } - } - tracing::error!(job_id = %job.id, err = err.alt(), "error handling job: {} {} {}", job.id, job.workspace_id, job.created_by); -} - -async fn insert_initial_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) { - sqlx::query!( - "INSERT INTO worker_ping (worker_instance, worker, ip) VALUES ($1, $2, $3)", - worker_instance, - worker_name, - ip - ) - .execute(db) - .await - .expect("insert worker_ping initial value"); -} - -struct Envs { - deno_path: String, - go_path: String, - python_path: String, - python_heavy_deps: Vec, - nsjail_path: String, - path_env: String, - gopath_env: String, - home_env: String, - pip_index_url: Option, - pip_extra_index_url: Option, - pip_trusted_host: Option, -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_queued_job( - job: QueuedJob, - db: &sqlx::Pool, - timeout: i32, - worker_name: &str, - worker_dir: &str, - job_dir: &str, - worker_config: &WorkerConfig, - metrics: Metrics, - envs: &Envs, - same_worker_tx: Sender, - base_internal_url: &str, -) -> crate::error::Result<()> { - if job.canceled { - return Err(Error::ExecutionErr(canceled_job_to_result(&job)))?; - } - match job.job_kind { - JobKind::FlowPreview | JobKind::Flow => { - let args = job.args.clone().unwrap_or(Value::Null); - handle_flow( - &job, - db, - args, - same_worker_tx, - worker_dir, - base_internal_url, - ) - .await?; - } - _ => { - let mut logs = "".to_string(); - - if job.is_flow_step { - update_flow_status_in_progress( - db, - &job.workspace_id, - job.parent_job - .ok_or_else(|| Error::InternalErr(format!("expected parent job")))?, - job.id, - ) - .await?; - } - - tracing::info!( - worker = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "handling job {}", - job.id - ); - - logs.push_str(&format!("job {} on worker {}\n", &job.id, &worker_name)); - - let result = match job.job_kind { - JobKind::Dependencies => { - handle_dependency_job(&job, &mut logs, job_dir, db, timeout, &envs).await - } - JobKind::Identity => Ok(job.args.clone().unwrap_or_else(|| Value::Null)), - _ => { - handle_code_execution_job( - &job, - db, - job_dir, - worker_dir, - &mut logs, - timeout, - worker_config, - envs, - ) - .await - } - }; - - match result { - Ok(r) => { - add_completed_job(db, &job, true, false, r.clone(), logs).await?; - if job.is_flow_step { - update_flow_status_after_job_completion( - db, - &job, - true, - r, - Some(metrics.clone()), - false, - same_worker_tx.clone(), - worker_dir, - worker_config.keep_job_dir, - &worker_config.base_internal_url, - ) - .await?; - } - } - Err(e) => { - let error_message = match e { - Error::ExitStatus(_) => { - let last_10_log_lines = logs - .lines() - .skip(logs.lines().count().max(10) - 10) - .join("\n") - .to_string() - .replace("\n\n", "\n"); - - let log_lines = last_10_log_lines - .split("CODE EXECUTION ---") - .last() - .unwrap_or(&logs); - format!("Error during execution of the script:\n{}", log_lines) - } - err @ _ => format!("error before termination: {err:#?}"), - }; - - let (_, output_map) = add_completed_job_error( - db, - &job, - logs, - error_message, - Some(metrics.clone()), - ) - .await?; - if job.is_flow_step { - update_flow_status_after_job_completion( - db, - &job, - false, - serde_json::Value::Object(output_map), - Some(metrics), - false, - same_worker_tx, - worker_dir, - worker_config.keep_job_dir, - &worker_config.base_internal_url, - ) - .await?; - } - } - }; - } - } - Ok(()) -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn write_file(dir: &str, path: &str, content: &str) -> Result { - let path = format!("{}/{}", dir, path); - let mut file = File::create(&path).await?; - file.write_all(content.as_bytes()).await?; - file.flush().await?; - Ok(file) -} - -#[async_recursion] -async fn transform_json_value( - token: &str, - workspace: &str, - base_url: &str, - v: Value, -) -> error::Result { - match v { - Value::String(y) if y.starts_with("$var:") => { - let path = y.strip_prefix("$var:").unwrap(); - let v = crate::client::get_variable(workspace, path, token, base_url).await?; - Ok(Value::String(v)) - } - Value::String(y) if y.starts_with("$res:") => { - let path = y.strip_prefix("$res:").unwrap(); - if path.split("/").count() < 2 { - return Err(Error::InternalErr( - format!("invalid resource path: {path}",), - )); - } - let v = crate::client::get_resource(workspace, path, token, base_url) - .await? - .ok_or_else(|| { - error::Error::InternalErr(format!("resource path: {path} not found",)) - })?; - transform_json_value(token, workspace, base_url, v).await - } - Value::Object(mut m) => { - for (a, b) in m.clone().into_iter() { - m.insert( - a, - transform_json_value(token, workspace, base_url, b).await?, - ); - } - Ok(Value::Object(m)) - } - a @ _ => Ok(a), - } -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_code_execution_job( - job: &QueuedJob, - db: &sqlx::Pool, - job_dir: &str, - worker_dir: &str, - logs: &mut String, - timeout: i32, - worker_config: &WorkerConfig, - envs: &Envs, -) -> error::Result { - let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview) - || (matches!(job.job_kind, JobKind::Script_Hub) && job.language == Some(ScriptLang::Deno)) - { - let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); - (code, None, job.language.to_owned()) - } else if matches!(job.job_kind, JobKind::Script_Hub) { - let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); - let script = get_hub_script( - job.script_path - .clone() - .unwrap_or_else(|| "missing script path".to_string()), - None, - &job.created_by, - ) - .await?; - (code, script.lockfile, job.language.to_owned()) - } else { - sqlx::query_as::<_, (String, Option, Option)>( - "SELECT content, lock, language FROM script WHERE hash = $1 AND (workspace_id = $2 OR \ - workspace_id = 'starter')", - ) - .bind(&job.script_hash.unwrap_or(ScriptHash(0)).0) - .bind(&job.workspace_id) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))? - }; - let worker_name = worker_dir.split("/").last().unwrap_or("unknown"); - let lang_str = job - .language - .as_ref() - .map(|x| format!("{x:?}")) - .unwrap_or_else(|| "NO_LANG".to_string()); - - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started {} job {}", - &lang_str, - job.id - ); - - let shared_mount = if job.same_worker { - format!( - r#" -mount {{ - src: "{worker_dir}/{}/shared" - dst: "/shared" - is_bind: true - rw: true -}} - "#, - job.parent_job.ok_or(Error::ExecutionErr( - "no parent job, required for same worker job".to_string() - ))?, - ) - } else { - "".to_string() - }; - let result = match language { - None => { - return Err(Error::ExecutionErr( - "Require language to be not null".to_string(), - ))?; - } - Some(ScriptLang::Python3) => { - handle_python_job( - worker_config, - envs, - requirements_o, - job_dir, - worker_dir, - worker_name, - job, - logs, - db, - timeout, - &inner_content, - &shared_mount, - ) - .await - } - Some(ScriptLang::Deno) => { - handle_deno_job( - worker_config, - envs, - logs, - job, - db, - job_dir, - &inner_content, - timeout, - &shared_mount, - ) - .await - } - Some(ScriptLang::Go) => { - handle_go_job( - worker_config, - envs, - logs, - job, - db, - &inner_content, - timeout, - job_dir, - requirements_o, - &shared_mount, - ) - .await - } - }; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - is_ok = result.is_ok(), - "finished {} job {}", - &lang_str, - job.id - ); - result -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_go_job( - WorkerConfig { base_internal_url, disable_nuser, disable_nsjail, base_url, .. }: &WorkerConfig, - Envs { nsjail_path, go_path, path_env, gopath_env, home_env, .. }: &Envs, - logs: &mut String, - job: &QueuedJob, - db: &sqlx::Pool, - inner_content: &str, - timeout: i32, - job_dir: &str, - requirements_o: Option, - shared_mount: &str, -) -> Result { - //go does not like executing modules at temp root - let job_dir = &format!("{job_dir}/go"); - if let Some(requirements) = requirements_o { - gen_go_mymod(inner_content, job_dir).await?; - let (md, sum) = requirements - .split_once(GO_REQ_SPLITTER) - .ok_or(Error::ExecutionErr( - "Invalid requirement file, missing splitter".to_string(), - ))?; - write_file(job_dir, "go.mod", md).await?; - write_file(job_dir, "go.sum", sum).await?; - } else { - logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n"); - set_logs(logs, job.id, db).await; - - install_go_dependencies( - &job.id, - inner_content, - logs, - job_dir, - db, - timeout, - go_path, - true, - ) - .await?; - } - - logs.push_str("\n\n--- GO CODE EXECUTION ---\n"); - set_logs(logs, job.id, db).await; - - let token = create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - "ephemeral-script", - timeout * 2, - &job.created_by, - ) - .await?; - create_args_and_out_file(job, &token, base_internal_url, job_dir).await?; - { - let sig = crate::parser_go::parse_go_sig(&inner_content)?; - drop(inner_content); - - const WRAPPER_CONTENT: &str = r#"package main - -import ( - "encoding/json" - "os" - "fmt" - "mymod/inner" -) - -func main() {{ - - dat, err := os.ReadFile("args.json") - if err != nil {{ - fmt.Println(err) - os.Exit(1) - }} - - var req inner.Req - - if err := json.Unmarshal(dat, &req); err != nil {{ - fmt.Println(err) - os.Exit(1) - }} - - res, err := inner.Run(req) - if err != nil {{ - fmt.Println(err) - os.Exit(1) - }} - res_json, err := json.Marshal(res) - if err != nil {{ - fmt.Println(err) - os.Exit(1) - }} - f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend) - if err != nil {{ - fmt.Println(err) - os.Exit(1) - }} - _, err = f.WriteString(string(res_json)) - if err != nil {{ - fmt.Println(err) - os.Exit(1) - }} -}}"#; - - write_file(job_dir, "main.go", WRAPPER_CONTENT).await?; - - { - let spread = &sig - .args - .clone() - .into_iter() - .map(|x| format!("req.{}", capitalize(&x.name))) - .join(", "); - let req_body = &sig - .args - .into_iter() - .map(|x| { - format!( - "{} {} `json:\"{}\"`", - capitalize(&x.name), - otyp_to_string(x.otyp), - x.name - ) - }) - .join("\n"); - let runner_content: String = format!( - r#"package inner -type Req struct {{ - {req_body} -}} - -func Run(req Req) (interface{{}}, error){{ - return main({spread}) -}} - -"#, - ); - write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?; - } - } - let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; - reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); - - let child = if !disable_nsjail { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_GO_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", GO_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()) - .replace("{SHARED_MOUNT}", shared_mount), - ) - .await?; - - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("GOMEMLIMIT", "2000MiB") - .args(vec![ - "--config", - "run.config.proto", - "--", - go_path, - "run", - "main.go", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - Command::new(go_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("GOPATH", gopath_env) - .env("HOME", home_env) - .args(vec!["run", "main.go"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - handle_child(&job.id, db, logs, timeout, child).await?; - read_result(job_dir).await -} - -fn capitalize(s: &str) -> String { - let mut c = s.chars(); - match c.next() { - None => String::new(), - Some(f) => f.to_uppercase().collect::() + c.as_str(), - } -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_deno_job( - WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, - Envs { nsjail_path, deno_path, path_env, .. }: &Envs, - logs: &mut String, - job: &QueuedJob, - db: &sqlx::Pool, - job_dir: &str, - inner_content: &String, - timeout: i32, - shared_mount: &str, -) -> error::Result { - logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); - set_logs(logs, job.id, db).await; - let _ = write_file(job_dir, "inner.ts", inner_content).await?; - let sig = trace_span!("parse_deno_signature") - .in_scope(|| crate::parser_ts::parse_deno_signature(inner_content))?; - let token = create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - "ephemeral-script", - timeout * 2, - &job.created_by, - ) - .await?; - create_args_and_out_file(job, &token, base_internal_url, job_dir).await?; - let spread = sig.args.into_iter().map(|x| x.name).join(","); - let wrapper_content: String = format!( - r#" -import {{ main }} from "./inner.ts"; - -const args = await Deno.readTextFile("args.json") - .then(JSON.parse) - .then(({{ {spread} }}) => [ {spread} ]) - -async function run() {{ - let res: any = await main(...args); - const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); - await Deno.writeTextFile("result.json", res_json); - Deno.exit(0); -}} -run(); -"#, - ); - write_file(job_dir, "main.ts", &wrapper_content).await?; - let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; - reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); - - let hostname_base = base_url.split("://").last().unwrap_or("localhost"); - let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost"); - let deno_auth_tokens = format!("{token}@{hostname_base};{token}@{hostname_internal}"); - let child = async { - Ok(if !disable_nsjail { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_DENO_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", DENO_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()) - .replace("{SHARED_MOUNT}", shared_mount), - ) - .await?; - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("DENO_AUTH_TOKENS", deno_auth_tokens) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec![ - "--config", - "run.config.proto", - "--", - deno_path, - "run", - "--unstable", - "--v8-flags=--max-heap-size=2048", - "-A", - "/tmp/main.ts", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - Command::new(deno_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("DENO_AUTH_TOKENS", deno_auth_tokens) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec![ - "run", - "--unstable", - "--v8-flags=--max-heap-size=2048", - "-A", - "main.ts", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }) as error::Result<_> - } - .instrument(trace_span!("create_deno_jail")) - .await?; - handle_child(&job.id, db, logs, timeout, child).await?; - read_result(job_dir).await -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn create_args_and_out_file( - job: &QueuedJob, - token: &String, - base_internal_url: &String, - job_dir: &str, -) -> Result<(), Error> { - let args = if let Some(args) = &job.args { - Some( - transform_json_value(token, &job.workspace_id, &base_internal_url, args.clone()) - .await?, - ) - } else { - None - }; - let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; - write_file(job_dir, "args.json", &ser_args).await?; - write_file(job_dir, "result.json", "").await?; - Ok(()) -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_python_job( - WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, - envs @ Envs { - nsjail_path, - python_path, - python_heavy_deps, - path_env, - pip_extra_index_url, - pip_index_url, - pip_trusted_host, - .. - }: &Envs, - requirements_o: Option, - job_dir: &str, - worker_dir: &str, - worker_name: &str, - job: &QueuedJob, - logs: &mut String, - db: &sqlx::Pool, - timeout: i32, - inner_content: &String, - shared_mount: &str, -) -> error::Result { - create_dependencies_dir(job_dir).await; - - let mut additional_python_paths: Vec = vec![]; - - let requirements = match requirements_o { - Some(r) => r, - None => { - let requirements = parser_py::parse_python_imports(&inner_content)?.join("\n"); - if requirements.is_empty() { - "".to_string() - } else { - pip_compile(job, &requirements, logs, job_dir, envs, db, timeout) - .await? - .map_err(|e| { - Error::ExecutionErr(format!("pip compile failed: {}", e.to_string())) - })? - } - } - }; - - if requirements.len() > 0 { - if !disable_nsjail { - let _ = write_file( - job_dir, - "download.config.proto", - &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{WORKER_DIR}", &worker_dir) - .replace("{CACHE_DIR}", PIP_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), - ) - .await?; - } - - let mut heavy_deps = DEFAULT_HEAVY_DEPS - .iter() - .map(|s| s.to_string()) - .collect::>(); - heavy_deps.extend(python_heavy_deps.into_iter().map(|s| s.to_string())); - - let (heavy, regular): (Vec<&str>, Vec<&str>) = requirements - .split("\n") - .partition(|d| heavy_deps.iter().any(|hd| d.starts_with(hd))); - - let _ = write_file(job_dir, "requirements.txt", ®ular.join("\n")).await?; - - let mut vars = vec![]; - if let Some(url) = pip_extra_index_url { - vars.push(("EXTRA_INDEX_URL", url)); - } - if let Some(url) = pip_index_url { - vars.push(("INDEX_URL", url)); - } - if let Some(host) = pip_trusted_host { - vars.push(("TRUSTED_HOST", host)); - } - - if heavy.len() > 0 { - logs.push_str(&format!( - "\nheavy deps detected, using supercache for: {heavy:?}" - )); - additional_python_paths = - handle_python_heavy_reqs(python_path, heavy, vars.clone(), job, logs, db, timeout) - .await?; - } - - if regular.len() > 0 { - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started setup python dependencies" - ); - - let child = if !disable_nsjail { - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - .envs(vars) - .args(vec!["--config", "download.config.proto"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - let mut args = vec![ - "-m", - "pip", - "install", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - "./dependencies", - "-r", - "./requirements.txt", - ]; - if let Some(url) = pip_extra_index_url { - args.extend(["--extra-index-url", url]); - } - if let Some(url) = pip_index_url { - args.extend(["--index-url", url]); - } - if let Some(host) = pip_trusted_host { - args.extend(["--trusted-host", host]); - } - Command::new(python_path) - .current_dir(job_dir) - .env_clear() - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - - logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n"); - let child = handle_child(&job.id, db, logs, timeout, child).await; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - is_ok = child.is_ok(), - "finished setting up python dependencies {}", - job.id - ); - child?; - } else { - logs.push_str("\nskipping pip install since not needed"); - }; - } - logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n"); - - set_logs(logs, job.id, db).await; - - let _ = write_file(job_dir, "inner.py", inner_content).await?; - - let sig = crate::parser_py::parse_python_signature(inner_content)?; - let transforms = sig - .args - .into_iter() - .map(|x| match x.typ { - Typ::Bytes => { - format!( - "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ - kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", - x.name, x.name, x.name, x.name - ) - } - Typ::Datetime => { - format!( - "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ - kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], \ - '%Y-%m-%dT%H:%M')\n", - x.name, x.name, x.name, x.name - ) - } - _ => "".to_string(), - }) - .collect::>() - .join(""); - - let token = create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - "ephemeral-script", - timeout * 2, - &job.created_by, - ) - .await?; - - create_args_and_out_file(job, &token, base_internal_url, job_dir).await?; - - let wrapper_content: String = format!( - r#" -import json -import base64 -from datetime import datetime - -inner_script = __import__("inner") - -with open("args.json") as f: - kwargs = json.load(f, strict=False) -for k, v in list(kwargs.items()): - if v == '': - del kwargs[k] -{transforms} -res = inner_script.main(**kwargs) -res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '') -with open("result.json", 'w') as f: - f.write(res_json) -"#, - ); - write_file(job_dir, "main.py", &wrapper_content).await?; - - let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; - let additional_python_paths_folders = additional_python_paths - .iter() - .map(|x| format!(":{x}")) - .join(""); - if !disable_nsjail { - let shared_deps = additional_python_paths - .into_iter() - .map(|pp| { - format!( - r#" -mount {{ - src: "{pp}" - dst: "{pp}" - is_bind: true - rw: false -}} - "# - ) - }) - .join("\n"); - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) - .replace( - "{ADDITIONAL_PYTHON_PATHS}", - additional_python_paths_folders.as_str(), - ), - ) - .await?; - } else { - reserved_variables.insert( - "PYTHONPATH".to_string(), - format!("{job_dir}/dependencies{additional_python_paths_folders}"), - ); - } - - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started python code execution {}", - job.id - ); - let child = if !disable_nsjail { - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - // inject PYTHONPATH here - for some reason I had to do it in nsjail conf - .envs(reserved_variables) - .env("PATH", path_env) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec![ - "--config", - "run.config.proto", - "--", - python_path, - "-u", - "/tmp/main.py", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - Command::new(python_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec!["-u", "main.py"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - - handle_child(&job.id, db, logs, timeout, child).await?; - read_result(job_dir).await -} - -async fn create_dependencies_dir(job_dir: &str) { - DirBuilder::new() - .recursive(true) - .create(&format!("{job_dir}/dependencies")) - .await - .expect("could not create dependencies dir"); -} - -async fn read_result(job_dir: &str) -> error::Result { - let mut file = File::open(format!("{job_dir}/result.json")).await?; - let mut content = "".to_string(); - file.read_to_string(&mut content).await?; - serde_json::from_str(&content) - .map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}"))) -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_dependency_job( - job: &QueuedJob, - logs: &mut String, - job_dir: &str, - db: &sqlx::Pool, - timeout: i32, - envs: &Envs, -) -> error::Result { - let content = match job.language { - Some(ScriptLang::Python3) => { - create_dependencies_dir(job_dir).await; - let requirements = &job - .raw_code - .as_ref() - .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))? - .clone(); - pip_compile(job, requirements, logs, job_dir, envs, db, timeout).await? - } - Some(ScriptLang::Go) => { - let requirements = job - .raw_code - .as_ref() - .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; - install_go_dependencies( - &job.id, - &requirements, - logs, - job_dir, - db, - timeout, - &envs.go_path, - false, - ) - .await - .map_err(|e| e.to_string()) - } - _ => Err("Language incompatible with dep job".to_string()), - }; - - match content { - Ok(content) => { - sqlx::query!( - "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", - &content, - &job.script_hash.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; - Ok(json!({ "success": "Successful lock file generation", "lock": content })) - } - Err(error) => { - sqlx::query!( - "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", - &format!("{logs}\n{error}"), - &job.script_hash.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; - Err(Error::ExecutionErr(format!("Error locking file: {error}")))? - } - } -} - -async fn pip_compile( - job: &QueuedJob, - requirements: &str, - logs: &mut String, - job_dir: &str, - Envs { pip_extra_index_url, pip_index_url, pip_trusted_host, .. }: &Envs, - db: &DB, - timeout: i32, -) -> Result, Error> { - logs.push_str(&format!("content of requirements:\n{}\n", requirements)); - let file = "requirements.in"; - write_file(job_dir, file, &requirements).await?; - let mut args = vec!["-q", "--no-header", file]; - if let Some(url) = pip_extra_index_url { - args.extend(["--extra-index-url", url]); - } - if let Some(url) = pip_index_url { - args.extend(["--index-url", url]); - } - if let Some(host) = pip_trusted_host { - args.extend(["--trusted-host", host]); - } - let child = Command::new("pip-compile") - .current_dir(job_dir) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - handle_child(&job.id, db, logs, timeout, child) - .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; - let path_lock = format!("{job_dir}/requirements.txt"); - let mut file = File::open(path_lock).await?; - let mut req_content = "".to_string(); - file.read_to_string(&mut req_content).await?; - Ok(Ok(req_content - .lines() - .filter(|x| !x.trim_start().starts_with('#')) - .map(|x| x.to_string()) - .collect::>() - .join("\n"))) -} - -async fn install_go_dependencies( - job_id: &Uuid, - code: &str, - logs: &mut String, - job_dir: &str, - db: &sqlx::Pool, - timeout: i32, - go_path: &str, - preview: bool, -) -> error::Result { - gen_go_mymod(code, job_dir).await?; - let child = Command::new("go") - .current_dir(job_dir) - .args(vec!["mod", "init", "mymod"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - - handle_child(job_id, db, logs, timeout, child).await?; - - let child = Command::new(go_path) - .current_dir(job_dir) - .env("GOMEMLIMIT", "2000MiB") - .args(vec!["mod", "tidy"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - handle_child(job_id, db, logs, timeout, child) - .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; - - if preview { - Ok(String::new()) - } else { - let mut req_content = "".to_string(); - - let mut file = File::open(format!("{job_dir}/go.mod")).await?; - file.read_to_string(&mut req_content).await?; - - req_content.push_str(&format!("\n{GO_REQ_SPLITTER}\n")); - - if let Ok(mut file) = File::open(format!("{job_dir}/go.sum")).await { - file.read_to_string(&mut req_content).await?; - } - - Ok(req_content) - } -} - -async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> { - let code = if code.trim_start().starts_with("package") { - code.to_string() - } else { - format!("package inner; {code}") - }; - - let mymod_dir = format!("{job_dir}/inner"); - DirBuilder::new() - .recursive(true) - .create(&mymod_dir) - .await - .expect("could not create go's mymod dir"); - - write_file(&mymod_dir, "inner_main.go", &code).await?; - - Ok(()) -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn get_reserved_variables( - job: &QueuedJob, - token: &str, - base_url: &str, - db: &sqlx::Pool, -) -> Result, Error> { - let flow_path = if let Some(uuid) = job.parent_job { - sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid) - .fetch_optional(db) - .await? - .flatten() - } else { - None - }; - let variables = variables::get_reserved_variables( - &job.workspace_id, - token, - &get_email_from_username(&job.created_by, db) - .await? - .unwrap_or_else(|| "nosuitable@email.xyz".to_string()), - &job.created_by, - &job.id.to_string(), - &job.permissioned_as, - base_url, - job.script_path.clone(), - job.parent_job.map(|x| x.to_string()), - flow_path, - job.schedule_path.clone(), - ); - Ok(variables - .into_iter() - .map(|rv| (rv.name, rv.value)) - .collect()) -} - -/// - wait until child exits and return with exit status -/// - read lines from stdout and stderr and append them to the "queue"."logs" -/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) -/// - update the `last_line` and `logs` strings with the program output -/// - update "queue"."last_ping" every five seconds -/// - kill process if we exceed timeout or "queue"."canceled" is set -#[tracing::instrument(level = "trace", skip_all)] -async fn handle_child( - job_id: &Uuid, - db: &DB, - logs: &mut String, - timeout: i32, - mut child: Child, -) -> error::Result<()> { - let timeout = Duration::from_secs(u64::try_from(timeout).expect("invalid timeout")); - let ping_interval = Duration::from_secs(5); - let cancel_check_interval = Duration::from_millis(500); - let write_logs_delay = Duration::from_millis(500); - - let (set_too_many_logs, mut too_many_logs) = watch::channel::(false); - - let output = child_joined_output_stream(&mut child); - let job_id = job_id.clone(); - - let (tx, mut rx) = mpsc::channel::<()>(1); - - /* the cancellation future is polled on by `wait_on_child` while - * waiting for the child to exit normally */ - let cancel_check = async { - let db = db.clone(); - - let mut interval = interval(cancel_check_interval); - interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - - loop { - tokio::select!( - _ = rx.recv() => break, - _ = interval.tick() => { - if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job_id) - .fetch_optional(&db) - .await - .map(|v| Some(true) == v) - .unwrap_or_else(|err| { - tracing::error!(%job_id, %err, "error checking cancelation for job {job_id}: {err}"); - false - }) - { - break; - } - }, - ); - } - }; - - #[derive(PartialEq, Debug)] - enum KillReason { - TooManyLogs, - Timeout, - Cancelled, - } - /* a future that completes when the child process exits */ - let wait_on_child = async { - let db = db.clone(); - - let kill_reason = tokio::select! { - biased; - result = child.wait() => return result.map(Ok), - Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs, - _ = cancel_check => KillReason::Cancelled, - _ = sleep(timeout) => KillReason::Timeout, - }; - tx.send(()).await.expect("rx should never be dropped"); - drop(tx); - - let set_reason = async { - if kill_reason == KillReason::Timeout { - if let Err(err) = sqlx::query( - r#" - UPDATE queue - SET canceled = true - , canceled_by = 'timeout', - , canceled_reason = $1 - WHERE id = $2 - r"#, - ) - .bind(format!("duration > {}", timeout.as_secs())) - .bind(job_id) - .execute(&db) - .await - { - tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}"); - } - } - }; - - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, child.kill()).await; - kill.map(|()| Err(kill_reason)) - }; - - /* a future that reads output from the child and appends to the database */ - let lines = async move { - /* log_remaining is zero when output limit was reached */ - let mut log_remaining = (MAX_LOG_SIZE as usize).saturating_sub(logs.chars().count()); - let mut result = io::Result::Ok(()); - let mut output = output; - /* `do_write` resolves the task, but does not contain the Result. - * It's useful to know if the task completed. */ - let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle(); - - while let Some(line) = output.by_ref().next().await { - let do_write_ = do_write.shared(); - - let mut read_lines = stream::once(async { line }) - .chain(output.by_ref()) - /* after receiving a line, continue until some delay has passed - * _and_ the previous database write is complete */ - .take_until(future::join(sleep(write_logs_delay), do_write_.clone())) - .boxed(); - - /* Read up until an error is encountered, - * handle log lines first and then the error... */ - let mut joined = String::new(); - - while let Some(line) = read_lines.next().await { - match line { - Ok(_) if log_remaining == 0 => (), - Ok(line) => { - append_with_limit(&mut joined, &line, &mut log_remaining); - - if log_remaining == 0 { - tracing::info!(%job_id, "Too many logs lines for job {job_id}"); - let _ = set_too_many_logs.send(true); - joined.push_str(&format!( - "Job logs or result reached character limit of {MAX_LOG_SIZE}; killing job." - )); - /* stop reading and drop our streams fairly quickly */ - break; - } - } - Err(err) => { - result = Err(err); - break; - } - } - } - - logs.push_str(&joined); - - /* Ensure the last flush completed before starting a new one. - * - * This shouldn't pause since `take_until()` reads lines until `do_write` - * resolves. We only stop reading lines before `take_until()` resolves if we reach - * EOF or a read error. In those cases, waiting on a database query to complete is - * fine because we're done. */ - - if let Some(Ok(p)) = do_write_ - .then(|()| write_result) - .await - .err() - .map(|err| err.try_into_panic()) - { - panic::resume_unwind(p); - } - - (do_write, write_result) = - tokio::spawn(append_logs(job_id, joined, db.clone())).remote_handle(); - - if let Err(err) = result { - tracing::error!(%job_id, %err, "error reading output for job {job_id}: {err}"); - break; - } - - if *set_too_many_logs.borrow() { - break; - } - } - - /* drop our end of the pipe */ - drop(output); - - if let Some(Ok(p)) = do_write - .then(|()| write_result) - .await - .err() - .map(|err| err.try_into_panic()) - { - panic::resume_unwind(p); - } - }.instrument(trace_span!("child_lines")); - - /* a stream updating "queue"."last_ping" at an interval */ - - let (kill_tx, mut kill_rx) = oneshot::channel::<()>(); - - let mut interval = interval(ping_interval); - interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - - let db1 = db.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = interval.tick() => { - if let Err(err) = sqlx::query!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id) - .execute(&db1) - .await - { - tracing::error!(%job_id, %err, "error setting last ping for job {job_id}: {err}"); - }; - }, - _ = (&mut kill_rx) => return, - } - } - }); - let (wait_result, _) = tokio::join!(wait_on_child, lines); - kill_tx.send(()).expect("send should always work"); - - match wait_result { - _ if *too_many_logs.borrow() => Err(Error::ExecutionErr( - "logs or result reached limit".to_string(), - )), - Ok(Ok(status)) => { - if status.success() { - Ok(()) - } else if let Some(code) = status.code() { - Err(error::Error::ExitStatus(code)) - } else { - Err(error::Error::ExecutionErr( - "process terminated by signal".to_string(), - )) - } - } - Ok(Err(kill_reason)) => Err(Error::ExecutionErr(format!( - "job process killed because {kill_reason:#?}" - ))), - Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))), - } -} - -/// takes stdout and stderr from Child, panics if either are not present -/// -/// builds a stream joining both stdout and stderr each read line by line -fn child_joined_output_stream( - child: &mut Child, -) -> impl stream::FusedStream> { - let stderr = child - .stderr - .take() - .expect("child did not have a handle to stdout"); - - let stdout = child - .stdout - .take() - .expect("child did not have a handle to stdout"); - - let stdout = BufReader::new(stdout).lines(); - let stderr = BufReader::new(stderr).lines(); - stream::select(lines_to_stream(stderr), lines_to_stream(stdout)) -} - -fn lines_to_stream( - mut lines: tokio::io::Lines, -) -> impl futures::Stream> { - stream::poll_fn(move |cx| { - std::pin::Pin::new(&mut lines) - .poll_next_line(cx) - .map(|result| result.transpose()) - }) -} - -// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields, -// so this pushes \n to thd destination string in each call -fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { - if *limit > 0 { - dst.push('\n'); - } - *limit -= 1; - - let src_len = src.chars().count(); - if src_len <= *limit { - dst.push_str(&src); - *limit -= src_len; - } else { - let byte_pos = src - .char_indices() - .skip(*limit) - .next() - .map(|(byte_pos, _)| byte_pos) - .unwrap_or(0); - dst.push_str(&src[0..byte_pos]); - *limit = 0; - } -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn set_logs(logs: &str, id: uuid::Uuid, db: &DB) { - if sqlx::query!( - "UPDATE queue SET logs = $1 WHERE id = $2", - logs.to_owned(), - id - ) - .execute(db) - .await - .is_err() - { - tracing::error!(%id, "error updating logs for id {id}") - }; -} - -/* TODO retry this? */ -#[tracing::instrument(level = "trace", skip_all)] -async fn append_logs(job_id: uuid::Uuid, logs: impl AsRef, db: impl Borrow) { - if logs.as_ref().is_empty() { - return; - } - - if let Err(err) = sqlx::query!( - "UPDATE queue SET logs = concat(logs, $1::text) WHERE id = $2", - logs.as_ref(), - job_id, - ) - .execute(db.borrow()) - .await - { - tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}"); - } -} - -pub async fn handle_zombie_jobs_periodically( - db: &DB, - timeout: i32, - mut rx: tokio::sync::broadcast::Receiver<()>, -) { - loop { - handle_zombie_jobs(db, timeout).await; - - tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(60)) => (), - _ = rx.recv() => { - println!("received killpill for monitor job"); - break; - } - } - } -} - -async fn handle_zombie_jobs(db: &DB, timeout: i32) { - let restarted = sqlx::query!( - "UPDATE queue SET running = false WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = false RETURNING id, workspace_id, last_ping", - (timeout * 5).to_string(), - JobKind::Flow: JobKind, - ) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); - - QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _); - for r in restarted { - tracing::info!( - "restarted zombie job {} {} {}", - r.id, - r.workspace_id, - r.last_ping - ); - } - - let timeouts = sqlx::query_as::<_, QueuedJob>( - "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = true", - ) - .bind((timeout * 5).to_string()) - .bind(JobKind::Flow) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); - - QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _); - for job in timeouts { - tracing::info!( - "timedouts zombie same_worker job {} {}", - job.id, - job.workspace_id, - ); - - // since the job is unrecoverable, the same worker queue should never be sent anything - let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1); - - let _ = handle_job_error( - db, - job, - error::Error::ExecutionErr("Same worker job timed out".to_string()), - None, - true, - same_worker_tx_never_used, - "", - true, - &std::env::var("BASE_INTERNAL_URL") - .unwrap_or_else(|_| "http://localhost:8000".to_string()), - ) - .await; - } -} - -async fn handle_python_heavy_reqs( - python_path: &String, - heavy_requirements: Vec<&str>, - vars: Vec<(&str, &String)>, - job: &QueuedJob, - logs: &mut String, - db: &sqlx::Pool, - timeout: i32, -) -> error::Result> { - let mut req_paths: Vec = vec![]; - for req in heavy_requirements { - // todo: handle many reqs - let venv_p = format!("{PIP_SUPERCACHE_DIR}/{req}"); - if metadata(&venv_p).await.is_ok() { - tracing::info!("already exists: {:?}", &venv_p); - req_paths.push(venv_p); - continue; - } - - logs.push_str("\n--- PIP SUPERCACHE INSTALL ---\n"); - logs.push_str(&format!("\nthe heavy dependency {req} is being installed for the first time.\nIt will take a bit longer but further execution will be much faster!")); - - logs.push_str("pip install\n"); - let child = Command::new(python_path) - .env_clear() - .envs(vars.clone()) - .args(vec![ - "-m", - "pip", - "install", - &req, - "-I", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - venv_p.as_str(), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - handle_child(&job.id, db, logs, timeout, child).await?; - - req_paths.push(venv_p); - } - Ok(req_paths) -} - -#[cfg(test)] -mod tests { - use futures::Stream; - use futures::StreamExt; - use serde_json::json; - use sqlx::{postgres::PgListener, query_scalar}; - use uuid::Uuid; - - use crate::{ - db::DB, - flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform}, - jobs::{push, JobPayload, RawCode}, - scripts::ScriptLang, - DEFAULT_SLEEP_QUEUE, - }; - - use super::*; - - async fn initialize_tracing() { - use std::sync::Once; - - static ONCE: Once = Once::new(); - ONCE.call_once(crate::tracing_init::initialize_tracing); - } - - /// it's important this is unique between tests as there is one prometheus registry and - /// run_worker shouldn't register the same metric with the same worker name more than once. - /// - /// this must fit in varchar(50) - fn next_worker_name() -> String { - use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; - - static ID: AtomicUsize = AtomicUsize::new(0); - - // n.b.: when tests are run with RUST_TEST_THREADS or --test-threads set to 1, the name - // will be "main"... The id provides uniqueness & thread_name gives context. - let id = ID.fetch_add(1, SeqCst); - let thread = std::thread::current(); - let thread_name = thread - .name() - .map(|s| { - s.len() - .checked_sub(39) - .and_then(|start| s.get(start..)) - .unwrap_or(s) - }) - .unwrap_or("no thread name"); - format!("{id}/{thread_name}") - } - - #[sqlx::test(fixtures("base"))] - async fn test_deno_flow(db: DB) { - initialize_tracing().await; - - let numbers = "export function main() { return [1, 2, 3]; }"; - let doubles = "export function main(n) { return n * 2; }"; - - let flow = { - FlowValue { - modules: vec![ - FlowModule { - id: "a".to_string(), - value: FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: numbers.to_string(), - path: None, - }, - input_transforms: Default::default(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }, - FlowModule { - id: "b".to_string(), - value: FlowModuleValue::ForloopFlow { - iterator: InputTransform::Javascript { expr: "result".to_string() }, - skip_failures: false, - modules: vec![FlowModule { - id: "c".to_string(), - value: FlowModuleValue::RawScript { - input_transforms: [( - "n".to_string(), - InputTransform::Javascript { - expr: "previous_result.iter.value".to_string(), - }, - )] - .into(), - language: ScriptLang::Deno, - content: doubles.to_string(), - path: None, - }, - input_transforms: Default::default(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }], - }, - input_transforms: Default::default(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }, - ], - same_worker: false, - ..Default::default() - } - }; - - let job = JobPayload::RawFlow { value: flow, path: None }; - - for i in 0..50 { - println!("deno flow iteration: {}", i); - let result = run_job_in_new_worker_until_complete(&db, job.clone(), None).await; - assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i); - } - } - - #[sqlx::test(fixtures("base"))] - async fn test_deno_flow_same_worker(db: DB) { - initialize_tracing().await; - - let write_file = r#"export async function main(loop: boolean, i: number, path: string) { - await Deno.writeTextFile(`/shared/${path}`, `${loop} ${i}`); - }"# - .to_string(); - - let flow = FlowValue { - modules: vec![ - FlowModule { - id: "a".to_string(), - value: FlowModuleValue::RawScript { - input_transforms: [ - ( - "loop".to_string(), - InputTransform::Static { value: json!(false) }, - ), - ("i".to_string(), InputTransform::Static { value: json!(1) }), - ( - "path".to_string(), - InputTransform::Static { value: json!("outer.txt") }, - ), - ] - .into(), - language: ScriptLang::Deno, - content: write_file.clone(), - path: None, - }, - input_transforms: Default::default(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }, - FlowModule { - id: "b".to_string(), - value: FlowModuleValue::ForloopFlow { - iterator: InputTransform::Static { value: json!([1, 2, 3]) }, - skip_failures: false, - modules: vec![ - FlowModule { - id: "d".to_string(), - input_transforms: [ - ( - "i".to_string(), - InputTransform::Javascript { - expr: "previous_result.iter.value".to_string(), - }, - ), - ( - "loop".to_string(), - InputTransform::Static { value: json!(true) }, - ), - ( - "path".to_string(), - InputTransform::Static { value: json!("inner.txt") }, - ), - ] - .into(), - value: FlowModuleValue::RawScript { - input_transforms: [].into(), - language: ScriptLang::Deno, - content: write_file, - path: None, - }, - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }, - FlowModule { - id: "e".to_string(), - value: FlowModuleValue::RawScript { - input_transforms: [( - "path".to_string(), - InputTransform::Static { value: json!("inner.txt") }, - ), ( - "path2".to_string(), - InputTransform::Static { value: json!("outer.txt") }, - )] - .into(), - language: ScriptLang::Deno, - content: r#"export async function main(path: string, path2: string) { - return await Deno.readTextFile(`/shared/${path}`) + "," + await Deno.readTextFile(`/shared/${path2}`); - }"# - .to_string(), - path: None, - }, - input_transforms: [].into(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }, - ], - }, - input_transforms: Default::default(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - - }, - FlowModule { - id: "c".to_string(), - value: FlowModuleValue::RawScript { - input_transforms: [ - ( - "loops".to_string(), - InputTransform::Javascript { expr: "previous_result".to_string() }, - ), - ( - "path".to_string(), - InputTransform::Static { value: json!("outer.txt") }, - ), - ( - "path2".to_string(), - InputTransform::Static { value: json!("inner.txt") }, - ), - ] - .into(), - language: ScriptLang::Deno, - content: r#"export async function main(path: string, loops: string[], path2: string) { - return await Deno.readTextFile(`/shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`/shared/${path2}`); - }"# - .to_string(), - path: None, - }, - input_transforms: [].into(), - stop_after_if: Default::default(), - summary: Default::default(), - suspend: Default::default(), - retry: None, - sleep: None, - }, - ], - same_worker: true, - ..Default::default() - }; - - let job = JobPayload::RawFlow { value: flow, path: None }; - - let result = run_job_in_new_worker_until_complete(&db, job.clone(), None).await; - assert_eq!( - result, - serde_json::json!("false 1,true 1,false 1,true 2,false 1,true 3,false 1,true 3") - ); - } - - #[sqlx::test(fixtures("base"))] - async fn test_flow_result_by_id(db: DB) { - initialize_tracing().await; - - let server = ApiServer::start(db.clone()).await; - let port = server.addr.port(); - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "id": "a", - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return 42 }", - } - }, - { - "value": { - "branches": [ - { - "modules": [{ - "value": { - "branches": [{"modules": [ { - "id": "d", - "value": { - "input_transforms": {"v": {"type": "javascript", "expr": "result_by_id(\"a\")"}}, - "type": "rawscript", - "language": "deno", - "content": "export function main(v){ return v }", - } - - },]}], - "type": "branchall", - } - }], - }], - "type": "branchall", - }, - } - ], - })) - .unwrap(); - - let job = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, job.clone(), Some(port)).await; - assert_eq!(result, serde_json::json!([[42]])); - } - #[sqlx::test(fixtures("base"))] - async fn test_stop_after_if(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [ - { - "input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(n): return n", - }, - "stop_after_if": { - "expr": "result < 0", - "skip_if_stopped": false, - }, - }, - { - "input_transforms": { "n": { "type": "javascript", "expr": "previous_result" } }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(n): return f'last step saw {n}'", - }, - }, - ], - })) - .unwrap(); - let job = JobPayload::RawFlow { value: flow, path: None }; - - let result = RunJob::from(job.clone()) - .arg("n", json!(123)) - .run_until_complete(&db, None) - .await; - assert_eq!(json!("last step saw 123"), result); - - let result = RunJob::from(job.clone()) - .arg("n", json!(-123)) - .run_until_complete(&db, None) - .await; - assert_eq!(json!(-123), result); - } - - #[sqlx::test(fixtures("base"))] - async fn test_python_flow(db: DB) { - initialize_tracing().await; - - let numbers = "def main(): return [1, 2, 3]"; - let doubles = "def main(n): return n * 2"; - - let flow: FlowValue = serde_json::from_value(serde_json::json!( { - "input_transform": {}, - "modules": [ - { - "value": { - "type": "rawscript", - "language": "python3", - "content": numbers, - }, - }, - { - "value": { - "type": "forloopflow", - "iterator": { "type": "javascript", "expr": "result" }, - "skip_failures": false, - "modules": [{ - "value": { - "type": "rawscript", - "language": "python3", - "content": doubles, - }, - "input_transform": { - "n": { - "type": "javascript", - "expr": "previous_result.iter.value", - }, - }, - }], - }, - }, - ], - })) - .unwrap(); - - for i in 0..50 { - println!("python flow iteration: {}", i); - let result = run_job_in_new_worker_until_complete( - &db, - JobPayload::RawFlow { value: flow.clone(), path: None }, - None, - ) - .await; - - assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}"); - } - } - - #[sqlx::test(fixtures("base"))] - async fn test_python_flow_2(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "content": "import wmill\ndef main(): return \"Hello\"", - "language": "python3" - }, - "input_transform": {} - } - ] - })) - .unwrap(); - - for i in 0..10 { - println!("python flow iteration: {}", i); - let result = run_job_in_new_worker_until_complete( - &db, - JobPayload::RawFlow { value: flow.clone(), path: None }, - None, - ) - .await; - - assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}"); - } - } - - #[sqlx::test(fixtures("base"))] - async fn test_go_job(db: DB) { - initialize_tracing().await; - - let content = r#" -import "fmt" - -func main(derp string) (string, error) { - fmt.Println("Hello, 世界") - return fmt.Sprintf("hello %s", derp), nil -} - "# - .to_owned(); - - let result = RunJob::from(JobPayload::Code(RawCode { - content, - path: None, - language: ScriptLang::Go, - })) - .arg("derp", json!("world")) - .run_until_complete(&db, None) - .await; - - assert_eq!(result, serde_json::json!("hello world")); - } - - #[sqlx::test(fixtures("base"))] - async fn test_python_job(db: DB) { - initialize_tracing().await; - - let content = r#" -def main(): - return "hello world" - "# - .to_owned(); - - let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 }); - - let result = run_job_in_new_worker_until_complete(&db, job, None).await; - - assert_eq!(result, serde_json::json!("hello world")); - } - - #[sqlx::test(fixtures("base"))] - async fn test_python_job_heavy_dep(db: DB) { - initialize_tracing().await; - - let content = r#" -import numpy as np - -def main(): - a = np.arange(15).reshape(3, 5) - return len(a) - "# - .to_owned(); - - let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 }); - - let result = run_job_in_new_worker_until_complete(&db, job, None).await; - - assert_eq!(result, serde_json::json!(3)); - } - - #[sqlx::test(fixtures("base"))] - async fn test_python_job_with_imports(db: DB) { - initialize_tracing().await; - - let content = r#" -import wmill - -def main(): - return wmill.get_workspace() - "# - .to_owned(); - - let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 }); - - let result = run_job_in_new_worker_until_complete(&db, job, None).await; - - assert_eq!(result, serde_json::json!("test-workspace")); - } - - #[sqlx::test(fixtures("base"))] - async fn test_empty_loop(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [ - { - "value": { - "type": "forloopflow", - "iterator": { "type": "static", "value": [] }, - "modules": [ - { - "input_transform": { - "n": { - "type": "javascript", - "expr": "previous_result.iter.value", - }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(n): return n", - }, - } - ], - }, - }, - { - "input_transform": { - "items": { - "type": "javascript", - "expr": "previous_result", - }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(items): return sum(items)", - }, - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!(result, serde_json::json!(0)); - } - - #[sqlx::test(fixtures("base"))] - async fn test_empty_loop_2(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [ - { - "value": { - "type": "forloopflow", - "iterator": { "type": "static", "value": [] }, - "modules": [ - { - "input_transform": { - "n": { - "type": "javascript", - "expr": "previous_result.iter.value", - }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(n): return n", - }, - } - ], - }, - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!(result, serde_json::json!([])); - } - - #[sqlx::test(fixtures("base"))] - async fn test_step_after_loop(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [ - { - "value": { - "type": "forloopflow", - "iterator": { "type": "static", "value": [2,3,4] }, - "modules": [ - { - "input_transform": { - "n": { - "type": "javascript", - "expr": "previous_result.iter.value", - }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(n): return n", - } , - } - ], - }, - }, - { - "input_transform": { - "items": { - "type": "javascript", - "expr": "previous_result", - }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(items): return sum(items)", - }, - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!(result, serde_json::json!(9)); - } - - fn module_add_item_to_list(i: i32) -> serde_json::Value { - json!({ - "input_transform": { - "array": { - "type": "javascript", - "expr": "previous_result", - }, - "i": { - "type": "static", - "value": json!(i), - } - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(array, i){ array.push(i); return array }", - } - }) - } - - fn module_failure() -> serde_json::Value { - json!({ - "input_transform": {}, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ throw Error('failure') }", - } - }) - } - - #[sqlx::test(fixtures("base"))] - async fn test_branchone_simple(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return [1] }", - } - }, - { - "value": { - "branches": [], - "default": [module_add_item_to_list(2)], - "type": "branchone", - } - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!(result, serde_json::json!([1, 2])); - } - - #[sqlx::test(fixtures("base"))] - async fn test_branchall_simple(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return [1] }", - } - }, - { - "value": { - "branches": [ - {"modules": [module_add_item_to_list(2)]}, - {"modules": [module_add_item_to_list(3)]}], - "type": "branchall", - } - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!(result, serde_json::json!([[1, 2], [1, 3]])); - } - - #[sqlx::test(fixtures("base"))] - async fn test_branchall_skip_failure(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return [1] }", - } - }, - { - "value": { - "branches": [ - {"modules": [module_failure()], "skip_failure": false}, - {"modules": [module_add_item_to_list(3)]}], - "type": "branchall", - } - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!( - result, - serde_json::json!({"error": "Error during execution of the script:\n\nerror: Uncaught (in promise) Error: failure\nexport function main(){ throw Error('failure') }\n ^\n at main (file:///tmp/inner.ts:1:31)\n at run (file:///tmp/main.ts:9:26)\n at file:///tmp/main.ts:14:1"}) - ); - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return [1] }", - } - }, - { - "value": { - "branches": [ - {"modules": [module_failure()], "skip_failure": true}, - {"modules": [module_add_item_to_list(2)]} - ], - "type": "branchall", - } - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!( - result, - serde_json::json!([{"error": "Error during execution of the script:\n\nerror: Uncaught (in promise) Error: failure\nexport function main(){ throw Error('failure') }\n ^\n at main (file:///tmp/inner.ts:1:31)\n at run (file:///tmp/main.ts:9:26)\n at file:///tmp/main.ts:14:1"}, [1, 2]]) - ); - } - - #[sqlx::test(fixtures("base"))] - async fn test_branchone_nested(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return [] }", - } - }, - module_add_item_to_list(1), - { - "value": { - "branches": [ - { - "expr": "false", - "modules": [] - }, - { - "expr": "true", - "modules": [ { - "value": { - "branches": [ - { - "expr": "false", - "modules": [] - }], - "default": [module_add_item_to_list(2)], - "type": "branchone", - } - }] - }, - ], - "default": [module_add_item_to_list(-4)], - "type": "branchone", - } - }, - module_add_item_to_list(3), - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!(result, serde_json::json!([1, 2, 3])); - } - - #[sqlx::test(fixtures("base"))] - async fn test_branchall_nested(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(json!({ - "modules": [ - { - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(){ return [1] }", - } - }, - { - "value": { - "branches": [ - { - "modules": [ { - "value": { - "branches": [ - {"modules": [module_add_item_to_list(2)]}, - {"modules": [module_add_item_to_list(3)]}], - "type": "branchall", - } - }, { - "value": { - "branches": [ - {"modules": [module_add_item_to_list(4)]}, - {"modules": [module_add_item_to_list(5)]}], - "type": "branchall", - } - } - ] - }, - {"modules": [module_add_item_to_list(6)]}], - "type": "branchall", - } - }, - ], - })) - .unwrap(); - - let flow = JobPayload::RawFlow { value: flow, path: None }; - let result = run_job_in_new_worker_until_complete(&db, flow, None).await; - - assert_eq!( - result, - serde_json::json!([[[[1, 2], [1, 3], 4], [[1, 2], [1, 3], 5]], [1, 6]]) - ); - } - - #[sqlx::test(fixtures("base"))] - async fn test_failure_module(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [{ - "input_transform": { - "l": { "type": "javascript", "expr": "[]", }, - "n": { "type": "javascript", "expr": "flow_input.n", }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(n, l) { if (n == 0) throw l; return { l: [...l, 0] } }", - }, - }, { - "input_transform": { - "l": { "type": "javascript", "expr": "previous_result.l", }, - "n": { "type": "javascript", "expr": "flow_input.n", }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(n, l) { if (n == 1) throw l; return { l: [...l, 1] } }", - }, - }, { - "input_transform": { - "l": { "type": "javascript", "expr": "previous_result.l", }, - "n": { "type": "javascript", "expr": "flow_input.n", }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(n, l) { if (n == 2) throw l; return { l: [...l, 2] } }", - }, - }], - "failure_module": { - "input_transform": { "error": { "type": "javascript", "expr": "previous_result", } }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(error) { return { 'from failure module': error } }", - } - }, - })) - .unwrap(); - - let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) - .arg("n", json!(0)) - .run_until_complete(&db, None) - .await; - assert!(result["from failure module"]["error"] - .as_str() - .unwrap() - .contains("Uncaught (in promise) []")); - - let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) - .arg("n", json!(1)) - .run_until_complete(&db, None) - .await; - assert!(result["from failure module"]["error"] - .as_str() - .unwrap() - .contains("Uncaught (in promise) [ 0 ]")); - - let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) - .arg("n", json!(2)) - .run_until_complete(&db, None) - .await; - assert!(result["from failure module"]["error"] - .as_str() - .unwrap() - .contains("Uncaught (in promise) [ 0, 1 ]")); - - let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) - .arg("n", json!(3)) - .run_until_complete(&db, None) - .await; - assert_eq!(json!({ "l": [0, 1, 2] }), result); - } - - pub struct ApiServer { - pub addr: std::net::SocketAddr, - tx: tokio::sync::broadcast::Sender<()>, - task: tokio::task::JoinHandle>, - } - - impl ApiServer { - pub async fn start(db: DB) -> Self { - let (tx, rx) = tokio::sync::broadcast::channel::<()>(1); - - let sock = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - - let addr = sock.local_addr().unwrap(); - drop(sock); - - let task = tokio::task::spawn({ - crate::run_server( - db.clone(), - addr, - format!("http://localhost:{}", addr.port()), - rx, - ) - }); - - return Self { addr, tx, task }; - } - - async fn close(self) -> anyhow::Result<()> { - let Self { tx, task, .. } = self; - drop(tx); - task.await.unwrap() - } - } - - mod suspend_resume { - - use super::*; - - async fn wait_until_flow_suspends( - flow: Uuid, - mut queue: impl Stream + Unpin, - db: &DB, - ) { - loop { - queue.by_ref().find(&flow).await.unwrap(); - if query_scalar("SELECT suspend > 0 FROM queue WHERE id = $1") - .bind(flow) - .fetch_one(db) - .await - .unwrap() - { - break; - } - } - } - - async fn _print_job(id: Uuid, db: &DB) -> Result<(), anyhow::Error> { - tracing::info!( - "{:#?}", - crate::jobs::get_job_by_id(db.begin().await?, "test-workspace", id) - .await? - .0 - ); - Ok(()) - } - - fn flow() -> FlowValue { - serde_json::from_value(serde_json::json!({ - "modules": [{ - "input_transform": { - "n": { "type": "javascript", "expr": "flow_input.n", }, - "port": { "type": "javascript", "expr": "flow_input.port", }, - "op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "\ - export async function main(n, port, op) {\ - const job = Deno.env.get('WM_JOB_ID'); - const token = Deno.env.get('WM_TOKEN'); - const r = await fetch( - `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ - {\ - method: 'GET',\ - headers: { 'Authorization': `Bearer ${token}` }\ - }\ - );\ - console.log(r);\ - const secret = await r.text();\ - console.log('Secret: ' + secret + ' ' + job + ' ' + token);\ - const r2 = await fetch( - `http://localhost:${port}/api/w/test-workspace/jobs/${op}/${job}/0/${secret}?approver=ruben`,\ - {\ - method: 'POST',\ - body: JSON.stringify('from job'),\ - headers: { 'content-type': 'application/json' }\ - }\ - );\ - console.log(await r2.text());\ - return n + 1;\ - }", - }, - "suspend": { - "required_events": 1 - }, - }, { - "input_transform": { - "n": { "type": "javascript", "expr": "previous_result", }, - "resume": { "type": "javascript", "expr": "resume", }, - "resumes": { "type": "javascript", "expr": "resumes", }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }" - }, - "suspend": { - "required_events": 1 - }, - }, { - "input_transform": { - "last": { "type": "javascript", "expr": "previous_result", }, - "resume": { "type": "javascript", "expr": "resume", }, - "resumes": { "type": "javascript", "expr": "resumes", }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": "export function main(last, resume, resumes) { return { last, resume, resumes } }" - }, - }], - })) - .unwrap() - } - - #[sqlx::test(fixtures("base"))] - async fn test(db: DB) { - initialize_tracing().await; - - let server = ApiServer::start(db.clone()).await; - let port = server.addr.port(); - - let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None }) - .arg("n", json!(1)) - .arg("port", json!(port)) - .push(&db) - .await; - - let mut completed = listen_for_completed_jobs(&db).await; - let queue = listen_for_queue(&db).await; - let db_ = db.clone(); - - in_test_worker(&db, async move { - let db = db_; - - wait_until_flow_suspends(flow, queue, &db).await; - // print_job(flow, &db).await; - /* The first job resumes itself. */ - let _first = completed.next().await.unwrap(); - // print_job(_first, &db).await; - - /* ... and send a request resume it. */ - let second = completed.next().await.unwrap(); - // print_job(second, &db).await; - - let token = create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "").await.unwrap(); - let secret = reqwest::get(format!( - "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben" - )) - .await - .unwrap() - .error_for_status() - .unwrap() - .text().await.unwrap(); - println!("{}", secret); - - /* ImZyb20gdGVzdCIK = base64 "from test" */ - reqwest::get(format!( - "http://localhost:{port}/api/w/test-workspace/jobs/resume/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK&approver=ruben" - )) - .await - .unwrap() - .error_for_status() - .unwrap(); - - completed.find(&flow).await.unwrap(); - }, None) - .await; - - server.close().await.unwrap(); - - let result = completed_job_result(flow, &db).await; - - assert_eq!( - json!({ - "last": { - "resume": "from job", - "resumes": ["from job"], - "n": 3, - }, - "resume": "from test", - "resumes": ["from test"], - }), - result - ); - - // ensure resumes are cleaned up through CASCADE when the flow is finished - assert_eq!( - 0, - query_scalar::<_, i64>("SELECT count(*) FROM resume_job") - .fetch_one(&db) - .await - .unwrap() - ); - } - - #[sqlx::test(fixtures("base"))] - async fn cancel_from_job(db: DB) { - initialize_tracing().await; - - let server = ApiServer::start(db.clone()).await; - let port = server.addr.port(); - - let result = RunJob::from(JobPayload::RawFlow { value: flow(), path: None }) - .arg("n", json!(1)) - .arg("op", json!("cancel")) - .arg("port", json!(port)) - .run_until_complete(&db, None) - .await; - - server.close().await.unwrap(); - - assert_eq!( - json!({"error": "Job canceled: approval request disapproved by ruben" }), - result - ); - } - - #[sqlx::test(fixtures("base"))] - async fn cancel_after_suspend(db: DB) { - initialize_tracing().await; - - let server = ApiServer::start(db.clone()).await; - let port = server.addr.port(); - - let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None }) - .arg("n", json!(1)) - .arg("port", json!(port)) - .push(&db) - .await; - - let mut completed = listen_for_completed_jobs(&db).await; - let queue = listen_for_queue(&db).await; - let db_ = db.clone(); - - in_test_worker(&db, async move { - let db = db_; - - wait_until_flow_suspends(flow, queue, &db).await; - /* The first job resumes itself. */ - let _first = completed.next().await.unwrap(); - /* ... and send a request resume it. */ - let second = completed.next().await.unwrap(); - - let token = create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "").await.unwrap(); - let secret = reqwest::get(format!( - "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}" - )) - .await - .unwrap() - .error_for_status() - .unwrap() - .text().await.unwrap(); - println!("{}", secret); - - /* ImZyb20gdGVzdCIK = base64 "from test" */ - reqwest::get(format!( - "http://localhost:{port}/api/w/test-workspace/jobs/cancel/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK" - )) - .await - .unwrap() - .error_for_status() - .unwrap(); - - completed.find(&flow).await.unwrap(); - }, None) - .await; - - server.close().await.unwrap(); - - let result = completed_job_result(flow, &db).await; - - assert_eq!( - json!({"error": "Job canceled: approval request disapproved by unknown" }), - result - ); - } - } - - mod retry { - use super::*; - - /// test helper provides some external state to help steps fail at specific points - struct Server { - addr: std::net::SocketAddr, - tx: tokio::sync::oneshot::Sender<()>, - task: tokio::task::JoinHandle>, - } - - impl Server { - async fn start(responses: Vec>) -> Self { - use tokio::net::TcpListener; - - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - let sock = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = sock.local_addr().unwrap(); - - let task = tokio::task::spawn(async move { - tokio::pin!(rx); - let mut results = vec![]; - - for next in responses { - let (mut peer, _) = tokio::select! { - _ = &mut rx => break, - r = sock.accept() => r, - } - .unwrap(); - - let n = peer.read_u8().await.unwrap(); - results.push(n); - - if let Some(next) = next { - peer.write_u8(next).await.unwrap(); - } - } - - results - }); - - return Self { addr, tx, task }; - } - - async fn close(self) -> Vec { - let Self { task, tx, .. } = self; - drop(tx); - task.await.unwrap() - } - } - - fn inner_step() -> &'static str { - r#" -export async function main(index, port) { - const buf = new Uint8Array([0]); - const sock = await Deno.connect({ port }); - await sock.write(new Uint8Array([index])); - if (await sock.read(buf) != 1) throw "read"; - return buf[0]; -} - "# - } - - fn last_step() -> &'static str { - r#" -def main(last, port): - with __import__("socket").create_connection((None, port)) as sock: - sock.send(b'\xff') - return last + [sock.recv(1)[0]] -"# - } - - fn flow_forloop_retry() -> FlowValue { - serde_json::from_value(serde_json::json!({ - "modules": [{ - "value": { - "type": "forloopflow", - "iterator": { "type": "javascript", "expr": "result.items" }, - "skip_failures": false, - "modules": [{ - "input_transform": { - "index": { "type": "javascript", "expr": "previous_result.iter.index" }, - "port": { "type": "javascript", "expr": "flow_input.port" }, - }, - "value": { - "type": "rawscript", - "language": "deno", - "content": inner_step(), - }, - }], - }, - "retry": { "constant": { "attempts": 2, "seconds": 0 } }, - }, { - "input_transform": { - "last": { "type": "javascript", "expr": "previous_result" }, - "port": { "type": "javascript", "expr": "flow_input.port" }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": last_step(), - }, - "retry": { "constant": { "attempts": 2, "seconds": 0 } }, - }], - })).unwrap() - } - - #[sqlx::test(fixtures("base"))] - async fn test_pass(db: DB) { - initialize_tracing().await; - - /* fails twice in the loop, then once on the last step - * retry attempts is measured per-step, so it _retries_ at most two times on each step, - * which means it may run the step three times in total */ - - let (attempts, responses) = [ - /* pass fail */ - (0, Some(99)), - (1, None), - /* pass pass fail */ - (0, Some(99)), - (1, Some(99)), - (2, None), - /* pass pass pass */ - (0, Some(3)), - (1, Some(5)), - (2, Some(7)), - /* fail the last step once */ - (0xff, None), - (0xff, Some(9)), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - let server = Server::start(responses).await; - let result = - RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None }) - .arg("items", json!(["unused", "unused", "unused"])) - .arg("port", json!(server.addr.port())) - .run_until_complete(&db, None) - .await; - - assert_eq!(server.close().await, attempts); - assert_eq!(json!([3, 5, 7, 9]), result); - } - - #[sqlx::test(fixtures("base"))] - async fn test_fail_step_zero(db: DB) { - initialize_tracing().await; - - /* attempt and fail the first step three times and stop */ - let (attempts, responses) = [ - /* pass fail x3 */ - (0, Some(99)), - (1, None), - (0, Some(99)), - (1, None), - (0, Some(99)), - (1, None), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - let server = Server::start(responses).await; - let result = - RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None }) - .arg("items", json!(["unused", "unused", "unused"])) - .arg("port", json!(server.addr.port())) - .run_until_complete(&db, None) - .await; - - assert_eq!(server.close().await, attempts); - assert!(result["error"] - .as_str() - .unwrap() - .contains(r#"Uncaught (in promise) "read""#)); - } - - #[sqlx::test(fixtures("base"))] - async fn test_fail_step_one(db: DB) { - initialize_tracing().await; - - /* attempt and fail the first step three times and stop */ - let (attempts, responses) = [ - /* fail once, then pass */ - (0, None), - (0, Some(1)), - (1, Some(2)), - (2, Some(3)), - /* fail three times */ - (0xff, None), - (0xff, None), - (0xff, None), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - let server = Server::start(responses).await; - let result = - RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None }) - .arg("items", json!(["unused", "unused", "unused"])) - .arg("port", json!(server.addr.port())) - .run_until_complete(&db, None) - .await; - - assert_eq!(server.close().await, attempts); - assert!(result["error"] - .as_str() - .unwrap() - .contains("index out of range")); - } - - #[sqlx::test(fixtures("base"))] - async fn test_with_failure_module(db: DB) { - let value = serde_json::from_value(json!({ - "modules": [{ - "input_transform": { "port": { "type": "javascript", "expr": "flow_input.port" } }, - "value": { - "type": "rawscript", - "language": "python3", - "content": r#" -def main(port): - with __import__("socket").create_connection((None, port)) as sock: - sock.send(b'\x00') - return sock.recv(1)[0]"#, - }, - "retry": { "constant": { "attempts": 1, "seconds": 0 } }, - }], - "failure_module": { - "input_transform": { "error": { "type": "javascript", "expr": "previous_result", }, - "port": { "type": "javascript", "expr": "flow_input.port" } }, - "value": { - "type": "rawscript", - "language": "python3", - "content": r#" -def main(error, port): - with __import__("socket").create_connection((None, port)) as sock: - sock.send(b'\xff') - return { "recv": sock.recv(1)[0], "from failure module": error }"#, - }, - "retry": { "constant": { "attempts": 1, "seconds": 0 } }, - }, - })) - .unwrap(); - let (attempts, responses) = [ - /* fail the first step twice */ - (0x00, None), - (0x00, None), - /* and the failure module once */ - (0xff, None), - (0xff, Some(42)), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - let server = Server::start(responses).await; - let result = RunJob::from(JobPayload::RawFlow { value, path: None }) - .arg("port", json!(server.addr.port())) - .run_until_complete(&db, None) - .await; - - assert_eq!(server.close().await, attempts); - assert_eq!( - result, - json!({ - "recv": 42, - "from failure module": { - "error": "Error during execution of the script:\n\nTraceback (most recent call last):\n File \"/tmp/main.py\", line 14, in \n res = inner_script.main(**kwargs)\n File \"/tmp/inner.py\", line 5, in main\n return sock.recv(1)[0]\nIndexError: index out of range", - } - }) - ); - } - } - - #[sqlx::test(fixtures("base"))] - async fn test_iteration(db: DB) { - initialize_tracing().await; - - let flow: FlowValue = serde_json::from_value(serde_json::json!({ - "modules": [{ - "value": { - "type": "forloopflow", - "iterator": { "type": "javascript", "expr": "result.items" }, - "skip_failures": false, - "modules": [{ - "input_transform": { - "n": { - "type": "javascript", - "expr": "previous_result.iter.value", - }, - }, - "value": { - "type": "rawscript", - "language": "python3", - "content": "def main(n):\n if 1 < n:\n raise StopIteration(n)", - }, - }], - }, - }], - })) - .unwrap(); - - let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) - .arg("items", json!([])) - .run_until_complete(&db, None) - .await; - assert_eq!(result, serde_json::json!([])); - - /* Don't actually test that this does 257 jobs or that will take forever. */ - let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) - .arg("items", json!((0..257).collect::>())) - .run_until_complete(&db, None) - .await; - assert!(matches!(result, Value::Object(_))); - assert!(result["error"] - .as_str() - .unwrap() - .contains("StopIteration: 2")); - } - - struct RunJob { - payload: JobPayload, - args: Map, - } - - impl From for RunJob { - fn from(payload: JobPayload) -> Self { - Self { payload, args: Default::default() } - } - } - - impl RunJob { - fn arg>(mut self, k: S, v: serde_json::Value) -> Self { - self.args.insert(k.into(), v); - self - } - - async fn push(self, db: &DB) -> Uuid { - let RunJob { payload, args } = self; - let tx = db.begin().await.unwrap(); - let (uuid, tx) = push( - tx, - "test-workspace", - payload, - Some(args), - /* user */ "test-user", - /* permissioned_as */ "u/admin".to_string(), - /* scheduled_for_o */ None, - /* schedule_path */ None, - /* parent_job */ None, - /* is_flow_step */ false, - /* running */ false, - ) - .await - .unwrap(); - - tx.commit().await.unwrap(); - - uuid - } - - /// push the job, spawn a worker, wait until the job is in completed_job - async fn run_until_complete(self, db: &DB, port: Option) -> serde_json::Value { - let uuid = self.push(db).await; - let listener = listen_for_completed_jobs(db).await; - in_test_worker(db, listener.find(&uuid), port).await; - completed_job_result(uuid, db).await - } - } - - async fn run_job_in_new_worker_until_complete( - db: &DB, - job: JobPayload, - port: Option, - ) -> serde_json::Value { - RunJob::from(job).run_until_complete(db, port).await - } - - /// Start a worker with a timeout and run a future, until the worker quits or we time out. - /// - /// Cleans up the worker before resolving. - async fn in_test_worker( - db: &DB, - inner: Fut, - port: Option, - ) -> ::Output { - let (quit, worker) = spawn_test_worker(db, port); - let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker); - tokio::pin!(worker); - - let res = tokio::select! { - biased; - res = inner => res, - res = &mut worker => match - res.expect("worker timed out") - .expect("worker panicked") { - _ => panic!("worker quit early"), - }, - }; - - /* ensure the worker quits before we return */ - drop(quit); - - let _: () = worker - .await - .expect("worker timed out") - .expect("worker panicked"); - - res - } - - fn spawn_test_worker( - db: &DB, - port: Option, - ) -> ( - tokio::sync::broadcast::Sender<()>, - tokio::task::JoinHandle<()>, - ) { - let (tx, rx) = tokio::sync::broadcast::channel(1); - let db = db.to_owned(); - let timeout = 4_000; - let worker_instance: &str = "test worker instance"; - let worker_name: String = next_worker_name(); - let i_worker: u64 = Default::default(); - let num_workers: u64 = 2; - let ip: &str = Default::default(); - let sleep_queue: u64 = DEFAULT_SLEEP_QUEUE / num_workers; - let port = port.unwrap_or(8000); - let worker_config = WorkerConfig { - base_internal_url: format!("http://localhost:{port}"), - base_url: format!("http://localhost:{port}"), - disable_nuser: std::env::var("DISABLE_NUSER") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(false), - disable_nsjail: std::env::var("DISABLE_NSJAIL") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(false), - keep_job_dir: std::env::var("KEEP_JOB_DIR") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(false), - }; - let future = async move { - run_worker( - &db, - timeout, - worker_instance, - worker_name, - i_worker, - num_workers, - ip, - sleep_queue, - worker_config, - rx, - ) - .await - }; - - (tx, tokio::task::spawn(future)) - } - - async fn listen_for_completed_jobs(db: &DB) -> impl Stream + Unpin { - listen_for_uuid_on(db, "insert on completed_job").await - } - - async fn listen_for_queue(db: &DB) -> impl Stream + Unpin { - listen_for_uuid_on(db, "queue").await - } - - async fn listen_for_uuid_on( - db: &DB, - channel: &'static str, - ) -> impl Stream + Unpin { - let mut listener = PgListener::connect_with(db).await.unwrap(); - listener.listen(channel).await.unwrap(); - - Box::pin(stream::unfold(listener, |mut listener| async move { - let uuid = listener - .try_recv() - .await - .unwrap() - .expect("lost database connection") - .payload() - .parse::() - .expect("invalid uuid"); - Some((uuid, listener)) - })) - } - - async fn completed_job_result(uuid: Uuid, db: &DB) -> Value { - query_scalar("SELECT result FROM completed_job WHERE id = $1") - .bind(uuid) - .fetch_one(db) - .await - .unwrap() - } - - #[axum::async_trait(?Send)] - trait StreamFind: futures::Stream + Unpin + Sized { - async fn find(self, item: &Self::Item) -> Option - where - for<'l> &'l Self::Item: std::cmp::PartialEq, - { - use futures::{future::ready, StreamExt}; - - self.filter(|i| ready(i == item)).next().await - } - } - - impl StreamFind for T {} -} diff --git a/backend/src/fixtures/base.sql b/backend/tests/fixtures/base.sql similarity index 100% rename from backend/src/fixtures/base.sql rename to backend/tests/fixtures/base.sql diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs new file mode 100644 index 0000000000..d4fb929cba --- /dev/null +++ b/backend/tests/worker.rs @@ -0,0 +1,1905 @@ +use futures::{stream, Stream}; +use serde_json::json; +use sqlx::{postgres::PgListener, query_scalar, types::Uuid, Pool, Postgres, Transaction}; +use windmill_api::jobs::{CompletedJob, Job}; +use windmill_common::{ + flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform}, + scripts::ScriptLang, + DEFAULT_SLEEP_QUEUE, +}; +use windmill_queue::{get_queued_job, JobPayload, RawCode}; +use windmill_worker::WorkerConfig; + +async fn initialize_tracing() { + use std::sync::Once; + + static ONCE: Once = Once::new(); + ONCE.call_once(windmill_common::tracing_init::initialize_tracing); +} + +/// it's important this is unique between tests as there is one prometheus registry and +/// run_worker shouldn't register the same metric with the same worker name more than once. +/// +/// this must fit in varchar(50) +fn next_worker_name() -> String { + use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + + static ID: AtomicUsize = AtomicUsize::new(0); + + // n.b.: when tests are run with RUST_TEST_THREADS or --test-threads set to 1, the name + // will be "main"... The id provides uniqueness & thread_name gives context. + let id = ID.fetch_add(1, SeqCst); + let thread = std::thread::current(); + let thread_name = thread + .name() + .map(|s| { + s.len() + .checked_sub(39) + .and_then(|start| s.get(start..)) + .unwrap_or(s) + }) + .unwrap_or("no thread name"); + format!("{id}/{thread_name}") +} + +pub async fn get_job_by_id<'c>( + mut tx: Transaction<'c, Postgres>, + w_id: &str, + id: Uuid, +) -> windmill_common::error::Result<(Option, Transaction<'c, Postgres>)> { + let cjob_option = sqlx::query_as::<_, CompletedJob>( + "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .fetch_optional(&mut tx) + .await?; + let job_option = match cjob_option { + Some(job) => Some(Job::CompletedJob(job)), + None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob), + }; + if job_option.is_some() { + Ok((job_option, tx)) + } else { + // check if a job had been moved in-between queries + let cjob_option = sqlx::query_as::<_, CompletedJob>( + "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .fetch_optional(&mut tx) + .await?; + Ok((cjob_option.map(Job::CompletedJob), tx)) + } +} + +pub struct ApiServer { + pub addr: std::net::SocketAddr, + tx: tokio::sync::broadcast::Sender<()>, + task: tokio::task::JoinHandle>, +} + +impl ApiServer { + pub async fn start(db: Pool) -> Self { + let (tx, rx) = tokio::sync::broadcast::channel::<()>(1); + + let sock = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + + let addr = sock.local_addr().unwrap(); + drop(sock); + + let task = tokio::task::spawn({ + windmill_api::run_server( + db.clone(), + addr, + format!("http://localhost:{}", addr.port()), + rx, + ) + }); + + return Self { addr, tx, task }; + } + + async fn close(self) -> anyhow::Result<()> { + let Self { tx, task, .. } = self; + drop(tx); + task.await.unwrap() + } +} + +mod suspend_resume { + + use futures::{Stream, StreamExt}; + use serde_json::json; + use sqlx::{query_scalar, types::Uuid}; + use windmill_common::flows::FlowValue; + use windmill_queue::JobPayload; + + use super::*; + + async fn wait_until_flow_suspends( + flow: Uuid, + mut queue: impl Stream + Unpin, + db: &Pool, + ) { + loop { + queue.by_ref().find(&flow).await.unwrap(); + if query_scalar("SELECT suspend > 0 FROM queue WHERE id = $1") + .bind(flow) + .fetch_one(db) + .await + .unwrap() + { + break; + } + } + } + + async fn _print_job(id: Uuid, db: &Pool) -> Result<(), anyhow::Error> { + tracing::info!( + "{:#?}", + get_job_by_id(db.begin().await?, "test-workspace", id) + .await? + .0 + ); + Ok(()) + } + + fn flow() -> FlowValue { + serde_json::from_value(serde_json::json!({ + "modules": [{ + "input_transform": { + "n": { "type": "javascript", "expr": "flow_input.n", }, + "port": { "type": "javascript", "expr": "flow_input.port", }, + "op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "\ + export async function main(n, port, op) {\ + const job = Deno.env.get('WM_JOB_ID'); + const token = Deno.env.get('WM_TOKEN'); + const r = await fetch( + `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ + {\ + method: 'GET',\ + headers: { 'Authorization': `Bearer ${token}` }\ + }\ + );\ + console.log(r);\ + const secret = await r.text();\ + console.log('Secret: ' + secret + ' ' + job + ' ' + token);\ + const r2 = await fetch( + `http://localhost:${port}/api/w/test-workspace/jobs/${op}/${job}/0/${secret}?approver=ruben`,\ + {\ + method: 'POST',\ + body: JSON.stringify('from job'),\ + headers: { 'content-type': 'application/json' }\ + }\ + );\ + console.log(await r2.text());\ + return n + 1;\ + }", + }, + "suspend": { + "required_events": 1 + }, + }, { + "input_transform": { + "n": { "type": "javascript", "expr": "previous_result", }, + "resume": { "type": "javascript", "expr": "resume", }, + "resumes": { "type": "javascript", "expr": "resumes", }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }" + }, + "suspend": { + "required_events": 1 + }, + }, { + "input_transform": { + "last": { "type": "javascript", "expr": "previous_result", }, + "resume": { "type": "javascript", "expr": "resume", }, + "resumes": { "type": "javascript", "expr": "resumes", }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(last, resume, resumes) { return { last, resume, resumes } }" + }, + }], + })) + .unwrap() + } + + #[sqlx::test(fixtures("base"))] + async fn test(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None }) + .arg("n", json!(1)) + .arg("port", json!(port)) + .push(&db) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + let queue = listen_for_queue(&db).await; + let db_ = db.clone(); + + in_test_worker(&db, async move { + let db = db_; + + wait_until_flow_suspends(flow, queue, &db).await; + // print_job(flow, &db).await; + /* The first job resumes itself. */ + let _first = completed.next().await.unwrap(); + // print_job(_first, &db).await; + + /* ... and send a request resume it. */ + let second = completed.next().await.unwrap(); + // print_job(second, &db).await; + + let tx = db.begin().await.unwrap(); + let (tx, token) = windmill_worker::create_token_for_owner(tx, "test-workspace", "u/test-user", "", 100, "").await.unwrap(); + tx.commit().await.unwrap(); + let secret = reqwest::get(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben" + )) + .await + .unwrap() + .error_for_status() + .unwrap() + .text().await.unwrap(); + println!("{}", secret); + + /* ImZyb20gdGVzdCIK = base64 "from test" */ + reqwest::get(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/resume/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK&approver=ruben" + )) + .await + .unwrap() + .error_for_status() + .unwrap(); + + completed.find(&flow).await.unwrap(); + }, port) + .await; + + server.close().await.unwrap(); + + let result = completed_job_result(flow, &db).await; + + assert_eq!( + json!({ + "last": { + "resume": "from job", + "resumes": ["from job"], + "n": 3, + }, + "resume": "from test", + "resumes": ["from test"], + }), + result + ); + + // ensure resumes are cleaned up through CASCADE when the flow is finished + assert_eq!( + 0, + query_scalar::<_, i64>("SELECT count(*) FROM resume_job") + .fetch_one(&db) + .await + .unwrap() + ); + } + + #[sqlx::test(fixtures("base"))] + async fn cancel_from_job(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let result = RunJob::from(JobPayload::RawFlow { value: flow(), path: None }) + .arg("n", json!(1)) + .arg("op", json!("cancel")) + .arg("port", json!(port)) + .run_until_complete(&db, port) + .await; + + server.close().await.unwrap(); + + assert_eq!( + json!({"error": "Job canceled: approval request disapproved by ruben" }), + result + ); + } + + #[sqlx::test(fixtures("base"))] + async fn cancel_after_suspend(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None }) + .arg("n", json!(1)) + .arg("port", json!(port)) + .push(&db) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + let queue = listen_for_queue(&db).await; + let db_ = db.clone(); + + in_test_worker(&db, async move { + let db = db_; + + wait_until_flow_suspends(flow, queue, &db).await; + /* The first job resumes itself. */ + let _first = completed.next().await.unwrap(); + /* ... and send a request resume it. */ + let second = completed.next().await.unwrap(); + + let tx = db.begin().await.unwrap(); + let (tx, token) = windmill_worker::create_token_for_owner(tx, "test-workspace", "u/test-user", "", 100, "").await.unwrap(); + tx.commit().await.unwrap(); + let secret = reqwest::get(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}" + )) + .await + .unwrap() + .error_for_status() + .unwrap() + .text().await.unwrap(); + println!("{}", secret); + + /* ImZyb20gdGVzdCIK = base64 "from test" */ + reqwest::get(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/cancel/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK" + )) + .await + .unwrap() + .error_for_status() + .unwrap(); + + completed.find(&flow).await.unwrap(); + }, port) + .await; + + server.close().await.unwrap(); + + let result = completed_job_result(flow, &db).await; + + assert_eq!( + json!({"error": "Job canceled: approval request disapproved by unknown" }), + result + ); + } +} + +mod retry { + use serde_json::json; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use windmill_common::flows::FlowValue; + use windmill_queue::JobPayload; + + use super::*; + + /// test helper provides some external state to help steps fail at specific points + struct Server { + addr: std::net::SocketAddr, + tx: tokio::sync::oneshot::Sender<()>, + task: tokio::task::JoinHandle>, + } + + impl Server { + async fn start(responses: Vec>) -> Self { + use tokio::net::TcpListener; + + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + let sock = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = sock.local_addr().unwrap(); + + let task = tokio::task::spawn(async move { + tokio::pin!(rx); + let mut results = vec![]; + + for next in responses { + let (mut peer, _) = tokio::select! { + _ = &mut rx => break, + r = sock.accept() => r, + } + .unwrap(); + + let n = peer.read_u8().await.unwrap(); + results.push(n); + + if let Some(next) = next { + peer.write_u8(next).await.unwrap(); + } + } + + results + }); + + return Self { addr, tx, task }; + } + + async fn close(self) -> Vec { + let Self { task, tx, .. } = self; + drop(tx); + task.await.unwrap() + } + } + + fn inner_step() -> &'static str { + r#" +export async function main(index, port) { + const buf = new Uint8Array([0]); + const sock = await Deno.connect({ port }); + await sock.write(new Uint8Array([index])); + if (await sock.read(buf) != 1) throw "read"; + return buf[0]; +} + "# + } + + fn last_step() -> &'static str { + r#" +def main(last, port): + with __import__("socket").create_connection((None, port)) as sock: + sock.send(b'\xff') + return last + [sock.recv(1)[0]] +"# + } + + fn flow_forloop_retry() -> FlowValue { + serde_json::from_value(serde_json::json!({ + "modules": [{ + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "result.items" }, + "skip_failures": false, + "modules": [{ + "input_transform": { + "index": { "type": "javascript", "expr": "previous_result.iter.index" }, + "port": { "type": "javascript", "expr": "flow_input.port" }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": inner_step(), + }, + }], + }, + "retry": { "constant": { "attempts": 2, "seconds": 0 } }, + }, { + "input_transform": { + "last": { "type": "javascript", "expr": "previous_result" }, + "port": { "type": "javascript", "expr": "flow_input.port" }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": last_step(), + }, + "retry": { "constant": { "attempts": 2, "seconds": 0 } }, + }], + })) + .unwrap() + } + + #[sqlx::test(fixtures("base"))] + async fn test_pass(db: Pool) { + initialize_tracing().await; + + // let server = ApiServer::start(db.clone()).await; + + /* fails twice in the loop, then once on the last step + * retry attempts is measured per-step, so it _retries_ at most two times on each step, + * which means it may run the step three times in total */ + + let (attempts, responses) = [ + /* pass fail */ + (0, Some(99)), + (1, None), + /* pass pass fail */ + (0, Some(99)), + (1, Some(99)), + (2, None), + /* pass pass pass */ + (0, Some(3)), + (1, Some(5)), + (2, Some(7)), + /* fail the last step once */ + (0xff, None), + (0xff, Some(9)), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + let server = Server::start(responses).await; + let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None }) + .arg("items", json!(["unused", "unused", "unused"])) + .arg("port", json!(server.addr.port())) + .run_until_complete(&db, server.addr.port()) + .await; + + assert_eq!(server.close().await, attempts); + assert_eq!(json!([3, 5, 7, 9]), result); + } + + #[sqlx::test(fixtures("base"))] + async fn test_fail_step_zero(db: Pool) { + initialize_tracing().await; + + /* attempt and fail the first step three times and stop */ + let (attempts, responses) = [ + /* pass fail x3 */ + (0, Some(99)), + (1, None), + (0, Some(99)), + (1, None), + (0, Some(99)), + (1, None), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + let server = Server::start(responses).await; + let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None }) + .arg("items", json!(["unused", "unused", "unused"])) + .arg("port", json!(server.addr.port())) + .run_until_complete(&db, server.addr.port()) + .await; + + assert_eq!(server.close().await, attempts); + assert!(result["error"] + .as_str() + .unwrap() + .contains(r#"Uncaught (in promise) "read""#)); + } + + #[sqlx::test(fixtures("base"))] + async fn test_fail_step_one(db: Pool) { + initialize_tracing().await; + + /* attempt and fail the first step three times and stop */ + let (attempts, responses) = [ + /* fail once, then pass */ + (0, None), + (0, Some(1)), + (1, Some(2)), + (2, Some(3)), + /* fail three times */ + (0xff, None), + (0xff, None), + (0xff, None), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + let server = Server::start(responses).await; + let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None }) + .arg("items", json!(["unused", "unused", "unused"])) + .arg("port", json!(server.addr.port())) + .run_until_complete(&db, server.addr.port()) + .await; + + assert_eq!(server.close().await, attempts); + assert!(result["error"] + .as_str() + .unwrap() + .contains("index out of range")); + } + + #[sqlx::test(fixtures("base"))] + async fn test_with_failure_module(db: Pool) { + initialize_tracing().await; + + // let server = ApiServer::start(db.clone()).await; + + let value = serde_json::from_value(json!({ + "modules": [{ + "input_transform": { "port": { "type": "javascript", "expr": "flow_input.port" } }, + "value": { + "type": "rawscript", + "language": "python3", + "content": r#" +def main(port): + with __import__("socket").create_connection((None, port)) as sock: + sock.send(b'\x00') + return sock.recv(1)[0]"#, + }, + "retry": { "constant": { "attempts": 1, "seconds": 0 } }, + }], + "failure_module": { + "input_transform": { "error": { "type": "javascript", "expr": "previous_result", }, + "port": { "type": "javascript", "expr": "flow_input.port" } }, + "value": { + "type": "rawscript", + "language": "python3", + "content": r#" +def main(error, port): + with __import__("socket").create_connection((None, port)) as sock: + sock.send(b'\xff') + return { "recv": sock.recv(1)[0], "from failure module": error }"#, + }, + "retry": { "constant": { "attempts": 1, "seconds": 0 } }, + }, + })) + .unwrap(); + let (attempts, responses) = [ + /* fail the first step twice */ + (0x00, None), + (0x00, None), + /* and the failure module once */ + (0xff, None), + (0xff, Some(42)), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + let server = Server::start(responses).await; + let result = RunJob::from(JobPayload::RawFlow { value, path: None }) + .arg("port", json!(server.addr.port())) + .run_until_complete(&db, server.addr.port()) + .await; + + assert_eq!(server.close().await, attempts); + assert_eq!( + result, + json!({ + "recv": 42, + "from failure module": { + "error": "Error during execution of the script:\n\nTraceback (most recent call last):\n File \"/tmp/main.py\", line 14, in \n res = inner_script.main(**kwargs)\n File \"/tmp/inner.py\", line 5, in main\n return sock.recv(1)[0]\nIndexError: index out of range", + } + }) + ); + } +} + +#[sqlx::test(fixtures("base"))] +async fn test_iteration(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [{ + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "result.items" }, + "skip_failures": false, + "modules": [{ + "input_transform": { + "n": { + "type": "javascript", + "expr": "previous_result.iter.value", + }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(n):\n if 1 < n:\n raise StopIteration(n)", + }, + }], + }, + }], + })) + .unwrap(); + + let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) + .arg("items", json!([])) + .run_until_complete(&db, server.addr.port()) + .await; + assert_eq!(result, serde_json::json!([])); + + /* Don't actually test that this does 257 jobs or that will take forever. */ + let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) + .arg("items", json!((0..257).collect::>())) + .run_until_complete(&db, server.addr.port()) + .await; + assert!(matches!(result, serde_json::Value::Object(_))); + assert!(result["error"] + .as_str() + .unwrap() + .contains("StopIteration: 2")); +} + +struct RunJob { + payload: JobPayload, + args: serde_json::Map, +} + +impl From for RunJob { + fn from(payload: JobPayload) -> Self { + Self { payload, args: Default::default() } + } +} + +impl RunJob { + fn arg>(mut self, k: S, v: serde_json::Value) -> Self { + self.args.insert(k.into(), v); + self + } + + async fn push(self, db: &Pool) -> Uuid { + let RunJob { payload, args } = self; + let tx = db.begin().await.unwrap(); + let (uuid, tx) = windmill_queue::push( + tx, + "test-workspace", + payload, + Some(args), + /* user */ "test-user", + /* permissioned_as */ "u/admin".to_string(), + /* scheduled_for_o */ None, + /* schedule_path */ None, + /* parent_job */ None, + /* is_flow_step */ false, + /* running */ false, + ) + .await + .expect("push has to succeed"); + + tx.commit().await.expect("push has to commit"); + + uuid + } + + /// push the job, spawn a worker, wait until the job is in completed_job + async fn run_until_complete(self, db: &Pool, port: u16) -> serde_json::Value { + let uuid = self.push(db).await; + let listener = listen_for_completed_jobs(db).await; + in_test_worker(db, listener.find(&uuid), port).await; + completed_job_result(uuid, db).await + } +} + +async fn run_job_in_new_worker_until_complete( + db: &Pool, + job: JobPayload, + port: u16, +) -> serde_json::Value { + RunJob::from(job).run_until_complete(db, port).await +} + +/// Start a worker with a timeout and run a future, until the worker quits or we time out. +/// +/// Cleans up the worker before resolving. +async fn in_test_worker( + db: &Pool, + inner: Fut, + port: u16, +) -> ::Output { + let (quit, worker) = spawn_test_worker(db, port); + let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker); + tokio::pin!(worker); + + let res = tokio::select! { + biased; + res = inner => res, + res = &mut worker => match + res.expect("worker timed out") + .expect("worker panicked") { + _ => panic!("worker quit early"), + }, + }; + + /* ensure the worker quits before we return */ + drop(quit); + + let _: () = worker + .await + .expect("worker timed out") + .expect("worker panicked"); + + res +} + +fn spawn_test_worker( + db: &Pool, + port: u16, +) -> ( + tokio::sync::broadcast::Sender<()>, + tokio::task::JoinHandle<()>, +) { + let (tx, rx) = tokio::sync::broadcast::channel(1); + let db = db.to_owned(); + let timeout = 4_000; + let worker_instance: &str = "test worker instance"; + let worker_name: String = next_worker_name(); + let i_worker: u64 = Default::default(); + let num_workers: u64 = 2; + let ip: &str = Default::default(); + let sleep_queue: u64 = DEFAULT_SLEEP_QUEUE / num_workers; + let port = port; + let worker_config = WorkerConfig { + base_internal_url: format!("http://localhost:{port}"), + base_url: format!("http://localhost:{port}"), + disable_nuser: std::env::var("DISABLE_NUSER") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false), + disable_nsjail: std::env::var("DISABLE_NSJAIL") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false), + keep_job_dir: std::env::var("KEEP_JOB_DIR") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false), + }; + let future = async move { + windmill_worker::run_worker( + &db, + timeout, + worker_instance, + worker_name, + i_worker, + num_workers, + ip, + sleep_queue, + worker_config, + rx, + ) + .await + }; + + (tx, tokio::task::spawn(future)) +} + +async fn listen_for_completed_jobs(db: &Pool) -> impl Stream + Unpin { + listen_for_uuid_on(db, "insert on completed_job").await +} + +async fn listen_for_queue(db: &Pool) -> impl Stream + Unpin { + listen_for_uuid_on(db, "queue").await +} + +async fn listen_for_uuid_on( + db: &Pool, + channel: &'static str, +) -> impl Stream + Unpin { + let mut listener = PgListener::connect_with(db).await.unwrap(); + listener.listen(channel).await.unwrap(); + + Box::pin(stream::unfold(listener, |mut listener| async move { + let uuid = listener + .try_recv() + .await + .unwrap() + .expect("lost database connection") + .payload() + .parse::() + .expect("invalid uuid"); + Some((uuid, listener)) + })) +} + +async fn completed_job_result(uuid: Uuid, db: &Pool) -> serde_json::Value { + query_scalar("SELECT result FROM completed_job WHERE id = $1") + .bind(uuid) + .fetch_one(db) + .await + .unwrap() +} + +#[axum::async_trait(?Send)] +trait StreamFind: futures::Stream + Unpin + Sized { + async fn find(self, item: &Self::Item) -> Option + where + for<'l> &'l Self::Item: std::cmp::PartialEq, + { + use futures::{future::ready, StreamExt}; + + self.filter(|i| ready(i == item)).next().await + } +} + +impl StreamFind for T {} + +#[sqlx::test(fixtures("base"))] +async fn test_deno_flow(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + + let numbers = "export function main() { return [1, 2, 3]; }"; + let doubles = "export function main(n) { return n * 2; }"; + + let flow = { + FlowValue { + modules: vec![ + FlowModule { + id: "a".to_string(), + value: FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: numbers.to_string(), + path: None, + }, + input_transforms: Default::default(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }, + FlowModule { + id: "b".to_string(), + value: FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { expr: "result".to_string() }, + skip_failures: false, + modules: vec![FlowModule { + id: "c".to_string(), + value: FlowModuleValue::RawScript { + input_transforms: [( + "n".to_string(), + InputTransform::Javascript { + expr: "previous_result.iter.value".to_string(), + }, + )] + .into(), + language: ScriptLang::Deno, + content: doubles.to_string(), + path: None, + }, + input_transforms: Default::default(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }], + }, + input_transforms: Default::default(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }, + ], + same_worker: false, + ..Default::default() + } + }; + + let job = JobPayload::RawFlow { value: flow, path: None }; + let port = server.addr.port(); + + for i in 0..50 { + println!("deno flow iteration: {}", i); + let result = run_job_in_new_worker_until_complete(&db, job.clone(), port).await; + assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i); + } +} + +#[sqlx::test(fixtures("base"))] +async fn test_deno_flow_same_worker(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + + let write_file = r#"export async function main(loop: boolean, i: number, path: string) { + await Deno.writeTextFile(`/shared/${path}`, `${loop} ${i}`); + }"# + .to_string(); + + let flow = FlowValue { + modules: vec![ + FlowModule { + id: "a".to_string(), + value: FlowModuleValue::RawScript { + input_transforms: [ + ( + "loop".to_string(), + InputTransform::Static { value: json!(false) }, + ), + ("i".to_string(), InputTransform::Static { value: json!(1) }), + ( + "path".to_string(), + InputTransform::Static { value: json!("outer.txt") }, + ), + ] + .into(), + language: ScriptLang::Deno, + content: write_file.clone(), + path: None, + }, + input_transforms: Default::default(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }, + FlowModule { + id: "b".to_string(), + value: FlowModuleValue::ForloopFlow { + iterator: InputTransform::Static { value: json!([1, 2, 3]) }, + skip_failures: false, + modules: vec![ + FlowModule { + id: "d".to_string(), + input_transforms: [ + ( + "i".to_string(), + InputTransform::Javascript { + expr: "previous_result.iter.value".to_string(), + }, + ), + ( + "loop".to_string(), + InputTransform::Static { value: json!(true) }, + ), + ( + "path".to_string(), + InputTransform::Static { value: json!("inner.txt") }, + ), + ] + .into(), + value: FlowModuleValue::RawScript { + input_transforms: [].into(), + language: ScriptLang::Deno, + content: write_file, + path: None, + }, + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }, + FlowModule { + id: "e".to_string(), + value: FlowModuleValue::RawScript { + input_transforms: [( + "path".to_string(), + InputTransform::Static { value: json!("inner.txt") }, + ), ( + "path2".to_string(), + InputTransform::Static { value: json!("outer.txt") }, + )] + .into(), + language: ScriptLang::Deno, + content: r#"export async function main(path: string, path2: string) { + return await Deno.readTextFile(`/shared/${path}`) + "," + await Deno.readTextFile(`/shared/${path2}`); + }"# + .to_string(), + path: None, + }, + input_transforms: [].into(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }, + ], + }, + input_transforms: Default::default(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + + }, + FlowModule { + id: "c".to_string(), + value: FlowModuleValue::RawScript { + input_transforms: [ + ( + "loops".to_string(), + InputTransform::Javascript { expr: "previous_result".to_string() }, + ), + ( + "path".to_string(), + InputTransform::Static { value: json!("outer.txt") }, + ), + ( + "path2".to_string(), + InputTransform::Static { value: json!("inner.txt") }, + ), + ] + .into(), + language: ScriptLang::Deno, + content: r#"export async function main(path: string, loops: string[], path2: string) { + return await Deno.readTextFile(`/shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`/shared/${path2}`); + }"# + .to_string(), + path: None, + }, + input_transforms: [].into(), + stop_after_if: Default::default(), + summary: Default::default(), + suspend: Default::default(), + retry: None, + sleep: None, + }, + ], + same_worker: true, + ..Default::default() + }; + + let job = JobPayload::RawFlow { value: flow, path: None }; + + let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port()).await; + assert_eq!( + result, + serde_json::json!("false 1,true 1,false 1,true 2,false 1,true 3,false 1,true 3") + ); +} + +#[sqlx::test(fixtures("base"))] +async fn test_flow_result_by_id(db: Pool) { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "a", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return 42 }", + } + }, + { + "value": { + "branches": [ + { + "modules": [{ + "value": { + "branches": [{"modules": [ { + "id": "d", + "value": { + "input_transforms": {"v": {"type": "javascript", "expr": "result_by_id(\"a\")"}}, + "type": "rawscript", + "language": "deno", + "content": "export function main(v){ return v }", + } + + },]}], + "type": "branchall", + } + }], + }], + "type": "branchall", + }, + } + ], + })) + .unwrap(); + + let job = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, job.clone(), port).await; + assert_eq!(result, serde_json::json!([[42]])); +} + +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_if(db: Pool) { + initialize_tracing().await; + // let server = ApiServer::start(db.clone()).await; + // let port = server.addr.port(); + + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + "stop_after_if": { + "expr": "result < 0", + "skip_if_stopped": false, + }, + }, + { + "input_transforms": { "n": { "type": "javascript", "expr": "previous_result" } }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(n): return f'last step saw {n}'", + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None }; + + let result = RunJob::from(job.clone()) + .arg("n", json!(123)) + .run_until_complete(&db, port) + .await; + assert_eq!(json!("last step saw 123"), result); + + let result = RunJob::from(job.clone()) + .arg("n", json!(-123)) + .run_until_complete(&db, port) + .await; + assert_eq!(json!(-123), result); +} + +#[sqlx::test(fixtures("base"))] +async fn test_python_flow(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let numbers = "def main(): return [1, 2, 3]"; + let doubles = "def main(n): return n * 2"; + + let flow: FlowValue = serde_json::from_value(serde_json::json!( { + "input_transform": {}, + "modules": [ + { + "value": { + "type": "rawscript", + "language": "python3", + "content": numbers, + }, + }, + { + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "result" }, + "skip_failures": false, + "modules": [{ + "value": { + "type": "rawscript", + "language": "python3", + "content": doubles, + }, + "input_transform": { + "n": { + "type": "javascript", + "expr": "previous_result.iter.value", + }, + }, + }], + }, + }, + ], + })) + .unwrap(); + + for i in 0..50 { + println!("python flow iteration: {}", i); + let result = run_job_in_new_worker_until_complete( + &db, + JobPayload::RawFlow { value: flow.clone(), path: None }, + port, + ) + .await; + + assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}"); + } +} + +#[sqlx::test(fixtures("base"))] +async fn test_python_flow_2(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "content": "import wmill\ndef main(): return \"Hello\"", + "language": "python3" + }, + "input_transform": {} + } + ] + })) + .unwrap(); + + for i in 0..10 { + println!("python flow iteration: {}", i); + let result = run_job_in_new_worker_until_complete( + &db, + JobPayload::RawFlow { value: flow.clone(), path: None }, + port, + ) + .await; + + assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}"); + } +} + +#[sqlx::test(fixtures("base"))] +async fn test_go_job(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +import "fmt" + +func main(derp string) (string, error) { + fmt.Println("Hello, 世界") + return fmt.Sprintf("hello %s", derp), nil +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + content, + path: None, + language: ScriptLang::Go, + })) + .arg("derp", json!("world")) + .run_until_complete(&db, port) + .await; + + assert_eq!(result, serde_json::json!("hello world")); +} + +#[sqlx::test(fixtures("base"))] +async fn test_python_job(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +def main(): + return "hello world" + "# + .to_owned(); + + let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 }); + + let result = run_job_in_new_worker_until_complete(&db, job, port).await; + + assert_eq!(result, serde_json::json!("hello world")); +} + +#[sqlx::test(fixtures("base"))] +async fn test_python_job_heavy_dep(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +import numpy as np + +def main(): + a = np.arange(15).reshape(3, 5) + return len(a) + "# + .to_owned(); + + let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 }); + + let result = run_job_in_new_worker_until_complete(&db, job, port).await; + + assert_eq!(result, serde_json::json!(3)); +} + +#[sqlx::test(fixtures("base"))] +async fn test_python_job_with_imports(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +import wmill + +def main(): + return wmill.get_workspace() + "# + .to_owned(); + + let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 }); + + let result = run_job_in_new_worker_until_complete(&db, job, port).await; + + assert_eq!(result, serde_json::json!("test-workspace")); +} + +#[sqlx::test(fixtures("base"))] +async fn test_empty_loop(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [] }, + "modules": [ + { + "input_transform": { + "n": { + "type": "javascript", + "expr": "previous_result.iter.value", + }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + } + ], + }, + }, + { + "input_transform": { + "items": { + "type": "javascript", + "expr": "previous_result", + }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(items): return sum(items)", + }, + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!(result, serde_json::json!(0)); +} + +#[sqlx::test(fixtures("base"))] +async fn test_empty_loop_2(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [] }, + "modules": [ + { + "input_transform": { + "n": { + "type": "javascript", + "expr": "previous_result.iter.value", + }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + }, + } + ], + }, + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!(result, serde_json::json!([])); +} + +#[sqlx::test(fixtures("base"))] +async fn test_step_after_loop(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [2,3,4] }, + "modules": [ + { + "input_transform": { + "n": { + "type": "javascript", + "expr": "previous_result.iter.value", + }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(n): return n", + } , + } + ], + }, + }, + { + "input_transform": { + "items": { + "type": "javascript", + "expr": "previous_result", + }, + }, + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(items): return sum(items)", + }, + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!(result, serde_json::json!(9)); +} + +fn module_add_item_to_list(i: i32) -> serde_json::Value { + json!({ + "input_transform": { + "array": { + "type": "javascript", + "expr": "previous_result", + }, + "i": { + "type": "static", + "value": json!(i), + } + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(array, i){ array.push(i); return array }", + } + }) +} + +fn module_failure() -> serde_json::Value { + json!({ + "input_transform": {}, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ throw Error('failure') }", + } + }) +} + +#[sqlx::test(fixtures("base"))] +async fn test_branchone_simple(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return [1] }", + } + }, + { + "value": { + "branches": [], + "default": [module_add_item_to_list(2)], + "type": "branchone", + } + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!(result, serde_json::json!([1, 2])); +} + +#[sqlx::test(fixtures("base"))] +async fn test_branchall_simple(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return [1] }", + } + }, + { + "value": { + "branches": [ + {"modules": [module_add_item_to_list(2)]}, + {"modules": [module_add_item_to_list(3)]}], + "type": "branchall", + } + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!(result, serde_json::json!([[1, 2], [1, 3]])); +} + +#[sqlx::test(fixtures("base"))] +async fn test_branchall_skip_failure(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return [1] }", + } + }, + { + "value": { + "branches": [ + {"modules": [module_failure()], "skip_failure": false}, + {"modules": [module_add_item_to_list(3)]}], + "type": "branchall", + } + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!( + result, + serde_json::json!({"error": "Error during execution of the script:\n\nerror: Uncaught (in promise) Error: failure\nexport function main(){ throw Error('failure') }\n ^\n at main (file:///tmp/inner.ts:1:31)\n at run (file:///tmp/main.ts:9:26)\n at file:///tmp/main.ts:14:1"}) + ); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return [1] }", + } + }, + { + "value": { + "branches": [ + {"modules": [module_failure()], "skip_failure": true}, + {"modules": [module_add_item_to_list(2)]} + ], + "type": "branchall", + } + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!( + result, + serde_json::json!([{"error": "Error during execution of the script:\n\nerror: Uncaught (in promise) Error: failure\nexport function main(){ throw Error('failure') }\n ^\n at main (file:///tmp/inner.ts:1:31)\n at run (file:///tmp/main.ts:9:26)\n at file:///tmp/main.ts:14:1"}, [1, 2]]) + ); +} + +#[sqlx::test(fixtures("base"))] +async fn test_branchone_nested(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return [] }", + } + }, + module_add_item_to_list(1), + { + "value": { + "branches": [ + { + "expr": "false", + "modules": [] + }, + { + "expr": "true", + "modules": [ { + "value": { + "branches": [ + { + "expr": "false", + "modules": [] + }], + "default": [module_add_item_to_list(2)], + "type": "branchone", + } + }] + }, + ], + "default": [module_add_item_to_list(-4)], + "type": "branchone", + } + }, + module_add_item_to_list(3), + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!(result, serde_json::json!([1, 2, 3])); +} + +#[sqlx::test(fixtures("base"))] +async fn test_branchall_nested(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(){ return [1] }", + } + }, + { + "value": { + "branches": [ + { + "modules": [ { + "value": { + "branches": [ + {"modules": [module_add_item_to_list(2)]}, + {"modules": [module_add_item_to_list(3)]}], + "type": "branchall", + } + }, { + "value": { + "branches": [ + {"modules": [module_add_item_to_list(4)]}, + {"modules": [module_add_item_to_list(5)]}], + "type": "branchall", + } + } + ] + }, + {"modules": [module_add_item_to_list(6)]}], + "type": "branchall", + } + }, + ], + })) + .unwrap(); + + let flow = JobPayload::RawFlow { value: flow, path: None }; + let result = run_job_in_new_worker_until_complete(&db, flow, port).await; + + assert_eq!( + result, + serde_json::json!([[[[1, 2], [1, 3], 4], [[1, 2], [1, 3], 5]], [1, 6]]) + ); +} + +#[sqlx::test(fixtures("base"))] +async fn test_failure_module(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [{ + "input_transform": { + "l": { "type": "javascript", "expr": "[]", }, + "n": { "type": "javascript", "expr": "flow_input.n", }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(n, l) { if (n == 0) throw l; return { l: [...l, 0] } }", + }, + }, { + "input_transform": { + "l": { "type": "javascript", "expr": "previous_result.l", }, + "n": { "type": "javascript", "expr": "flow_input.n", }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(n, l) { if (n == 1) throw l; return { l: [...l, 1] } }", + }, + }, { + "input_transform": { + "l": { "type": "javascript", "expr": "previous_result.l", }, + "n": { "type": "javascript", "expr": "flow_input.n", }, + }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(n, l) { if (n == 2) throw l; return { l: [...l, 2] } }", + }, + }], + "failure_module": { + "input_transform": { "error": { "type": "javascript", "expr": "previous_result", } }, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(error) { return { 'from failure module': error } }", + } + }, + })) + .unwrap(); + + let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) + .arg("n", json!(0)) + .run_until_complete(&db, port) + .await; + assert!(result["from failure module"]["error"] + .as_str() + .unwrap() + .contains("Uncaught (in promise) []")); + + let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) + .arg("n", json!(1)) + .run_until_complete(&db, port) + .await; + assert!(result["from failure module"]["error"] + .as_str() + .unwrap() + .contains("Uncaught (in promise) [ 0 ]")); + + let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) + .arg("n", json!(2)) + .run_until_complete(&db, port) + .await; + assert!(result["from failure module"]["error"] + .as_str() + .unwrap() + .contains("Uncaught (in promise) [ 0, 1 ]")); + + let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None }) + .arg("n", json!(3)) + .run_until_complete(&db, port) + .await; + assert_eq!(json!({ "l": [0, 1, 2] }), result); +} diff --git a/backend/windmill-api-client/.gitignore b/backend/windmill-api-client/.gitignore new file mode 100644 index 0000000000..7bcd4c6b1e --- /dev/null +++ b/backend/windmill-api-client/.gitignore @@ -0,0 +1 @@ +bundled.json \ No newline at end of file diff --git a/backend/windmill-api-client/Cargo.toml b/backend/windmill-api-client/Cargo.toml new file mode 100644 index 0000000000..910f27f148 --- /dev/null +++ b/backend/windmill-api-client/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "windmill-api-client" +version.workspace = true +authors.workspace = true +edition.workspace = true +build = "build.rs" + +[lib] +name = "windmill_api_client" +path = "./src/lib.rs" + + +[dependencies] +progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" } +reqwest = { version = "0.11", features = ["json", "stream"] } +serde = { version = "1.0", features = ["derive"] } +chrono.workspace = true +uuid.workspace = true +serde_json.workspace = true +rand.workspace = true +base64.workspace = true + +[build-dependencies] +progenitor = { git = "https://github.com/oxidecomputer/progenitor" } +serde_json = "1.0" diff --git a/backend/windmill-api-client/README.md b/backend/windmill-api-client/README.md new file mode 100644 index 0000000000..fb265cc2b5 --- /dev/null +++ b/backend/windmill-api-client/README.md @@ -0,0 +1,8 @@ +# Windmill API Client + +This holds an autogenerated OpenAPI client in Rust. It's exclusively used in the backend to talk to the [api server](../windmill-api/). + +## Generate + +Simply run `sh bundle.sh` to update bundled.json. The source code will automatically update. +This requires the swagger-cli to be installed for bundling. diff --git a/backend/windmill-api-client/build.rs b/backend/windmill-api-client/build.rs new file mode 100644 index 0000000000..294c9c8f76 --- /dev/null +++ b/backend/windmill-api-client/build.rs @@ -0,0 +1,22 @@ +use std::{ + env, + fs::{self, File}, + path::Path, + process::Command, +}; + +fn main() { + let src = "../windmill-api/openapi.yaml"; + println!("cargo:rerun-if-changed={}", src); + Command::new("sh").args(&["bundle.sh"]).status().unwrap(); + let file = File::open("./bundled.json").unwrap(); + let spec = serde_json::from_reader(file).unwrap(); + let mut generator = progenitor::Generator::default(); + + let content = generator.generate_text(&spec).unwrap(); + + let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf(); + out_file.push("codegen.rs"); + + fs::write(out_file, content).unwrap(); +} diff --git a/backend/windmill-api-client/bundle.sh b/backend/windmill-api-client/bundle.sh new file mode 100644 index 0000000000..0fe4b172ca --- /dev/null +++ b/backend/windmill-api-client/bundle.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +npx swagger-cli bundle ../windmill-api/openapi.yaml > bundled.json \ No newline at end of file diff --git a/backend/windmill-api-client/src/lib.rs b/backend/windmill-api-client/src/lib.rs new file mode 100644 index 0000000000..0b7222c5a5 --- /dev/null +++ b/backend/windmill-api-client/src/lib.rs @@ -0,0 +1,14 @@ +include!(concat!(env!("OUT_DIR"), "/codegen.rs")); + +pub fn create_client(base_url: &str, token: String) -> Client { + let mut val = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")) + .expect("header creation"); + val.set_sensitive(true); + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert(reqwest::header::AUTHORIZATION, val); + let client = reqwest::ClientBuilder::new() + .default_headers(headers) + .build() + .expect("client build"); + Client::new_with_client(&format!("{}/api", base_url.trim_end_matches('/')), client) +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml new file mode 100644 index 0000000000..c86f0b0f5a --- /dev/null +++ b/backend/windmill-api/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "windmill-api" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api" +path = "src/lib.rs" + +[[bin]] +name = "windmill_api" +path = "src/main.rs" + +[dependencies] +windmill-queue.workspace = true +windmill-common = { workspace = true, features = [ + "reqwest", + "prometheus", + "axum", + "tokio", + "hyper", + "sqlx", + "tracing_init", +] } +windmill-audit.workspace = true +windmill-parser.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-go.workspace = true +windmill-parser-py.workspace = true +tokio.workspace = true +anyhow.workspace = true +argon2.workspace = true +axum.workspace = true +futures.workspace = true +git-version.workspace = true +tower.workspace = true +tower-cookies.workspace = true +tower-http.workspace = true +hyper.workspace = true +itertools.workspace = true +reqwest.workspace = true +serde.workspace = true +sqlx.workspace = true +async-oauth2.workspace = true +tracing.workspace = true +sql-builder.workspace = true +serde_json.workspace = true +chrono.workspace = true +hex.workspace = true +base64.workspace = true +serde_urlencoded.workspace = true +cron.workspace = true +mime_guess.workspace = true +rust-embed.workspace = true +tracing-subscriber.workspace = true +retainer.workspace = true +rand.workspace = true +time.workspace = true +magic-crypt.workspace = true +tempfile.workspace = true +tokio-util.workspace = true +tokio-tar.workspace = true +hmac.workspace = true diff --git a/backend/windmill-api/README.md b/backend/windmill-api/README.md new file mode 100644 index 0000000000..f427ca0689 --- /dev/null +++ b/backend/windmill-api/README.md @@ -0,0 +1,5 @@ +# Windmill API + +The API server, exposing functionality to other components and the frontend + +This crate exposes both a library as well as a binary target. diff --git a/backend/openapi.yaml b/backend/windmill-api/openapi.yaml similarity index 98% rename from backend/openapi.yaml rename to backend/windmill-api/openapi.yaml index 355cf8f247..06b1edb4ea 100644 --- a/backend/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1608,7 +1608,7 @@ paths: type: object properties: flow: - $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow" + $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" /scripts/hub/get/{path}: get: @@ -1936,7 +1936,6 @@ paths: schema: type: string - /w/{workspace}/scripts/exists/p/{path}: get: summary: exists script by path @@ -2086,6 +2085,36 @@ paths: application/json: schema: {} + /w/{workspace}/jobs/result_by_id/{flow_job_id}/{node_id}: + get: + summary: get job result by id + operationId: resultById + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: flow_job_id + in: path + required: true + schema: + type: string + - name: node_id + in: path + required: true + schema: + type: string + - name: skip_direct + description: Skip checking that the node is part of the given flow. + in: query + schema: + type: boolean + responses: + "200": + description: job result + content: + application/json: + schema: {} + /w/{workspace}/flows/list: get: summary: list all available flows @@ -2601,7 +2630,6 @@ paths: schema: type: string - /w/{workspace}/jobs/resume/{id}/{resume_id}/{signature}: get: summary: resume a job for a suspended flow @@ -2621,10 +2649,6 @@ paths: required: true schema: type: string - - name: payload - in: query - schema: - type: object - name: approver in: query schema: @@ -2692,10 +2716,6 @@ paths: required: true schema: type: string - - name: payload - in: query - schema: - type: object - name: approver in: query schema: @@ -3496,7 +3516,7 @@ components: # explode: false schemas: - $ref: "../openflow.openapi.yaml#/components/schemas" + $ref: "../../openflow.openapi.yaml#/components/schemas" Script: type: object properties: @@ -3616,7 +3636,7 @@ components: "flow", "flowpreview", "script_hub", - "identity" + "identity", ] schedule_path: type: string @@ -3626,9 +3646,9 @@ components: The user (u/userfoo) or group (g/groupfoo) whom the execution of this script will be permissioned_as and by extension its DT_TOKEN. flow_status: - $ref: "../openflow.openapi.yaml#/components/schemas/FlowStatus" + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus" raw_flow: - $ref: "../openflow.openapi.yaml#/components/schemas/FlowValue" + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue" is_flow_step: type: boolean language: @@ -3695,7 +3715,7 @@ components: "flow", "flowpreview", "script_hub", - "identity" + "identity", ] schedule_path: type: string @@ -3705,9 +3725,9 @@ components: The user (u/userfoo) or group (g/groupfoo) whom the execution of this script will be permissioned_as and by extension its DT_TOKEN. flow_status: - $ref: "../openflow.openapi.yaml#/components/schemas/FlowStatus" + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus" raw_flow: - $ref: "../openflow.openapi.yaml#/components/schemas/FlowValue" + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue" is_flow_step: type: boolean language: @@ -4379,7 +4399,7 @@ components: Flow: allOf: - - $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow" + - $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" - $ref: "#/components/schemas/FlowMetadata" FlowMetadata: @@ -4409,7 +4429,7 @@ components: OpenFlowWPath: allOf: - - $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow" + - $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" - type: object properties: path: @@ -4421,7 +4441,7 @@ components: type: object properties: value: - $ref: "../openflow.openapi.yaml#/components/schemas/FlowValue" + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue" path: type: string args: diff --git a/backend/windmill-api/src/audit.rs b/backend/windmill-api/src/audit.rs new file mode 100644 index 0000000000..57fd8cff49 --- /dev/null +++ b/backend/windmill-api/src/audit.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Path, Query}, + routing::get, + Extension, Json, Router, +}; +use windmill_audit::{AuditLog, ListAuditLogQuery}; +use windmill_common::{error::JsonResult, utils::Pagination}; + +use crate::{db::UserDB, users::Authed}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_audit)) + .route("/get/:id", get(get_audit)) +} + +async fn get_audit( + authed: Authed, + Extension(user_db): Extension, + Path(id): Path, +) -> JsonResult { + let tx = user_db.begin(&authed).await?; + let audit = windmill_audit::get_audit(tx, id).await?; + Ok(Json(audit)) +} +async fn list_audit( + authed: Authed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, + Query(lq): Query, +) -> JsonResult> { + let tx = user_db.begin(&authed).await?; + let rows = windmill_audit::list_audit(tx, w_id, pagination, lq).await?; + Ok(Json(rows)) +} diff --git a/backend/src/capture.rs b/backend/windmill-api/src/capture.rs similarity index 91% rename from backend/src/capture.rs rename to backend/windmill-api/src/capture.rs index 19705860c8..26aa2f6985 100644 --- a/backend/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -1,15 +1,25 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + use axum::{ extract::{Extension, Path}, routing::{get, post, put}, Json, Router, }; use hyper::StatusCode; +use windmill_common::{ + error::{JsonResult, Result}, + utils::{not_found_if_none, StripPath}, +}; use crate::{ db::{UserDB, DB}, - error::{JsonResult, Result}, users::Authed, - utils::{not_found_if_none, StripPath}, }; const KEEP_LAST: i64 = 8; diff --git a/backend/src/db.rs b/backend/windmill-api/src/db.rs similarity index 75% rename from backend/src/db.rs rename to backend/windmill-api/src/db.rs index 8f6ca4b377..1b88c16340 100644 --- a/backend/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -6,23 +6,15 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::{error::Error, users::Authed}; -use sqlx::{postgres::PgPoolOptions, Pool, Postgres, Transaction}; -use std::time::Duration; +use sqlx::{Pool, Postgres, Transaction}; +use windmill_common::error::Error; + +use crate::users::Authed; pub type DB = Pool; -pub async fn connect(database_url: &str, max_connections: u32) -> Result { - PgPoolOptions::new() - .max_connections(max_connections) - .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins - .connect(database_url) - .await - .map_err(|err| Error::ConnectingToDatabase(err.to_string())) -} - pub async fn migrate(db: &DB) -> Result<(), Error> { - match sqlx::migrate!("./migrations").run(db).await { + match sqlx::migrate!("../migrations").run(db).await { Ok(_) => Ok(()), Err(err) => Err(err), }?; diff --git a/backend/src/flows.rs b/backend/windmill-api/src/flows.rs similarity index 71% rename from backend/src/flows.rs rename to backend/windmill-api/src/flows.rs index db65370240..9345ffb527 100644 --- a/backend/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -6,9 +6,6 @@ * LICENSE-AGPL for a copy of the license. */ -use std::collections::HashMap; -use std::time::Duration; - use reqwest::Client; use sql_builder::prelude::*; @@ -17,18 +14,20 @@ use axum::{ routing::{get, post}, Json, Router, }; -use serde::{Deserialize, Serialize}; use sql_builder::SqlBuilder; -use sqlx::{FromRow, Postgres, Transaction}; +use sqlx::{Postgres, Transaction}; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{self, to_anyhow, Error, JsonResult, Result}, + flows::{Flow, ListFlowQuery, NewFlow}, + utils::{ + http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath, + }, +}; use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{self, to_anyhow, Error, JsonResult, Result}, - more_serde::{default_id, default_true, is_default}, - scripts::{Schema, ScriptLang}, users::Authed, - utils::{http_get_from_hub, list_elems_from_hub, Pagination, StripPath}, }; pub fn workspaced_service() -> Router { @@ -47,208 +46,6 @@ pub fn global_service() -> Router { .route("/hub/get/:id", get(get_hub_flow_by_id)) } -#[derive(FromRow, Serialize)] -pub struct Flow { - pub workspace_id: String, - pub path: String, - pub summary: String, - pub description: String, - pub value: serde_json::Value, - pub edited_by: String, - pub edited_at: chrono::DateTime, - pub archived: bool, - pub schema: Option, - pub extra_perms: serde_json::Value, -} - -#[derive(FromRow, Deserialize)] -pub struct NewFlow { - pub path: String, - pub summary: String, - pub description: String, - pub value: serde_json::Value, - pub schema: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default)] -pub struct FlowValue { - pub modules: Vec, - #[serde(default)] - pub failure_module: Option, - #[serde(default)] - #[serde(skip_serializing_if = "is_default")] - pub same_worker: bool, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct StopAfterIf { - pub expr: String, - pub skip_if_stopped: bool, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct Retry { - constant: ConstantDelay, - exponential: ExponentialDelay, -} - -impl Retry { - /// Takes the number of previous retries and returns the interval until the next retry if any. - /// - /// May return [`Duration::ZERO`] to retry immediately. - pub fn interval(&self, previous_attempts: u16) -> Option { - let Self { constant, exponential } = self; - - if previous_attempts < constant.attempts { - Some(Duration::from_secs(constant.seconds as u64)) - } else if previous_attempts - constant.attempts < exponential.attempts { - let exp = previous_attempts.saturating_add(1) as u32; - let secs = exponential.multiplier * exponential.seconds.saturating_pow(exp); - Some(Duration::from_secs(secs as u64)) - } else { - None - } - } - - pub fn has_attempts(&self) -> bool { - self.constant.attempts != 0 || self.exponential.attempts != 0 - } - - pub fn max_attempts(&self) -> u16 { - self.constant - .attempts - .saturating_add(self.exponential.attempts) - } - - pub fn max_interval(&self) -> Option { - self.max_attempts() - .checked_sub(1) - .and_then(|p| self.interval(p)) - } -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct ConstantDelay { - pub attempts: u16, - pub seconds: u16, -} - -/// multiplier * seconds ^ failures -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] -#[serde(default)] -pub struct ExponentialDelay { - pub attempts: u16, - pub multiplier: u16, - pub seconds: u16, -} - -impl Default for ExponentialDelay { - fn default() -> Self { - Self { attempts: 0, multiplier: 1, seconds: 0 } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct Suspend { - #[serde(skip_serializing_if = "Option::is_none")] - pub required_events: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct FlowModule { - #[serde(default = "default_id")] - pub id: String, - #[serde(default)] - #[serde(alias = "input_transform")] - pub input_transforms: HashMap, - pub value: FlowModuleValue, - pub stop_after_if: Option, - pub summary: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub suspend: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub sleep: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum InputTransform { - Static { value: serde_json::Value }, - Javascript { expr: String }, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct BranchOneModules { - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - pub expr: String, - pub modules: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct BranchAllModules { - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - pub modules: Vec, - #[serde(default = "default_true")] - pub skip_failure: bool, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum FlowModuleValue { - Script { - #[serde(default)] - #[serde(alias = "input_transform")] - input_transforms: HashMap, - path: String, - }, - ForloopFlow { - iterator: InputTransform, - modules: Vec, - #[serde(default = "default_true")] - skip_failures: bool, - }, - BranchOne { - branches: Vec, - default: Vec, - }, - BranchAll { - branches: Vec, - }, - RawScript { - #[serde(default)] - #[serde(alias = "input_transform")] - input_transforms: HashMap, - content: String, - path: Option, - language: ScriptLang, - }, - Identity, -} - -#[derive(Deserialize)] -pub struct ListFlowQuery { - pub path_start: Option, - pub path_exact: Option, - pub edited_by: Option, - pub show_archived: Option, - pub order_by: Option, - pub order_desc: Option, -} - async fn list_flows( authed: Authed, Extension(user_db): Extension, @@ -256,7 +53,7 @@ async fn list_flows( Query(pagination): Query, Query(lq): Query, ) -> JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); + let (per_page, offset) = paginate(pagination); let mut sqlb = SqlBuilder::select_from("flow as o") .fields(&[ @@ -428,7 +225,7 @@ async fn update_flow( ) .fetch_optional(&mut tx) .await?; - crate::utils::not_found_if_none(flow, "Flow", flow_path)?; + not_found_if_none(flow, "Flow", flow_path)?; audit_log( &mut tx, @@ -467,7 +264,7 @@ async fn get_flow_by_path( .await?; tx.commit().await?; - let flow = crate::utils::not_found_if_none(flow_o, "Flow", path)?; + let flow = not_found_if_none(flow_o, "Flow", path)?; Ok(Json(flow)) } @@ -524,8 +321,15 @@ async fn archive_flow_by_path( #[cfg(test)] mod tests { - // Note this useful idiom: importing names from outer (for mod tests) scope. - use super::*; + use std::{collections::HashMap, time::Duration}; + + use windmill_common::{ + flows::{ + ConstantDelay, ExponentialDelay, FlowModule, FlowModuleValue, FlowValue, + InputTransform, Retry, StopAfterIf, + }, + scripts, + }; const SECOND: Duration = Duration::from_secs(1); @@ -556,7 +360,7 @@ mod tests { value: FlowModuleValue::RawScript { input_transforms: HashMap::new(), content: "test".to_string(), - language: crate::scripts::ScriptLang::Deno, + language: scripts::ScriptLang::Deno, path: None, }, stop_after_if: Some(StopAfterIf { diff --git a/backend/src/granular_acls.rs b/backend/windmill-api/src/granular_acls.rs similarity index 92% rename from backend/src/granular_acls.rs rename to backend/windmill-api/src/granular_acls.rs index d8645ae007..22116ac7e1 100644 --- a/backend/src/granular_acls.rs +++ b/backend/windmill-api/src/granular_acls.rs @@ -6,12 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::{ - db::UserDB, - error::{Error, JsonResult, Result}, - users::Authed, - utils::StripPath, -}; +use crate::{db::UserDB, users::Authed}; use axum::{ extract::{Extension, Path}, routing::{get, post}, @@ -19,6 +14,10 @@ use axum::{ }; use serde::{Deserialize, Serialize}; +use windmill_common::{ + error::{Error, JsonResult, Result}, + utils::{not_found_if_none, StripPath}, +}; pub fn workspaced_service() -> Router { Router::new() @@ -56,7 +55,7 @@ async fn add_granular_acl( .fetch_optional(&mut tx) .await?; - let _ = crate::utils::not_found_if_none(obj_o, &kind, &path)?; + let _ = not_found_if_none(obj_o, &kind, &path)?; tx.commit().await?; Ok("Successfully modified granular acl".to_string()) @@ -85,7 +84,7 @@ async fn remove_granular_acl( .fetch_optional(&mut tx) .await?; - let _ = crate::utils::not_found_if_none(obj_o, &kind, &path)?; + let _ = not_found_if_none(obj_o, &kind, &path)?; tx.commit().await?; Ok("Successfully removed granular acl".to_string()) @@ -112,7 +111,7 @@ async fn get_granular_acls( .fetch_optional(&mut tx) .await?; - let obj = crate::utils::not_found_if_none(obj_o, &kind, &path)?; + let obj = not_found_if_none(obj_o, &kind, &path)?; tx.commit().await?; Ok(Json(obj)) diff --git a/backend/src/groups.rs b/backend/windmill-api/src/groups.rs similarity index 91% rename from backend/src/groups.rs rename to backend/windmill-api/src/groups.rs index f92f6e33ba..2b96d04df9 100644 --- a/backend/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -7,17 +7,20 @@ */ use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{Error, JsonResult, Result}, - users::{owner_to_token_owner, Authed}, - utils::Pagination, + users::Authed, }; use axum::{ extract::{Extension, Path, Query}, routing::{delete, get, post}, Json, Router, }; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{Error, JsonResult, Result}, + users::owner_to_token_owner, + utils::{not_found_if_none, paginate, Pagination}, +}; use serde::{Deserialize, Serialize}; use sqlx::{FromRow, Postgres, Transaction}; @@ -72,7 +75,7 @@ async fn list_groups( Path(w_id): Path, Query(pagination): Query, ) -> JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); + let (per_page, offset) = paginate(pagination); let rows = sqlx::query_as!( Group, @@ -158,11 +161,7 @@ async fn get_group( ) -> JsonResult { let mut tx = user_db.begin(&authed).await?; - let group = crate::utils::not_found_if_none( - get_group_opt(&mut tx, &w_id, &name).await?, - "Group", - &name, - )?; + let group = not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; let members = sqlx::query_scalar!( "SELECT usr.username @@ -191,7 +190,7 @@ async fn delete_group( ) -> Result { let mut tx = user_db.begin(&authed).await?; - crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; sqlx::query!( "DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2", @@ -229,7 +228,7 @@ async fn update_group( ) -> Result { let mut tx = user_db.begin(&authed).await?; - crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; sqlx::query_as!( Group, @@ -263,7 +262,7 @@ async fn add_user( ) -> Result { let mut tx = user_db.begin(&authed).await?; - crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; sqlx::query_as!( Group, @@ -297,7 +296,7 @@ async fn remove_user( ) -> Result { let mut tx = user_db.begin(&authed).await?; - crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; if &name == "all" { return Err(Error::BadRequest(format!("Cannot delete users from all"))); } diff --git a/backend/src/jobs.rs b/backend/windmill-api/src/jobs.rs similarity index 61% rename from backend/src/jobs.rs rename to backend/windmill-api/src/jobs.rs index a39d37965b..92bc126279 100644 --- a/backend/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6,48 +6,35 @@ * LICENSE-AGPL for a copy of the license. */ -use axum::extract::Host; - use anyhow::Context; +use axum::{ + extract::{FromRequest, Path, Query}, + response::{IntoResponse, Response}, + routing::{get, post}, + Extension, Json, Router, +}; use hmac::Mac; -use sql_builder::prelude::*; -use sqlx::{query_scalar, Postgres, Transaction}; -use std::{collections::HashMap, str::FromStr}; -use tracing::instrument; - -use crate::{ - audit::{audit_log, ActionKind}, - db::{UserDB, DB}, +use hyper::StatusCode; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sql_builder::{prelude::*, quote, SqlBuilder}; +use sqlx::{query_scalar, types::Uuid, Postgres, Transaction}; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ error::{self, to_anyhow, Error}, flows::FlowValue, oauth2::HmacSha256, - schedule::get_schedule_opt, - scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang}, - users::{owner_to_token_owner, Authed}, - utils::{now_from_db, require_admin, Pagination, StripPath}, + scripts::{ScriptHash, ScriptLang}, + users::owner_to_token_owner, + utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, + worker_flow::{Approval, FlowStatus, FlowStatusModule}, +}; +use windmill_queue::{get_queued_job, push, JobKind, JobPayload, QueuedJob, RawCode}; + +use crate::{ + db::{UserDB, DB}, + users::Authed, variables::get_workspace_key, - worker, - worker_flow::{ - init_flow_status, Approval, FlowStatus, FlowStatusModule, MAX_RETRY_ATTEMPTS, - MAX_RETRY_INTERVAL, - }, }; -use axum::{ - extract::{Extension, FromRequest, Path, Query}, - response::{IntoResponse, Response}, - routing::{get, post}, - Json, Router, -}; -use hyper::StatusCode; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::{Map, Value}; -use sql_builder::SqlBuilder; - -use ulid::Ulid; -use uuid::Uuid; - -const MAX_NB_OF_JOBS_IN_Q_PER_USER: i64 = 10; -const MAX_DURATION_LAST_1200: std::time::Duration = std::time::Duration::from_secs(900); pub fn workspaced_service() -> Router { Router::new() @@ -78,6 +65,10 @@ pub fn workspaced_service() -> Router { get(create_job_signature), ) .route("/result_by_id/:job_id/:node_id", get(get_result_by_id)) + .route( + "/get_flow/:job_id/:resume_id/:secret", + get(get_suspended_job_flow), + ) } pub fn global_service() -> Router { @@ -98,47 +89,120 @@ pub fn global_service() -> Router { "/cancel/:job_id/:resume_id/:secret", post(cancel_suspended_job), ) - .route( - "/get_flow/:job_id/:resume_id/:secret", - get(get_suspended_job_flow), +} + +async fn get_result_by_id( + Extension(db): Extension, + Query(ResultByIdQuery { skip_direct }): Query, + Path((w_id, flow_id, node_id)): Path<(String, String, String)>, +) -> windmill_common::error::JsonResult { + let res = windmill_queue::get_result_by_id(db, skip_direct, w_id, flow_id, node_id).await?; + Ok(Json(res)) +} + +async fn cancel_job_api( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, + Json(CancelJob { reason }): Json, +) -> error::Result { + let tx = user_db.begin(&authed).await?; + + let (mut tx, job_option) = + windmill_queue::cancel_job(&authed.username, reason, id, &w_id, tx).await?; + + if let Some(id) = job_option { + audit_log( + &mut tx, + &authed.username, + "jobs.cancel", + ActionKind::Delete, + &w_id, + Some(&id.to_string()), + None, ) + .await?; + tx.commit().await?; + Ok(id.to_string()) + } else { + let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?; + tx.commit().await?; + let err = match job_o { + Some(Job::CompletedJob(_)) => error::Error::BadRequest(format!( + "queued job id {} exists but is already completed and cannot be canceled", + id + )), + _ => error::Error::NotFound(format!("queued job id {} does not exist", id)), + }; + Err(err) + } } -#[derive(Debug, sqlx::FromRow, Serialize, Clone)] -pub struct QueuedJob { - pub workspace_id: String, - pub id: Uuid, - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub scheduled_for: chrono::DateTime, - pub running: bool, - pub script_hash: Option, - pub script_path: Option, - pub args: Option, - pub logs: Option, - pub raw_code: Option, - pub canceled: bool, - pub canceled_by: Option, - pub canceled_reason: Option, - pub last_ping: Option>, - pub job_kind: JobKind, - pub schedule_path: Option, - pub permissioned_as: String, - pub flow_status: Option, - pub raw_flow: Option, - pub is_flow_step: bool, - pub language: Option, - pub same_worker: bool, +pub async fn get_path_for_hash<'c>( + db: &mut Transaction<'c, Postgres>, + w_id: &str, + hash: i64, +) -> error::Result { + let path = sqlx::query_scalar!( + "select path from script where hash = $1 AND (workspace_id = $2 OR workspace_id = \ + 'starter')", + hash, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!( + "querying getting path for hash {hash} in {w_id}: {e}" + )) + })?; + Ok(path) } -impl QueuedJob { - pub fn script_path(&self) -> &str { - self.script_path - .as_ref() - .map(String::as_str) - .unwrap_or("NO_FLOW_PATH") +async fn get_job( + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::JsonResult { + let tx = db.begin().await?; + let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?; + let job = not_found_if_none(job_o, "Job", id.to_string())?; + tx.commit().await?; + Ok(Json(job)) +} + +#[derive(Deserialize)] +pub struct ResultByIdQuery { + pub skip_direct: bool, +} + +pub async fn get_job_by_id<'c>( + mut tx: Transaction<'c, Postgres>, + w_id: &str, + id: Uuid, +) -> error::Result<(Option, Transaction<'c, Postgres>)> { + let cjob_option = sqlx::query_as::<_, CompletedJob>( + "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .fetch_optional(&mut tx) + .await?; + let job_option = match cjob_option { + Some(job) => Some(Job::CompletedJob(job)), + None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob), + }; + if job_option.is_some() { + Ok((job_option, tx)) + } else { + // check if a job had been moved in-between queries + let cjob_option = sqlx::query_as::<_, CompletedJob>( + "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .fetch_optional(&mut tx) + .await?; + Ok((cjob_option.map(Job::CompletedJob), tx)) } } @@ -195,303 +259,6 @@ impl RunJobQuery { } } -pub async fn run_flow_by_path( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, flow_path)): Path<(String, StripPath)>, - axum::Json(args): axum::Json>>, - Query(run_query): Query, -) -> error::Result<(StatusCode, String)> { - let flow_path = flow_path.to_path(); - let mut tx = user_db.begin(&authed).await?; - let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; - let (uuid, tx) = push( - tx, - &w_id, - JobPayload::Flow(flow_path.to_string()), - args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - run_query.parent_job, - false, - false, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, uuid.to_string())) -} - -pub async fn run_job_by_path( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, script_path)): Path<(String, StripPath)>, - axum::Json(args): axum::Json>>, - Query(run_query): Query, -) -> error::Result<(StatusCode, String)> { - let script_path = script_path.to_path(); - let mut tx = user_db.begin(&authed).await?; - let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?; - let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; - - let (uuid, tx) = push( - tx, - &w_id, - job_payload, - args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - run_query.parent_job, - false, - false, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, uuid.to_string())) -} - -async fn run_wait_result( - authed: Authed, - Extension(user_db): Extension, - uuid: Uuid, - Path((w_id, _)): Path<(String, T)>, -) -> error::JsonResult { - let mut result = None; - for i in 0..48 { - let mut tx = user_db.clone().begin(&authed).await?; - - result = sqlx::query_scalar!( - "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", - uuid, - &w_id - ) - .fetch_optional(&mut tx) - .await? - .flatten(); - - if result.is_some() { - break; - } - let delay = if i < 10 { 100 } else { 500 }; - tokio::time::sleep(core::time::Duration::from_millis(delay)).await; - } - if let Some(result) = result { - Ok(Json(result)) - } else { - Err(Error::ExecutionErr("timeout after 20s".to_string())) - } -} - -pub async fn run_wait_result_job_by_path( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, script_path)): Path<(String, StripPath)>, - axum::Json(args): axum::Json>>, - Query(run_query): Query, -) -> error::JsonResult { - let script_path = script_path.to_path(); - let mut tx = user_db.clone().begin(&authed).await?; - let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?; - let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; - - let (uuid, tx) = push( - tx, - &w_id, - job_payload, - args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - run_query.parent_job, - false, - false, - ) - .await?; - tx.commit().await?; - - run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_path))).await -} - -pub async fn run_wait_result_job_by_hash( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, script_hash)): Path<(String, ScriptHash)>, - axum::Json(args): axum::Json>>, - Query(run_query): Query, -) -> error::JsonResult { - let hash = script_hash.0; - let mut tx = user_db.clone().begin(&authed).await?; - let path = get_path_for_hash(&mut tx, &w_id, hash).await?; - let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; - - let (uuid, tx) = push( - tx, - &w_id, - JobPayload::ScriptHash { hash: ScriptHash(hash), path }, - args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - run_query.parent_job, - false, - false, - ) - .await?; - tx.commit().await?; - - run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_hash))).await -} - -pub async fn script_path_to_payload<'c>( - script_path: &str, - db: &mut Transaction<'c, Postgres>, - w_id: &String, -) -> Result { - let job_payload = if script_path.starts_with("hub/") { - JobPayload::ScriptHub { path: script_path.to_owned() } - } else { - let script_hash = get_latest_hash_for_path(db, w_id, script_path).await?; - JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() } - }; - Ok(job_payload) -} - -pub async fn get_latest_hash_for_path<'c>( - db: &mut Transaction<'c, Postgres>, - w_id: &str, - script_path: &str, -) -> error::Result { - let script_hash_o = sqlx::query_scalar!( - "select hash from script where path = $1 AND (workspace_id = $2 OR workspace_id = \ - 'starter') AND - created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 OR \ - workspace_id = 'starter')) AND - deleted = false", - script_path, - w_id - ) - .fetch_optional(db) - .await?; - - let script_hash = crate::utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?; - - Ok(ScriptHash(script_hash)) -} - -pub async fn run_job_by_hash( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, script_hash)): Path<(String, ScriptHash)>, - axum::Json(args): axum::Json>>, - Query(run_query): Query, -) -> error::Result<(StatusCode, String)> { - let hash = script_hash.0; - let mut tx = user_db.begin(&authed).await?; - let path = get_path_for_hash(&mut tx, &w_id, hash).await?; - let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; - - let (uuid, tx) = push( - tx, - &w_id, - JobPayload::ScriptHash { hash: ScriptHash(hash), path }, - args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - run_query.parent_job, - false, - false, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, uuid.to_string())) -} - -pub async fn get_path_for_hash<'c>( - db: &mut Transaction<'c, Postgres>, - w_id: &str, - hash: i64, -) -> error::Result { - let path = sqlx::query_scalar!( - "select path from script where hash = $1 AND (workspace_id = $2 OR workspace_id = \ - 'starter')", - hash, - w_id - ) - .fetch_one(db) - .await - .map_err(|e| { - Error::InternalErr(format!( - "querying getting path for hash {hash} in {w_id}: {e}" - )) - })?; - Ok(path) -} - -async fn run_preview_job( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(preview): Json, - Query(sch_query): Query, -) -> error::Result<(StatusCode, String)> { - let mut tx = user_db.begin(&authed).await?; - let scheduled_for = sch_query.get_scheduled_for(&mut tx).await?; - - let (uuid, tx) = push( - tx, - &w_id, - JobPayload::Code(RawCode { - content: preview.content, - path: preview.path, - language: preview.language, - }), - preview.args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - None, - false, - false, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, uuid.to_string())) -} - -async fn run_preview_flow_job( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(raw_flow): Json, - Query(sch_query): Query, -) -> error::Result<(StatusCode, String)> { - let mut tx = user_db.begin(&authed).await?; - let scheduled_for = sch_query.get_scheduled_for(&mut tx).await?; - let (uuid, tx) = push( - tx, - &w_id, - JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path }, - raw_flow.args, - &authed.username, - owner_to_token_owner(&authed.username, false), - scheduled_for, - None, - None, - false, - false, - ) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, uuid.to_string())) -} #[derive(Deserialize)] pub struct ListQueueQuery { pub script_path_start: Option, @@ -565,7 +332,8 @@ async fn list_jobs( Query(pagination): Query, Query(lq): Query, ) -> error::JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); + // TODO: todo!("rewrite this to just run list_queue_jobs and list_completed_jobs separately and return as one"); + let (per_page, offset) = paginate(pagination); let lqc = lq.clone(); let sqlq = list_queue_jobs_query( &w_id, @@ -652,449 +420,6 @@ async fn list_jobs( tx.commit().await?; Ok(Json(jobs.into_iter().map(From::from).collect())) } -#[derive(Deserialize, Clone)] -pub struct ListCompletedQuery { - pub script_path_start: Option, - pub script_path_exact: Option, - pub script_hash: Option, - pub created_by: Option, - pub created_before: Option>, - pub created_after: Option>, - pub success: Option, - pub parent_job: Option, - pub order_desc: Option, - pub job_kinds: Option, - pub is_skipped: Option, - pub is_flow_step: Option, -} -fn list_completed_jobs_query( - w_id: &str, - per_page: usize, - offset: usize, - lq: &ListCompletedQuery, - fields: &[&str], -) -> SqlBuilder { - let mut sqlb = SqlBuilder::select_from("completed_job") - .fields(fields) - .order_by("created_at", lq.order_desc.unwrap_or(true)) - .and_where_eq("workspace_id", "?".bind(&w_id)) - .offset(offset) - .limit(per_page) - .clone(); - - if let Some(ps) = &lq.script_path_start { - sqlb.and_where_like_left("script_path", "?".bind(ps)); - } - if let Some(p) = &lq.script_path_exact { - sqlb.and_where_eq("script_path", "?".bind(p)); - } - if let Some(h) = &lq.script_hash { - sqlb.and_where_eq("script_hash", "?".bind(h)); - } - if let Some(cb) = &lq.created_by { - sqlb.and_where_eq("created_by", "?".bind(cb)); - } - if let Some(r) = &lq.success { - sqlb.and_where_eq("success", r); - } - if let Some(pj) = &lq.parent_job { - sqlb.and_where_eq("parent_job", "?".bind(pj)); - } - if let Some(dt) = &lq.created_before { - sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp())); - } - if let Some(dt) = &lq.created_after { - sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp())); - } - if let Some(sk) = &lq.is_skipped { - sqlb.and_where_eq("is_skipped", sk); - } - if let Some(fs) = &lq.is_flow_step { - sqlb.and_where_eq("is_flow_step", fs); - } - if let Some(jk) = &lq.job_kinds { - sqlb.and_where_in( - "job_kind", - &jk.split(',').into_iter().map(quote).collect::>(), - ); - } - - sqlb -} - -async fn list_completed_jobs( - Extension(db): Extension, - Path(w_id): Path, - Query(pagination): Query, - Query(lq): Query, -) -> error::JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); - - let sql = list_completed_jobs_query( - &w_id, - per_page, - offset, - &lq, - &[ - "id", - "workspace_id", - "parent_job", - "created_by", - "created_at", - "started_at", - "duration_ms", - "success", - "script_hash", - "script_path", - "args", - "result", - "null as logs", - "deleted", - "canceled", - "canceled_by", - "canceled_reason", - "job_kind", - "schedule_path", - "permissioned_as", - "null as raw_code", - "null as flow_status", - "null as raw_flow", - "is_flow_step", - "language", - "is_skipped", - ], - ) - .sql()?; - let jobs = sqlx::query_as::<_, CompletedJob>(&sql) - .fetch_all(&db) - .await?; - Ok(Json(jobs)) -} - -async fn get_completed_job( - Extension(db): Extension, - Path((w_id, id)): Path<(String, Uuid)>, -) -> error::JsonResult { - let job_o = sqlx::query_as::<_, CompletedJob>( - "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(&db) - .await?; - - let job = crate::utils::not_found_if_none(job_o, "Completed Job", id.to_string())?; - Ok(Json(job)) -} - -async fn get_completed_job_result( - Extension(db): Extension, - Path((w_id, id)): Path<(String, Uuid)>, -) -> error::JsonResult> { - let result_o = sqlx::query_scalar!( - "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", - id, - w_id, - ) - .fetch_optional(&db) - .await?; - - let result = crate::utils::not_found_if_none(result_o, "Completed Job", id.to_string())?; - Ok(Json(result)) -} - -async fn cancel_job_api( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, id)): Path<(String, Uuid)>, - Json(CancelJob { reason }): Json, -) -> error::Result { - let tx = user_db.begin(&authed).await?; - - let (mut tx, job_option) = cancel_job(&authed.username, reason, id, &w_id, tx).await?; - - if let Some(id) = job_option { - audit_log( - &mut tx, - &authed.username, - "jobs.cancel", - ActionKind::Delete, - &w_id, - Some(&id.to_string()), - None, - ) - .await?; - tx.commit().await?; - Ok(id.to_string()) - } else { - let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?; - tx.commit().await?; - let err = match job_o { - Some(Job::CompletedJob(_)) => error::Error::BadRequest(format!( - "queued job id {} exists but is already completed and cannot be canceled", - id - )), - _ => error::Error::NotFound(format!("queued job id {} does not exist", id)), - }; - Err(err) - } -} - -async fn cancel_job<'c>( - username: &str, - reason: Option, - id: Uuid, - w_id: &str, - mut tx: Transaction<'c, Postgres>, -) -> error::Result<(Transaction<'c, Postgres>, Option)> { - let job_option = sqlx::query_scalar!( - "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 \ - AND workspace_id = $4 RETURNING id", - username, - reason, - id, - w_id - ) - .fetch_optional(&mut tx) - .await?; - let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default(); - while !jobs.is_empty() { - let p_job = jobs.pop(); - let new_jobs = sqlx::query_scalar!( - "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2 WHERE parent_job = $3 \ - AND workspace_id = $4 RETURNING id", - username, - reason, - p_job, - w_id - ) - .fetch_all(&mut tx) - .await?; - jobs.extend(new_jobs); - } - Ok((tx, job_option)) -} - -async fn delete_completed_job( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, id)): Path<(String, Uuid)>, -) -> error::JsonResult { - let mut tx = user_db.begin(&authed).await?; - - require_admin(authed.is_admin, &authed.username)?; - let job_o = sqlx::query_as::<_, CompletedJob>( - "UPDATE completed_job SET logs = '', deleted = true WHERE id = $1 AND workspace_id = $2 \ - RETURNING *", - ) - .bind(id) - .bind(&w_id) - .fetch_optional(&mut tx) - .await?; - - let job = crate::utils::not_found_if_none(job_o, "Completed Job", id.to_string())?; - - audit_log( - &mut tx, - &authed.username, - "jobs.delete", - ActionKind::Delete, - &w_id, - Some(&id.to_string()), - None, - ) - .await?; - - tx.commit().await?; - Ok(Json(job)) -} - -#[derive(Deserialize)] -pub struct JobUpdateQuery { - pub running: bool, - pub log_offset: i32, -} - -#[derive(Serialize)] -pub struct JobUpdate { - pub running: Option, - pub completed: Option, - pub new_logs: Option, -} - -async fn get_job_update( - Extension(db): Extension, - Path((w_id, id)): Path<(String, Uuid)>, - Query(JobUpdateQuery { running, log_offset }): Query, -) -> error::JsonResult { - let mut tx = db.begin().await?; - - let logs = query_scalar!( - "SELECT substr(logs, $1) as logs FROM queue WHERE workspace_id = $2 AND id = $3", - log_offset, - &w_id, - &id - ) - .fetch_optional(&mut tx) - .await?; - - if let Some(logs) = logs { - tx.commit().await?; - Ok(Json(JobUpdate { - running: if !running { Some(true) } else { None }, - completed: None, - new_logs: logs, - })) - } else { - let logs = query_scalar!( - "SELECT substr(logs, $1) as logs FROM completed_job WHERE workspace_id = $2 AND id = \ - $3", - log_offset, - &w_id, - &id - ) - .fetch_optional(&mut tx) - .await?; - let logs = crate::utils::not_found_if_none(logs, "Job", id.to_string())?; - tx.commit().await?; - Ok(Json(JobUpdate { - running: Some(false), - completed: Some(true), - new_logs: logs, - })) - } -} - -async fn get_job( - Extension(db): Extension, - Path((w_id, id)): Path<(String, Uuid)>, -) -> error::JsonResult { - let tx = db.begin().await?; - let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?; - let job = crate::utils::not_found_if_none(job_o, "Job", id.to_string())?; - tx.commit().await?; - Ok(Json(job)) -} - -#[derive(Deserialize)] -pub struct ResultByIdQuery { - pub skip_direct: bool, -} - -async fn get_result_by_id( - Extension(db): Extension, - Query(ResultByIdQuery { mut skip_direct }): Query, - Path((w_id, flow_id, node_id)): Path<(String, String, String)>, -) -> error::JsonResult { - let mut result_id: Option = None; - let mut parent_id = Uuid::from_str(&flow_id).ok(); - while result_id.is_none() && parent_id.is_some() { - if !skip_direct { - let r = sqlx::query!( - "SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 ", - parent_id.unwrap(), - w_id, - ) - .fetch_optional(&db) - .await?; - if let Some(r) = r { - let value = r - .flow_status - .as_ref() - .ok_or_else(|| Error::InternalErr(format!("requiring a flow status value")))? - .to_owned(); - parent_id = r.parent_job; - let status_o = serde_json::from_value::(value).ok(); - result_id = status_o.and_then(|status| { - status - .modules - .iter() - .find(|m| m.id() == node_id) - .and_then(|m| m.job()) - }); - } else { - parent_id = None; - } - } else { - let q_parent = sqlx::query_scalar!( - "SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2", - parent_id.unwrap(), - w_id, - ) - .fetch_optional(&db) - .await? - .flatten(); - parent_id = q_parent; - skip_direct = false - } - } - let result_id = crate::utils::not_found_if_none( - result_id, - "Flow result by id", - format!("{}, {}", flow_id, node_id), - )?; - let value = sqlx::query_scalar!( - "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", - result_id, - w_id, - ) - .fetch_optional(&db) - .await? - .flatten() - .unwrap_or(serde_json::Value::Null); - Ok(Json(value)) -} - -pub async fn get_job_by_id<'c>( - mut tx: Transaction<'c, Postgres>, - w_id: &str, - id: Uuid, -) -> error::Result<(Option, Transaction<'c, Postgres>)> { - let cjob_option = sqlx::query_as::<_, CompletedJob>( - "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(&mut tx) - .await?; - let job_option = match cjob_option { - Some(job) => Some(Job::CompletedJob(job)), - None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob), - }; - if job_option.is_some() { - Ok((job_option, tx)) - } else { - // check if a job had been moved in-between queries - let cjob_option = sqlx::query_as::<_, CompletedJob>( - "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(&mut tx) - .await?; - Ok((cjob_option.map(Job::CompletedJob), tx)) - } -} - -#[derive(Deserialize)] -pub struct QueryApprover { - pub approver: Option, -} -pub async fn get_queued_job<'c>( - id: Uuid, - w_id: &str, - tx: &mut Transaction<'c, Postgres>, -) -> error::Result> { - let r = sqlx::query_as::<_, QueuedJob>( - "SELECT * - FROM queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(tx) - .await?; - Ok(r) -} pub async fn resume_suspended_job( /* unauthed */ @@ -1192,7 +517,7 @@ pub async fn cancel_suspended_job( let whom = approver.approver.unwrap_or_else(|| "unknown".to_string()); let parent_flow = get_root_job(db, &w_id, job).await?; - let (mut tx, job) = cancel_job( + let (mut tx, job) = windmill_queue::cancel_job( &whom, Some("approval request disapproved".to_string()), parent_flow, @@ -1240,6 +565,11 @@ pub struct SuspendedJobFlow { pub approvers: Vec, } +#[derive(Deserialize)] +pub struct QueryApprover { + pub approver: Option, +} + pub async fn get_suspended_job_flow( /* unauthed */ Extension(db): Extension, @@ -1274,7 +604,7 @@ pub async fn get_suspended_job_flow( .flatten() .ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?; let (flow_o, mut tx) = get_job_by_id(tx, &w_id, flow_id).await?; - let flow = crate::utils::not_found_if_none(flow_o, "Parent Flow", job.to_string())?; + let flow = not_found_if_none(flow_o, "Parent Flow", job.to_string())?; let flow_status = flow .flow_status() @@ -1353,19 +683,6 @@ impl Job { value.map(|v| serde_json::from_value(v).ok()).flatten() } } -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] -#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase"))] -pub enum JobKind { - Script, - #[allow(non_camel_case_types)] - Script_Hub, - Preview, - Dependencies, - Flow, - FlowPreview, - Identity, -} #[derive(sqlx::FromRow)] struct UnifiedJob { @@ -1462,18 +779,11 @@ struct CancelJob { reason: Option, } -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct RawCode { - pub content: String, - pub path: Option, - pub language: ScriptLang, -} - #[derive(Deserialize)] struct Preview { content: String, path: Option, - args: Option>, + args: Option>, language: ScriptLang, } @@ -1481,496 +791,7 @@ struct Preview { struct PreviewFlow { value: FlowValue, path: Option, - args: Option>, -} - -#[derive(Debug, Clone)] -pub enum JobPayload { - ScriptHub { path: String }, - ScriptHash { hash: ScriptHash, path: String }, - Code(RawCode), - Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang }, - Flow(String), - RawFlow { value: FlowValue, path: Option }, - Identity, -} - -lazy_static::lazy_static! { - // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens. - static ref QUEUE_PUSH_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( - "queue_push_count", - "Total number of jobs pushed to the queue." - ) - .unwrap(); - static ref QUEUE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( - "queue_delete_count", - "Total number of jobs deleted from the queue." - ) - .unwrap(); - static ref QUEUE_PULL_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( - "queue_pull_count", - "Total number of jobs pulled from the queue." - ) - .unwrap(); -} - -#[instrument(level = "trace", skip_all)] -pub async fn push<'c>( - mut tx: Transaction<'c, Postgres>, - workspace_id: &str, - job_payload: JobPayload, - args: Option>, - user: &str, - permissioned_as: String, - scheduled_for_o: Option>, - schedule_path: Option, - parent_job: Option, - is_flow_step: bool, - mut same_worker: bool, -) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { - let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now); - let args_json = args.map(serde_json::Value::Object); - let job_id: Uuid = Ulid::new().into(); - - let premium_workspace = - sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id) - .fetch_one(&mut tx) - .await - .map_err(|e| { - Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}")) - })?; - - if !premium_workspace && std::env::var("CLOUD_HOSTED").is_ok() { - let rate_limiting_queue = sqlx::query_scalar!( - "SELECT COUNT(id) FROM queue WHERE permissioned_as = $1 AND workspace_id = $2", - permissioned_as, - workspace_id - ) - .fetch_one(&mut tx) - .await?; - - if let Some(nb_jobs) = rate_limiting_queue { - if nb_jobs > MAX_NB_OF_JOBS_IN_Q_PER_USER { - return Err(error::Error::ExecutionErr(format!( - "You have exceeded the number of authorized elements of queue at any given \ - time: {}", - MAX_NB_OF_JOBS_IN_Q_PER_USER - ))); - } - } - - let rate_limiting_duration_ms = sqlx::query_scalar!( - " - SELECT SUM(duration_ms) - FROM completed_job - WHERE permissioned_as = $1 - AND created_at > NOW() - INTERVAL '1200 seconds' - AND workspace_id = $2", - permissioned_as, - workspace_id - ) - .fetch_one(&mut tx) - .await?; - - if let Some(sum_duration_ms) = rate_limiting_duration_ms { - if sum_duration_ms as u128 > MAX_DURATION_LAST_1200.as_millis() { - return Err(error::Error::ExecutionErr(format!( - "You have exceeded the scripts cumulative duration limit over the last 20m \ - which is: {} seconds", - MAX_DURATION_LAST_1200.as_secs() - ))); - } - } - } - - let (script_hash, script_path, raw_code, job_kind, raw_flow, language) = match job_payload { - JobPayload::ScriptHash { hash, path } => { - let language = sqlx::query_scalar!( - "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND \ - (workspace_id = $2 OR workspace_id = 'starter')", - hash.0, - workspace_id - ) - .fetch_one(&mut tx) - .await - .map_err(|e| { - Error::InternalErr(format!( - "fetching language for hash {hash} in {workspace_id}: {e}" - )) - })?; - ( - Some(hash.0), - Some(path), - None, - JobKind::Script, - None, - Some(language), - ) - } - JobPayload::ScriptHub { path } => { - let email = sqlx::query_scalar!( - "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", - user, - workspace_id - ) - .fetch_optional(&mut tx) - .await?; - let script = get_hub_script(path.clone(), email, user).await?; - ( - None, - Some(path), - Some(script.content.clone()), - JobKind::Script_Hub, - None, - Some(script.language.clone()), - ) - } - JobPayload::Code(RawCode { content, path, language }) => ( - None, - path, - Some(content), - JobKind::Preview, - None, - Some(language), - ), - JobPayload::Dependencies { hash, dependencies, language } => ( - Some(hash.0), - None, - Some(dependencies), - JobKind::Dependencies, - None, - Some(language), - ), - JobPayload::RawFlow { value, path } => { - (None, path, None, JobKind::FlowPreview, Some(value), None) - } - JobPayload::Flow(flow) => { - let value_json = sqlx::query_scalar!( - "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \ - 'starter')", - flow, - workspace_id - ) - .fetch_optional(&mut tx) - .await? - .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?; - let value = serde_json::from_value::(value_json).map_err(|err| { - Error::InternalErr(format!( - "could not convert json to flow for {flow}: {err:?}" - )) - })?; - (None, Some(flow), None, JobKind::Flow, Some(value), None) - } - JobPayload::Identity => (None, None, None, JobKind::Identity, None, None), - }; - - let is_running = same_worker; - if let Some(flow) = raw_flow.as_ref() { - same_worker = same_worker || flow.same_worker; - - for module in flow.modules.iter() { - if let Some(retry) = &module.retry { - if retry.max_attempts() > MAX_RETRY_ATTEMPTS { - Err(Error::BadRequest(format!( - "retry attempts exceeds the maximum of {MAX_RETRY_ATTEMPTS}" - )))? - } - - if matches!(retry.max_interval(), Some(interval) if interval > MAX_RETRY_INTERVAL) { - let max = MAX_RETRY_INTERVAL.as_secs(); - Err(Error::BadRequest(format!( - "retry interval exceeds the maximum of {max} seconds" - )))? - } - } - } - } - - let flow_status = raw_flow.as_ref().map(init_flow_status); - let uuid = sqlx::query_scalar!( - "INSERT INTO queue - (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, - script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, \ - flow_status, is_flow_step, language, started_at, same_worker) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, CASE WHEN $3 THEN now() END, $18) \ - RETURNING id", - workspace_id, - job_id, - is_running, - parent_job, - user, - permissioned_as, - scheduled_for, - script_hash, - script_path.clone(), - raw_code, - args_json, - job_kind: JobKind, - schedule_path, - raw_flow.map(|f| serde_json::json!(f)), - flow_status.map(|f| serde_json::json!(f)), - is_flow_step, - language: ScriptLang, - same_worker - ) - .fetch_one(&mut tx) - .await - .map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id}: {e}")))?; - // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction. - QUEUE_PUSH_COUNT.inc(); - - { - let uuid_string = job_id.to_string(); - let uuid_str = uuid_string.as_str(); - let mut hm = HashMap::from([("uuid", uuid_str), ("permissioned_as", &permissioned_as)]); - - let s: String; - let operation_name = match job_kind { - JobKind::Preview => "jobs.run.preview", - JobKind::Script => { - s = ScriptHash(script_hash.unwrap()).to_string(); - hm.insert("hash", s.as_str()); - "jobs.run.script" - } - JobKind::Flow => "jobs.run.flow", - JobKind::FlowPreview => "jobs.run.flow_preview", - JobKind::Script_Hub => "jobs.run.script_hub", - JobKind::Dependencies => "jobs.run.dependencies", - JobKind::Identity => "jobs.run.identity", - }; - - audit_log( - &mut tx, - &user, - operation_name, - ActionKind::Execute, - workspace_id, - script_path.as_ref().map(|x| x.as_str()), - Some(hm), - ) - .await?; - } - Ok((uuid, tx)) -} - -pub async fn get_hub_script( - path: String, - email: Option, - user: &str, -) -> error::Result { - get_full_hub_script_by_path( - Authed { email, username: user.to_string(), is_admin: false, groups: vec![] }, - Path(StripPath(path)), - Extension( - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .build() - .map_err(to_anyhow)?, - ), - Host(std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string())), - ) - .await - .map(|e| e.0) -} - -pub fn canceled_job_to_result(job: &QueuedJob) -> String { - let reason = job - .canceled_reason - .as_deref() - .unwrap_or_else(|| "no reason given"); - let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown"); - format!("Job canceled: {reason} by {canceler}") -} -#[instrument(level = "trace", skip_all)] -pub async fn add_completed_job_error( - db: &DB, - queued_job: &QueuedJob, - logs: String, - e: E, - metrics: Option, -) -> Result<(Uuid, Map), Error> { - metrics.map(|m| m.worker_execution_failed.inc()); - let mut output_map = serde_json::Map::new(); - output_map.insert( - "error".to_string(), - serde_json::Value::String(e.to_string()), - ); - let a = add_completed_job( - db, - &queued_job, - false, - false, - serde_json::Value::Object(output_map.clone()), - logs, - ) - .await?; - Ok((a, output_map)) -} - -#[instrument(level = "trace", skip_all)] -pub async fn add_completed_job( - db: &DB, - queued_job: &QueuedJob, - success: bool, - skipped: bool, - result: serde_json::Value, - logs: String, -) -> Result { - let mut tx = db.begin().await?; - let job_id = queued_job.id.clone(); - sqlx::query!( - "INSERT INTO completed_job AS cj - ( workspace_id - , id - , parent_job - , created_by - , created_at - , started_at - , duration_ms - , success - , script_hash - , script_path - , args - , result - , logs - , raw_code - , canceled - , canceled_by - , canceled_reason - , job_kind - , schedule_path - , permissioned_as - , flow_status - , raw_flow - , is_flow_step - , is_skipped - , language ) - VALUES ($1, $2, $3, $4, $5, $6, EXTRACT(milliseconds FROM (now() - $6)), $7, $8, $9,\ - $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) - ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)", - queued_job.workspace_id, - queued_job.id, - queued_job.parent_job, - queued_job.created_by, - queued_job.created_at, - queued_job.started_at, - success, - queued_job.script_hash.map(|x| x.0), - queued_job.script_path, - queued_job.args, - result, - logs, - queued_job.raw_code, - queued_job.canceled, - queued_job.canceled_by, - queued_job.canceled_reason, - queued_job.job_kind: JobKind, - queued_job.schedule_path, - queued_job.permissioned_as, - queued_job.flow_status, - queued_job.raw_flow, - queued_job.is_flow_step, - skipped, - queued_job.language: ScriptLang, - ) - .execute(&mut tx) - .await - .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?; - let _ = delete_job(db, &queued_job.workspace_id, job_id).await?; - if !queued_job.is_flow_step - && queued_job.job_kind != JobKind::Flow - && queued_job.job_kind != JobKind::FlowPreview - && queued_job.schedule_path.is_some() - && queued_job.script_path.is_some() - { - tx = schedule_again_if_scheduled( - tx, - queued_job.schedule_path.as_ref().unwrap(), - queued_job.script_path.as_ref().unwrap(), - &queued_job.workspace_id, - ) - .await?; - } - tx.commit().await?; - tracing::debug!("Added completed job {}", queued_job.id); - Ok(queued_job.id) -} - -#[instrument(level = "trace", skip_all)] -pub async fn schedule_again_if_scheduled<'c>( - mut tx: Transaction<'c, Postgres>, - schedule_path: &str, - script_path: &str, - w_id: &str, -) -> crate::error::Result> { - let schedule = get_schedule_opt(&mut tx, &w_id, schedule_path) - .await? - .ok_or_else(|| { - Error::InternalErr(format!( - "Could not find schedule {:?} for workspace {}", - schedule_path, w_id - )) - })?; - if schedule.enabled && script_path == schedule.script_path { - tx = crate::schedule::push_scheduled_job(tx, schedule).await?; - } - - Ok(tx) -} - -pub async fn pull(db: &DB) -> Result, crate::Error> { - /* Jobs can be started if they: - * - haven't been started before, - * running = false - * - are flows with a step that needed resume, - * suspend_until is non-null - * and suspend = 0 when the resume messages are received - * or suspend_until <= now() if it has timed out */ - let job: Option = sqlx::query_as::<_, QueuedJob>( - "UPDATE queue - SET running = true - , started_at = coalesce(started_at, now()) - , last_ping = now() - , suspend_until = null - WHERE id = ( - SELECT id - FROM queue - WHERE ( running = false - AND scheduled_for <= now()) - OR (suspend_until IS NOT NULL - AND ( suspend <= 0 - OR suspend_until <= now())) - ORDER BY scheduled_for - FOR UPDATE SKIP LOCKED - LIMIT 1 - ) - RETURNING *", - ) - .fetch_optional(db) - .await?; - - if job.is_some() { - QUEUE_PULL_COUNT.inc(); - } - - Ok(job) -} - -#[instrument(level = "trace", skip_all)] -pub async fn delete_job(db: &DB, w_id: &str, job_id: Uuid) -> Result<(), crate::Error> { - QUEUE_DELETE_COUNT.inc(); - let job_removed = sqlx::query_scalar!( - "DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", - w_id, - job_id - ) - .fetch_one(db) - .await - .map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))? - .unwrap_or(0) - == 1; - tracing::debug!("Job {job_id} deletion was achieved with success: {job_removed}"); - Ok(()) + args: Option>, } pub struct QueryOrBody(pub Option); @@ -1987,7 +808,7 @@ where async fn from_request( req: &mut axum::extract::RequestParts, - ) -> Result { + ) -> std::result::Result { return if req.method() == axum::http::Method::GET { let Query(InPayload { payload }) = Query::from_request(req) .await @@ -2018,3 +839,498 @@ where } } } +pub async fn run_flow_by_path( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, flow_path)): Path<(String, StripPath)>, + axum::Json(args): axum::Json>>, + Query(run_query): Query, +) -> error::Result<(StatusCode, String)> { + let flow_path = flow_path.to_path(); + let mut tx = user_db.begin(&authed).await?; + let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; + let (uuid, tx) = push( + tx, + &w_id, + JobPayload::Flow(flow_path.to_string()), + args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + run_query.parent_job, + false, + false, + ) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, uuid.to_string())) +} + +pub async fn run_job_by_path( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, script_path)): Path<(String, StripPath)>, + axum::Json(args): axum::Json>>, + Query(run_query): Query, +) -> error::Result<(StatusCode, String)> { + let script_path = script_path.to_path(); + let mut tx = user_db.begin(&authed).await?; + let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?; + let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; + + let (uuid, tx) = push( + tx, + &w_id, + job_payload, + args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + run_query.parent_job, + false, + false, + ) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, uuid.to_string())) +} + +async fn run_wait_result( + authed: Authed, + Extension(user_db): Extension, + uuid: Uuid, + Path((w_id, _)): Path<(String, T)>, +) -> error::JsonResult { + let mut result = None; + for i in 0..48 { + let mut tx = user_db.clone().begin(&authed).await?; + + result = sqlx::query_scalar!( + "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", + uuid, + &w_id + ) + .fetch_optional(&mut tx) + .await? + .flatten(); + + if result.is_some() { + break; + } + let delay = if i < 10 { 100 } else { 500 }; + tokio::time::sleep(core::time::Duration::from_millis(delay)).await; + } + if let Some(result) = result { + Ok(Json(result)) + } else { + Err(Error::ExecutionErr("timeout after 20s".to_string())) + } +} + +pub async fn run_wait_result_job_by_path( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, script_path)): Path<(String, StripPath)>, + axum::Json(args): axum::Json>>, + Query(run_query): Query, +) -> error::JsonResult { + let script_path = script_path.to_path(); + let mut tx = user_db.clone().begin(&authed).await?; + let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?; + let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; + + let (uuid, tx) = push( + tx, + &w_id, + job_payload, + args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + run_query.parent_job, + false, + false, + ) + .await?; + tx.commit().await?; + + run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_path))).await +} + +pub async fn run_wait_result_job_by_hash( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, script_hash)): Path<(String, ScriptHash)>, + axum::Json(args): axum::Json>>, + Query(run_query): Query, +) -> error::JsonResult { + let hash = script_hash.0; + let mut tx = user_db.clone().begin(&authed).await?; + let path = get_path_for_hash(&mut tx, &w_id, hash).await?; + let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; + + let (uuid, tx) = push( + tx, + &w_id, + JobPayload::ScriptHash { hash: ScriptHash(hash), path }, + args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + run_query.parent_job, + false, + false, + ) + .await?; + tx.commit().await?; + + run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_hash))).await +} + +// a similar function exists on the worker +pub async fn script_path_to_payload<'c>( + script_path: &str, + db: &mut Transaction<'c, Postgres>, + w_id: &String, +) -> std::result::Result { + let job_payload = if script_path.starts_with("hub/") { + JobPayload::ScriptHub { path: script_path.to_owned() } + } else { + let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?; + JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() } + }; + Ok(job_payload) +} + +async fn run_preview_job( + authed: Authed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(preview): Json, + Query(sch_query): Query, +) -> error::Result<(StatusCode, String)> { + let mut tx = user_db.begin(&authed).await?; + let scheduled_for = sch_query.get_scheduled_for(&mut tx).await?; + + let (uuid, tx) = push( + tx, + &w_id, + JobPayload::Code(RawCode { + content: preview.content, + path: preview.path, + language: preview.language, + }), + preview.args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + None, + false, + false, + ) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, uuid.to_string())) +} + +async fn run_preview_flow_job( + authed: Authed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(raw_flow): Json, + Query(sch_query): Query, +) -> error::Result<(StatusCode, String)> { + let mut tx = user_db.begin(&authed).await?; + let scheduled_for = sch_query.get_scheduled_for(&mut tx).await?; + let (uuid, tx) = push( + tx, + &w_id, + JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path }, + raw_flow.args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + None, + false, + false, + ) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, uuid.to_string())) +} + +pub async fn run_job_by_hash( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, script_hash)): Path<(String, ScriptHash)>, + axum::Json(args): axum::Json>>, + Query(run_query): Query, +) -> error::Result<(StatusCode, String)> { + let hash = script_hash.0; + let mut tx = user_db.begin(&authed).await?; + let path = get_path_for_hash(&mut tx, &w_id, hash).await?; + let scheduled_for = run_query.get_scheduled_for(&mut tx).await?; + + let (uuid, tx) = push( + tx, + &w_id, + JobPayload::ScriptHash { hash: ScriptHash(hash), path }, + args, + &authed.username, + owner_to_token_owner(&authed.username, false), + scheduled_for, + None, + run_query.parent_job, + false, + false, + ) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, uuid.to_string())) +} + +#[derive(Deserialize)] +pub struct JobUpdateQuery { + pub running: bool, + pub log_offset: i32, +} + +#[derive(Serialize)] +pub struct JobUpdate { + pub running: Option, + pub completed: Option, + pub new_logs: Option, +} + +async fn get_job_update( + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, + Query(JobUpdateQuery { running, log_offset }): Query, +) -> error::JsonResult { + let mut tx = db.begin().await?; + + let logs = query_scalar!( + "SELECT substr(logs, $1) as logs FROM queue WHERE workspace_id = $2 AND id = $3", + log_offset, + &w_id, + &id + ) + .fetch_optional(&mut tx) + .await?; + + if let Some(logs) = logs { + tx.commit().await?; + Ok(Json(JobUpdate { + running: if !running { Some(true) } else { None }, + completed: None, + new_logs: logs, + })) + } else { + let logs = query_scalar!( + "SELECT substr(logs, $1) as logs FROM completed_job WHERE workspace_id = $2 AND id = \ + $3", + log_offset, + &w_id, + &id + ) + .fetch_optional(&mut tx) + .await?; + let logs = not_found_if_none(logs, "Job", id.to_string())?; + tx.commit().await?; + Ok(Json(JobUpdate { + running: Some(false), + completed: Some(true), + new_logs: logs, + })) + } +} + +fn list_completed_jobs_query( + w_id: &str, + per_page: usize, + offset: usize, + lq: &ListCompletedQuery, + fields: &[&str], +) -> SqlBuilder { + let mut sqlb = SqlBuilder::select_from("completed_job") + .fields(fields) + .order_by("created_at", lq.order_desc.unwrap_or(true)) + .and_where_eq("workspace_id", "?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + + if let Some(ps) = &lq.script_path_start { + sqlb.and_where_like_left("script_path", "?".bind(ps)); + } + if let Some(p) = &lq.script_path_exact { + sqlb.and_where_eq("script_path", "?".bind(p)); + } + if let Some(h) = &lq.script_hash { + sqlb.and_where_eq("script_hash", "?".bind(h)); + } + if let Some(cb) = &lq.created_by { + sqlb.and_where_eq("created_by", "?".bind(cb)); + } + if let Some(r) = &lq.success { + sqlb.and_where_eq("success", r); + } + if let Some(pj) = &lq.parent_job { + sqlb.and_where_eq("parent_job", "?".bind(pj)); + } + if let Some(dt) = &lq.created_before { + sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp())); + } + if let Some(dt) = &lq.created_after { + sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp())); + } + if let Some(sk) = &lq.is_skipped { + sqlb.and_where_eq("is_skipped", sk); + } + if let Some(fs) = &lq.is_flow_step { + sqlb.and_where_eq("is_flow_step", fs); + } + if let Some(jk) = &lq.job_kinds { + sqlb.and_where_in( + "job_kind", + &jk.split(',').into_iter().map(quote).collect::>(), + ); + } + + sqlb +} +#[derive(Deserialize, Clone)] +pub struct ListCompletedQuery { + pub script_path_start: Option, + pub script_path_exact: Option, + pub script_hash: Option, + pub created_by: Option, + pub created_before: Option>, + pub created_after: Option>, + pub success: Option, + pub parent_job: Option, + pub order_desc: Option, + pub job_kinds: Option, + pub is_skipped: Option, + pub is_flow_step: Option, +} +async fn list_completed_jobs( + Extension(db): Extension, + Path(w_id): Path, + Query(pagination): Query, + Query(lq): Query, +) -> error::JsonResult> { + let (per_page, offset) = paginate(pagination); + + let sql = list_completed_jobs_query( + &w_id, + per_page, + offset, + &lq, + &[ + "id", + "workspace_id", + "parent_job", + "created_by", + "created_at", + "started_at", + "duration_ms", + "success", + "script_hash", + "script_path", + "args", + "result", + "null as logs", + "deleted", + "canceled", + "canceled_by", + "canceled_reason", + "job_kind", + "schedule_path", + "permissioned_as", + "null as raw_code", + "null as flow_status", + "null as raw_flow", + "is_flow_step", + "language", + "is_skipped", + ], + ) + .sql()?; + let jobs = sqlx::query_as::<_, CompletedJob>(&sql) + .fetch_all(&db) + .await?; + Ok(Json(jobs)) +} + +async fn get_completed_job( + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::JsonResult { + let job_o = sqlx::query_as::<_, CompletedJob>( + "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .fetch_optional(&db) + .await?; + + let job = not_found_if_none(job_o, "Completed Job", id.to_string())?; + Ok(Json(job)) +} + +async fn get_completed_job_result( + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::JsonResult> { + let result_o = sqlx::query_scalar!( + "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", + id, + w_id, + ) + .fetch_optional(&db) + .await?; + + let result = not_found_if_none(result_o, "Completed Job", id.to_string())?; + Ok(Json(result)) +} + +async fn delete_completed_job( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::JsonResult { + let mut tx = user_db.begin(&authed).await?; + + require_admin(authed.is_admin, &authed.username)?; + let job_o = sqlx::query_as::<_, CompletedJob>( + "UPDATE completed_job SET logs = '', deleted = true WHERE id = $1 AND workspace_id = $2 \ + RETURNING *", + ) + .bind(id) + .bind(&w_id) + .fetch_optional(&mut tx) + .await?; + + let job = not_found_if_none(job_o, "Completed Job", id.to_string())?; + + audit_log( + &mut tx, + &authed.username, + "jobs.delete", + ActionKind::Delete, + &w_id, + Some(&id.to_string()), + None, + ) + .await?; + + tx.commit().await?; + Ok(Json(job)) +} diff --git a/backend/src/lib.rs b/backend/windmill-api/src/lib.rs similarity index 58% rename from backend/src/lib.rs rename to backend/windmill-api/src/lib.rs index 97c4463368..b7db777ab4 100644 --- a/backend/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -6,41 +6,31 @@ * LICENSE-AGPL for a copy of the license. */ -use anyhow::Context; use argon2::Argon2; use axum::{handler::Handler, middleware::from_extractor, routing::get, Extension, Router}; use db::DB; -use futures::FutureExt; use git_version::git_version; use std::{net::SocketAddr, sync::Arc}; use tower::ServiceBuilder; use tower_cookies::CookieManagerLayer; use tower_http::trace::TraceLayer; +use windmill_common::{error::to_anyhow, utils::rd_string}; -extern crate magic_crypt; - -extern crate dotenv; +use crate::{ + db::UserDB, + oauth2::{build_oauth_clients, SlackVerifier}, + tracing_init::{MyMakeSpan, MyOnResponse}, + users::Authed, +}; mod audit; mod capture; -mod client; mod db; -mod error; -mod external_ip; mod flows; mod granular_acls; mod groups; -mod jobs; -mod js_eval; -mod more_serde; +pub mod jobs; mod oauth2; -mod parser; -mod parser_go; -mod parser_go_ast; -mod parser_go_scanner; -mod parser_go_token; -mod parser_py; -mod parser_ts; mod resources; mod schedule; mod scripts; @@ -49,51 +39,17 @@ mod tracing_init; mod users; mod utils; mod variables; -mod worker; -mod worker_flow; mod worker_ping; mod workspaces; -use error::Error; - -use crate::{ - db::UserDB, - error::to_anyhow, - oauth2::{build_oauth_clients, SlackVerifier}, - tracing_init::{MyMakeSpan, MyOnResponse}, - utils::rd_string, -}; - -pub use crate::tracing_init::initialize_tracing; -pub use crate::worker::WorkerConfig; - const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); -pub const DEFAULT_NUM_WORKERS: usize = 3; -pub const DEFAULT_TIMEOUT: i32 = 300; -pub const DEFAULT_SLEEP_QUEUE: u64 = 50; -pub const DEFAULT_MAX_CONNECTIONS: u32 = 100; - -pub async fn migrate_db(db: &DB) -> anyhow::Result<()> { - db::migrate(db).await?; - Ok(()) -} - -pub async fn connect_db() -> anyhow::Result { - let database_url = std::env::var("DATABASE_URL") - .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?; - - let max_connections = match std::env::var("DATABASE_CONNECTIONS") { - Ok(n) => n.parse::().context("invalid DATABASE_CONNECTIONS")?, - Err(_) => DEFAULT_MAX_CONNECTIONS, - }; - - Ok(db::connect(&database_url, max_connections).await?) -} struct BaseUrl(String); struct IsSecure(bool); struct CloudHosted(bool); +pub use users::delete_expired_items_perdiodically; + pub async fn run_server( db: DB, addr: SocketAddr, @@ -168,7 +124,7 @@ pub async fn run_server( .nest("/scripts", scripts::global_service()) .nest("/flows", flows::global_service()) .nest("/schedules", schedule::global_service()) - .route_layer(from_extractor::()) + .route_layer(from_extractor::()) .route_layer(from_extractor::()) .nest("/w/:workspace_id/jobs", jobs::global_service()) .nest("/w/:workspace_id/capture", capture::global_service()) @@ -202,64 +158,6 @@ pub async fn run_server( Ok(()) } -pub fn monitor_db(db: &DB, timeout: i32, rx: tokio::sync::broadcast::Receiver<()>) { - let db1 = db.clone(); - let db2 = db.clone(); - - let rx2 = rx.resubscribe(); - - tokio::spawn(async move { worker::handle_zombie_jobs_periodically(&db1, timeout, rx).await }); - tokio::spawn(async move { users::delete_expired_items_perdiodically(&db2, rx2).await }); -} - -pub async fn run_workers( - db: DB, - addr: SocketAddr, - timeout: i32, - num_workers: i32, - sleep_queue: u64, - worker_config: WorkerConfig, - rx: tokio::sync::broadcast::Receiver<()>, -) -> anyhow::Result<()> { - let instance_name = rd_string(5); - let monitor = tokio_metrics::TaskMonitor::new(); - - let ip = external_ip::get_ip().await.unwrap_or_else(|e| { - tracing::warn!(error = e.to_string(), "failed to get external IP"); - "unretrievable IP".to_string() - }); - - let mut handles = Vec::with_capacity(num_workers as usize); - - for i in 1..(num_workers + 1) { - let db1 = db.clone(); - let instance_name = instance_name.clone(); - let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5)); - let ip = ip.clone(); - let rx = rx.resubscribe(); - let worker_config = worker_config.clone(); - handles.push(tokio::spawn(monitor.instrument(async move { - tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker"); - worker::run_worker( - &db1, - timeout, - &instance_name, - worker_name, - i as u64, - num_workers as u64, - &ip, - sleep_queue, - worker_config, - rx, - ) - .await - }))); - } - - futures::future::try_join_all(handles).await?; - Ok(()) -} - async fn git_v() -> &'static str { GIT_VERSION } @@ -267,44 +165,7 @@ async fn git_v() -> &'static str { async fn openapi() -> &'static str { include_str!("../openapi.yaml") } - -pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> { - use std::io; - use tokio::signal::unix::SignalKind; - - async fn terminate() -> io::Result<()> { - tokio::signal::unix::signal(SignalKind::terminate())? - .recv() - .await; - Ok(()) - } - - tokio::select! { - _ = terminate() => {}, - _ = tokio::signal::ctrl_c() => {}, - } - println!("signal received, starting graceful shutdown"); - let _ = tx.send(()); +pub async fn migrate_db(db: &DB) -> anyhow::Result<()> { + db::migrate(db).await?; Ok(()) } - -pub async fn serve_metrics( - addr: SocketAddr, - mut rx: tokio::sync::broadcast::Receiver<()>, -) -> Result<(), hyper::Error> { - axum::Server::bind(&addr) - .serve( - Router::new() - .route("/metrics", get(metrics)) - .into_make_service(), - ) - .with_graceful_shutdown(rx.recv().map(drop)) - .await -} - -async fn metrics() -> Result { - let metric_families = prometheus::gather(); - Ok(prometheus::TextEncoder::new() - .encode_to_string(&metric_families) - .map_err(anyhow::Error::from)?) -} diff --git a/backend/windmill-api/src/main.rs b/backend/windmill-api/src/main.rs new file mode 100644 index 0000000000..46fa33a6b4 --- /dev/null +++ b/backend/windmill-api/src/main.rs @@ -0,0 +1,70 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::net::SocketAddr; + +use anyhow::Ok; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + windmill_common::tracing_init::initialize_tracing(); + + let db = windmill_common::connect_db().await?; + + let num_workers = std::env::var("NUM_WORKERS") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32); + + let metrics_addr: Option = std::env::var("METRICS_ADDR") + .ok() + .map(|s| { + s.parse::() + .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001)))) + .or_else(|_| s.parse::().map(Some)) + }) + .transpose()? + .flatten(); + + let server_mode = !std::env::var("DISABLE_SERVER") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + + if server_mode { + windmill_api::migrate_db(&db).await?; + } + + let (tx, rx) = tokio::sync::broadcast::channel::<()>(3); + let shutdown_signal = windmill_common::shutdown_signal(tx); + + let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()); + + if server_mode || num_workers > 0 { + let addr = SocketAddr::from(([0, 0, 0, 0], 8000)); + + let server_f = async { + if server_mode { + windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?; + } + Ok(()) as anyhow::Result<()> + }; + + let metrics_f = async { + match metrics_addr { + Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe()) + .await + .map_err(anyhow::Error::from), + None => Ok(()), + } + }; + + futures::try_join!(shutdown_signal, server_f, metrics_f)?; + } + Ok(()) +} diff --git a/backend/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs similarity index 97% rename from backend/src/oauth2.rs rename to backend/windmill-api/src/oauth2.rs index e2e2d00322..79b21c9230 100644 --- a/backend/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -1,3 +1,11 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + use std::{collections::HashMap, fmt::Debug}; use std::sync::Arc; @@ -10,6 +18,7 @@ use axum::{ routing::{get, post}, Json, Router, }; +use hmac::Mac; use hyper::StatusCode; use itertools::Itertools; @@ -19,29 +28,24 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sqlx::{Postgres, Transaction}; use tokio::{fs::File, io::AsyncReadExt}; use tower_cookies::{Cookie, Cookies}; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::utils::{not_found_if_none, now_from_db}; -use crate::utils::now_from_db; +use crate::users::Authed; use crate::IsSecure; use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{self, to_anyhow, Error, Result}, - jobs, - jobs::{get_latest_hash_for_path, JobPayload}, - users::Authed, - utils::not_found_if_none, variables::{build_crypt, encrypt}, workspaces::WorkspaceSettings, BaseUrl, }; +use windmill_common::error::{self, to_anyhow, Error, Result}; +use windmill_common::oauth2::*; + +use windmill_queue::JobPayload; use std::str; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -pub type HmacSha256 = Hmac; - pub fn global_service() -> Router { Router::new() .route("/login/:client", get(login)) @@ -97,10 +101,11 @@ pub struct AllClients { pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result { let connect_configs = serde_json::from_str::>(include_str!( - "../oauth_connect.json" + "../../oauth_connect.json" + ))?; + let login_configs = serde_json::from_str::>(include_str!( + "../../oauth_login.json" ))?; - let login_configs = - serde_json::from_str::>(include_str!("../oauth_login.json"))?; let mut content = String::new(); let path = "./oauth.json"; @@ -665,7 +670,8 @@ async fn slack_command( if let Some(settings) = settings { if let Some(script) = &settings.slack_command_script { let script_hash = - get_latest_hash_for_path(&mut tx, &settings.workspace_id, script).await?; + windmill_common::get_latest_hash_for_path(&mut tx, &settings.workspace_id, script) + .await?; let mut map = serde_json::Map::new(); map.insert("text".to_string(), serde_json::Value::String(form.text)); map.insert( @@ -673,7 +679,7 @@ async fn slack_command( serde_json::Value::String(form.response_url), ); - let (uuid, tx) = jobs::push( + let (uuid, tx) = windmill_queue::push( tx, &settings.workspace_id, JobPayload::ScriptHash { hash: script_hash, path: script.to_owned() }, diff --git a/backend/src/resources.rs b/backend/windmill-api/src/resources.rs similarity index 96% rename from backend/src/resources.rs rename to backend/windmill-api/src/resources.rs index dd999b3a02..f1007f7487 100644 --- a/backend/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -7,11 +7,8 @@ */ use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{Error, JsonResult, Result}, users::Authed, - utils::{require_admin, Pagination, StripPath}, }; use axum::{ extract::{Extension, Path, Query}, @@ -22,6 +19,11 @@ use hyper::StatusCode; use serde::{Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::FromRow; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{Error, JsonResult, Result}, + utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, +}; pub fn workspaced_service() -> Router { Router::new() @@ -99,7 +101,7 @@ async fn list_resources( Extension(user_db): Extension, Path(w_id): Path, ) -> JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); + let (per_page, offset) = paginate(pagination); let mut sqlb = SqlBuilder::select_from("resource") .fields(&[ @@ -150,7 +152,7 @@ async fn get_resource( .await?; tx.commit().await?; - let resource = crate::utils::not_found_if_none(resource_o, "Resource", path)?; + let resource = not_found_if_none(resource_o, "Resource", path)?; Ok(Json(resource)) } @@ -190,7 +192,7 @@ async fn get_resource_value( .await?; tx.commit().await?; - let value = crate::utils::not_found_if_none(value_o, "Resource", path)?; + let value = not_found_if_none(value_o, "Resource", path)?; Ok(Json(value)) } @@ -294,7 +296,7 @@ async fn update_resource( let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?; - let npath = crate::utils::not_found_if_none(npath_o, "Resource", path)?; + let npath = not_found_if_none(npath_o, "Resource", path)?; audit_log( &mut tx, @@ -360,7 +362,7 @@ async fn get_resource_type( .await?; tx.commit().await?; - let resource_type = crate::utils::not_found_if_none(resource_type_o, "ResourceType", name)?; + let resource_type = not_found_if_none(resource_type_o, "ResourceType", name)?; Ok(Json(resource_type)) } diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs new file mode 100644 index 0000000000..36e9d74511 --- /dev/null +++ b/backend/windmill-api/src/schedule.rs @@ -0,0 +1,126 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::{ + db::{UserDB, DB}, + users::Authed, +}; +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::DateTime; +use windmill_common::{ + error::{JsonResult, Result}, + utils::{not_found_if_none, Pagination, StripPath}, +}; +use windmill_queue::{ + self, + schedule::{EditSchedule, NewSchedule, PreviewPayload, Schedule, SetEnabled}, +}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_schedule)) + .route("/get/*path", get(get_schedule)) + .route("/exists/*path", get(exists_schedule)) + .route("/create", post(create_schedule)) + .route("/update/*path", post(edit_schedule)) + .route("/delete/*path", delete(delete_schedule)) + .route("/setenabled/*path", post(set_enabled)) +} + +pub fn global_service() -> Router { + Router::new().route("/preview", post(preview_schedule)) +} + +async fn create_schedule( + authed: Authed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(ns): Json, +) -> Result { + let tx = user_db.begin(&authed).await?; + let res = windmill_queue::schedule::create_schedule(tx, w_id, ns, &authed.username).await?; + Ok(res) +} + +async fn edit_schedule( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(es): Json, +) -> Result { + let tx = user_db.begin(&authed).await?; + let res = windmill_queue::schedule::edit_schedule(tx, w_id, path, es, &authed.username).await?; + Ok(res) +} + +async fn list_schedule( + authed: Authed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> JsonResult> { + let tx = user_db.begin(&authed).await?; + let res = windmill_queue::schedule::list_schedule(tx, w_id, pagination).await?; + Ok(Json(res)) +} + +async fn get_schedule( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + let schedule_o = windmill_queue::schedule::get_schedule_opt(&mut tx, &w_id, path).await?; + let schedule = not_found_if_none(schedule_o, "Schedule", path)?; + tx.commit().await?; + Ok(Json(schedule)) +} + +async fn exists_schedule( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let mut tx = db.begin().await?; + let res = windmill_queue::schedule::exists_schedule(&mut tx, w_id, path).await?; + tx.commit().await?; + Ok(Json(res)) +} + +pub async fn preview_schedule( + Json(payload): Json, +) -> JsonResult>> { + Ok(Json(windmill_queue::schedule::preview_schedule(payload)?)) +} + +pub async fn set_enabled( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> Result { + let tx = user_db.begin(&authed).await?; + let res = + windmill_queue::schedule::set_enabled(tx, w_id, path, payload, &authed.username).await?; + Ok(res) +} + +async fn delete_schedule( + authed: Authed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result { + let tx = user_db.begin(&authed).await?; + let res = windmill_queue::schedule::delete_schedule(tx, w_id, path, &authed.username).await?; + Ok(res) +} diff --git a/backend/src/scripts.rs b/backend/windmill-api/src/scripts.rs similarity index 73% rename from backend/src/scripts.rs rename to backend/windmill-api/src/scripts.rs index 730932adef..9b3056f1b0 100644 --- a/backend/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -7,16 +7,12 @@ */ use reqwest::Client; -use serde::Deserializer; use sql_builder::prelude::*; +use windmill_audit::{audit_log, ActionKind}; use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{to_anyhow, Error, JsonResult, Result}, - jobs, parser, parser_go, parser_py, parser_ts, - users::{owner_to_token_owner, truncate_token, Authed, Tokened}, - utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath}, + users::{truncate_token, Authed, Tokened}, }; use axum::{ extract::{Extension, Host, Path, Query}, @@ -24,15 +20,25 @@ use axum::{ Json, Router, }; use hyper::StatusCode; -use serde::{de::Error as _, ser::SerializeSeq, Deserialize, Serialize}; -use serde_json::{json, to_string_pretty}; +use serde::Serialize; +use serde_json::json; use sql_builder::SqlBuilder; use sqlx::{FromRow, Postgres, Transaction}; use std::{ collections::hash_map::DefaultHasher, - fmt::Display, hash::{Hash, Hasher}, }; +use windmill_common::{ + error::{Error, JsonResult, Result}, + scripts::{ + to_i64, HubScript, ListScriptQuery, NewScript, Script, ScriptHash, ScriptKind, ScriptLang, + }, + users::owner_to_token_owner, + utils::{ + list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath, + }, +}; +use windmill_queue; const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20; @@ -63,145 +69,6 @@ pub fn workspaced_service() -> Router { .route("/raw/h/:hash", get(raw_script_by_hash)) .route("/deployment_status/h/:hash", get(get_deployment_status)) } - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Hash)] -#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum ScriptLang { - Deno, - Python3, - Go, -} - -impl ScriptLang { - pub fn as_str(&self) -> &'static str { - match self { - ScriptLang::Deno => "deno", - ScriptLang::Python3 => "python3", - ScriptLang::Go => "go", - } - } -} - -#[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)] -#[sqlx(transparent)] -pub struct ScriptHash(pub i64); - -#[derive(sqlx::Type, PartialEq)] -#[sqlx(transparent)] -pub struct ScriptHashes(Vec); - -impl Display for ScriptHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", to_hex_string(&self.0)) - } -} -impl Serialize for ScriptHash { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - serializer.serialize_str(to_hex_string(&self.0).as_str()) - } -} -impl<'de> Deserialize<'de> for ScriptHash { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - let i = to_i64(&s).map_err(|e| D::Error::custom(format!("{}", e)))?; - Ok(ScriptHash(i)) - } -} - -impl Serialize for ScriptHashes { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.0.len()))?; - for element in &self.0 { - seq.serialize_element(&ScriptHash(*element))?; - } - seq.end() - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Hash)] -#[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum ScriptKind { - Trigger, - Failure, - Script, - Approval, -} - -#[derive(FromRow, Serialize)] -pub struct Script { - pub workspace_id: String, - pub hash: ScriptHash, - pub path: String, - pub parent_hashes: Option, - pub summary: String, - pub description: String, - pub content: String, - pub created_by: String, - pub created_at: chrono::DateTime, - pub archived: bool, - pub schema: Option, - pub deleted: bool, - pub is_template: bool, - pub extra_perms: serde_json::Value, - pub lock: Option, - pub lock_error_logs: Option, - pub language: ScriptLang, - pub kind: ScriptKind, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug)] -#[sqlx(transparent)] -#[serde(transparent)] -pub struct Schema(pub serde_json::Value); - -impl Hash for Schema { - fn hash(&self, state: &mut H) { - if let Ok(s) = to_string_pretty(&self.0) { - s.hash(state); - } - } -} - -#[derive(Serialize, Deserialize, Hash)] -pub struct NewScript { - pub path: String, - pub parent_hash: Option, - pub summary: String, - pub description: String, - pub content: String, - pub schema: Option, - pub is_template: Option, - pub lock: Option>, - pub language: ScriptLang, - pub kind: Option, -} - -#[derive(Deserialize)] -pub struct ListScriptQuery { - pub path_start: Option, - pub path_exact: Option, - pub created_by: Option, - pub first_parent_hash: Option, - pub last_parent_hash: Option, - pub parent_hash: Option, - pub show_archived: Option, - pub order_by: Option, - pub order_desc: Option, - pub is_template: Option, - pub kind: Option, -} - async fn list_scripts( authed: Authed, Extension(user_db): Extension, @@ -209,7 +76,7 @@ async fn list_scripts( Query(pagination): Query, Query(lq): Query, ) -> JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); + let (per_page, offset) = paginate(pagination); let mut sqlb = SqlBuilder::select_from("script as o") .fields(&[ @@ -443,13 +310,15 @@ async fn create_script( let mut tx = if ns.lock.is_none() && ns.language != ScriptLang::Deno { let dependencies = match ns.language { - ScriptLang::Python3 => parser_py::parse_python_imports(&ns.content)?.join("\n"), + ScriptLang::Python3 => { + windmill_parser_py::parse_python_imports(&ns.content)?.join("\n") + } _ => ns.content, }; - let (_, tx) = jobs::push( + let (_, tx) = windmill_queue::push( tx, &w_id, - jobs::JobPayload::Dependencies { hash, dependencies, language: ns.language }, + windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language }, None, &authed.username, owner_to_token_owner(&authed.username, false), @@ -508,63 +377,37 @@ async fn create_script( } pub async fn get_hub_script_by_path( - Authed { email, username, .. }: Authed, + authed: Authed, Path(path): Path, Extension(http_client): Extension, Host(host): Host, ) -> Result { - let path = path - .to_path() - .strip_prefix("hub/") - .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?; - - let content = http_get_from_hub( + windmill_common::scripts::get_hub_script_by_path( + authed.email, + authed.username, + path, http_client, - &format!("https://hub.windmill.dev/raw/{path}.ts"), - email, - username, host, - true, ) - .await? - .text() .await - .map_err(to_anyhow)?; - Ok(content) -} - -#[derive(Deserialize, Serialize)] -pub struct HubScript { - pub content: String, - pub lockfile: Option, - pub language: ScriptLang, - pub schema: Option, } pub async fn get_full_hub_script_by_path( - Authed { email, username, .. }: Authed, + Authed { username, email, .. }: Authed, Path(path): Path, Extension(http_client): Extension, Host(host): Host, ) -> JsonResult { - let path = path - .to_path() - .strip_prefix("hub/") - .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?; - - let value = http_get_from_hub( - http_client, - &format!("https://hub.windmill.dev/raw2/{path}"), - email, - username, - host, - true, - ) - .await? - .json::() - .await - .map_err(to_anyhow)?; - Ok(Json(value)) + Ok(Json( + windmill_common::scripts::get_full_hub_script_by_path( + email, + username, + path, + http_client, + host, + ) + .await?, + )) } async fn get_script_by_path( @@ -586,7 +429,7 @@ async fn get_script_by_path( .await?; tx.commit().await?; - let script = crate::utils::not_found_if_none(script_o, "Script", path)?; + let script = not_found_if_none(script_o, "Script", path)?; Ok(Json(script)) } @@ -612,7 +455,7 @@ async fn raw_script_by_path( .await?; tx.commit().await?; - let content = crate::utils::not_found_if_none(content_o, "Script", path)?; + let content = not_found_if_none(content_o, "Script", path)?; Ok(content) } @@ -650,7 +493,7 @@ async fn get_script_by_hash_internal<'c>( .fetch_optional(db) .await?; - let script = crate::utils::not_found_if_none(script_o, "Script", hash.to_string())?; + let script = not_found_if_none(script_o, "Script", hash.to_string())?; Ok(script) } @@ -702,7 +545,7 @@ async fn get_deployment_status( .fetch_optional(&mut tx) .await?; - let status = crate::utils::not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?; + let status = not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?; tx.commit().await?; Ok(Json(status)) @@ -806,31 +649,17 @@ async fn delete_script_by_hash( async fn parse_python_code_to_jsonschema( Json(code): Json, -) -> JsonResult { - parser_py::parse_python_signature(&code).map(Json) +) -> JsonResult { + windmill_parser_py::parse_python_signature(&code).map(Json) } async fn parse_deno_code_to_jsonschema( Json(code): Json, -) -> JsonResult { - parser_ts::parse_deno_signature(&code).map(Json) +) -> JsonResult { + windmill_parser_ts::parse_deno_signature(&code).map(Json) } async fn parse_go_code_to_jsonschema( Json(code): Json, -) -> JsonResult { - parser_go::parse_go_sig(&code).map(Json) -} - -pub fn to_i64(s: &str) -> Result { - let v = hex::decode(s)?; - let nb: u64 = u64::from_be_bytes( - v[0..8] - .try_into() - .map_err(|_| hex::FromHexError::InvalidStringLength)?, - ); - Ok(nb as i64) -} - -pub fn to_hex_string(i: &i64) -> String { - hex::encode(i.to_be_bytes()) +) -> JsonResult { + windmill_parser_go::parse_go_sig(&code).map(Json) } diff --git a/backend/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs similarity index 98% rename from backend/src/static_assets.rs rename to backend/windmill-api/src/static_assets.rs index c7edf9c1f7..b2dbd33dbd 100644 --- a/backend/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -22,7 +22,7 @@ pub async fn static_handler(uri: Uri) -> impl IntoResponse { } #[derive(RustEmbed)] -#[folder = "../frontend/build/"] +#[folder = "../../frontend/build/"] struct Asset; pub struct StaticFile(pub T); diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs new file mode 100644 index 0000000000..951408c10f --- /dev/null +++ b/backend/windmill-api/src/tracing_init.rs @@ -0,0 +1,45 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use ::tracing::{field, Span}; +use hyper::Response; +use tower_http::trace::{MakeSpan, OnResponse}; + +#[derive(Clone)] +pub struct MyOnResponse {} + +impl OnResponse for MyOnResponse { + fn on_response( + self, + response: &Response, + latency: std::time::Duration, + _span: &tracing::Span, + ) { + tracing::info!( + latency = latency.as_millis(), + status = response.status().as_u16(), + "response" + ) + } +} + +#[derive(Clone)] +pub struct MyMakeSpan {} + +impl MakeSpan for MyMakeSpan { + fn make_span(&mut self, request: &hyper::Request) -> Span { + tracing::info_span!( + "request", + method = %request.method(), + uri = %request.uri(), + username = field::Empty, + workspace_id = field::Empty, + email = field::Empty, + ) + } +} diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs new file mode 100644 index 0000000000..156bf9c650 --- /dev/null +++ b/backend/windmill-api/src/utils.rs @@ -0,0 +1,30 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use sqlx::{Postgres, Transaction}; +use windmill_common::error::{self, Error}; + +pub async fn require_super_admin<'c>( + db: &mut Transaction<'c, Postgres>, + email: Option, +) -> error::Result<()> { + let is_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + email.as_ref() + ) + .fetch_one(db) + .await + .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?; + if !is_admin { + Err(Error::NotAuthorized( + "This endpoint require caller to be a super admin".to_owned(), + )) + } else { + Ok(()) + } +} diff --git a/backend/src/variables.rs b/backend/windmill-api/src/variables.rs similarity index 71% rename from backend/src/variables.rs rename to backend/windmill-api/src/variables.rs index 8c11ac583a..e6ac7d2109 100644 --- a/backend/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -9,25 +9,36 @@ use std::sync::Arc; use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{Error, JsonResult, Result}, oauth2::{AllClients, _refresh_token}, users::Authed, - utils::StripPath, BaseUrl, }; +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + use axum::{ extract::{Extension, Path, Query}, routing::{delete, get, post}, Json, Router, }; use hyper::StatusCode; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{Error, JsonResult, Result}, + utils::{not_found_if_none, StripPath}, + variables::{get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable}, +}; use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use reqwest::Client; -use serde::{Deserialize, Serialize}; -use sqlx::{FromRow, Postgres, Transaction}; +use serde::Deserialize; +use sqlx::{Postgres, Transaction}; pub fn workspaced_service() -> Router { Router::new() @@ -40,122 +51,6 @@ pub fn workspaced_service() -> Router { .route("/create", post(create_variable)) } -#[derive(Serialize, Clone)] - -pub struct ContextualVariable { - pub name: String, - pub value: String, - pub description: String, -} - -#[derive(Serialize, Deserialize, FromRow)] - -pub struct ListableVariable { - pub workspace_id: String, - pub path: String, - pub value: Option, - pub is_secret: bool, - pub description: String, - pub extra_perms: serde_json::Value, - pub account: Option, - pub is_oauth: bool, - pub is_expired: Option, -} - -#[derive(Deserialize)] -pub struct CreateVariable { - pub path: String, - pub value: String, - pub is_secret: bool, - pub description: String, - pub account: Option, - pub is_oauth: Option, -} - -#[derive(Deserialize)] -struct EditVariable { - path: Option, - value: Option, - is_secret: Option, - description: Option, -} - -pub fn get_reserved_variables( - w_id: &str, - token: &str, - email: &str, - username: &str, - job_id: &str, - permissioned_as: &str, - base_url: &str, - path: Option, - flow_id: Option, - flow_path: Option, - schedule_path: Option, -) -> [ContextualVariable; 11] { - [ - ContextualVariable { - name: "WM_WORKSPACE".to_string(), - value: w_id.to_string(), - description: "Workspace id of the current script".to_string(), - }, - ContextualVariable { - name: "WM_TOKEN".to_string(), - value: token.to_string(), - description: "Token ephemeral to the current script with equal permission to the \ - permission of the run (Usable as a bearer token)" - .to_string(), - }, - ContextualVariable { - name: "WM_EMAIL".to_string(), - value: email.to_string(), - description: "Email of the user that executed the current script".to_string(), - }, - ContextualVariable { - name: "WM_USERNAME".to_string(), - value: username.to_string(), - description: "Username of the user that executed the current script".to_string(), - }, - ContextualVariable { - name: "WM_BASE_URL".to_string(), - value: base_url.to_string(), - description: "base url of this instance".to_string(), - }, - ContextualVariable { - name: "WM_JOB_ID".to_string(), - value: job_id.to_string(), - description: "Job id of the current script".to_string(), - }, - ContextualVariable { - name: "WM_JOB_PATH".to_string(), - value: path.unwrap_or_else(|| "".to_string()), - description: "Path of the script or flow being run if any".to_string(), - }, - ContextualVariable { - name: "WM_FLOW_JOB_ID".to_string(), - value: flow_id.unwrap_or_else(|| "".to_string()), - description: "Job id of the encapsulating flow if the job is a flow step".to_string(), - }, - ContextualVariable { - name: "WM_FLOW_PATH".to_string(), - value: flow_path.unwrap_or_else(|| "".to_string()), - description: "Path of the encapsulating flow if the job is a flow step".to_string(), - }, - ContextualVariable { - name: "WM_SCHEDULE_PATH".to_string(), - value: schedule_path.unwrap_or_else(|| "".to_string()), - description: "Path of the schedule if the job of the step or encapsulating step has \ - been triggered by a schedule" - .to_string(), - }, - ContextualVariable { - name: "WM_PERMISSIONED_AS".to_string(), - value: permissioned_as.to_string(), - description: "Fully Qualified (u/g) owner name of executor of the job".to_string(), - }, - ] -} - async fn list_contextual_variables( Path(w_id): Path, Extension(base_url): Extension>, @@ -229,7 +124,7 @@ async fn get_variable( .fetch_optional(&mut tx) .await?; - let variable = crate::utils::not_found_if_none(variable_o, "Variable", &path)?; + let variable = not_found_if_none(variable_o, "Variable", &path)?; let decrypt_secret = q.decrypt_secret.unwrap_or(true); @@ -376,6 +271,14 @@ async fn delete_variable( Ok(format!("variable {} deleted", path)) } +#[derive(Deserialize)] +struct EditVariable { + path: Option, + value: Option, + is_secret: Option, + description: Option, +} + async fn update_variable( authed: Authed, Extension(user_db): Extension, @@ -432,7 +335,7 @@ async fn update_variable( let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?; - let npath = crate::utils::not_found_if_none(npath_o, "Variable", path)?; + let npath = not_found_if_none(npath_o, "Variable", path)?; audit_log( &mut tx, diff --git a/backend/src/worker_ping.rs b/backend/windmill-api/src/worker_ping.rs similarity index 87% rename from backend/src/worker_ping.rs rename to backend/windmill-api/src/worker_ping.rs index 94bfa5fda5..62b0707151 100644 --- a/backend/src/worker_ping.rs +++ b/backend/windmill-api/src/worker_ping.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::{db::UserDB, error::JsonResult, users::Authed, utils::Pagination}; +use crate::{db::UserDB, users::Authed}; use axum::{ extract::{Extension, Query}, routing::get, @@ -15,6 +15,10 @@ use axum::{ use serde::{Deserialize, Serialize}; use sqlx::FromRow; +use windmill_common::{ + error::JsonResult, + utils::{paginate, Pagination}, +}; pub fn global_service() -> Router { Router::new().route("/list", get(list_worker_pings)) @@ -37,7 +41,7 @@ async fn list_worker_pings( ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - let (per_page, offset) = crate::utils::paginate(pagination); + let (per_page, offset) = paginate(pagination); let rows = sqlx::query_as!( WorkerPing, diff --git a/backend/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs similarity index 98% rename from backend/src/workspaces.rs rename to backend/windmill-api/src/workspaces.rs index 3621fff3dd..687c3e5f2b 100644 --- a/backend/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -7,23 +7,27 @@ */ use crate::{ - audit::{audit_log, ActionKind}, db::{UserDB, DB}, - error::{Error, JsonResult, Result}, - flows::Flow, resources::{Resource, ResourceType}, - scripts::{Schema, Script, ScriptLang}, users::{Authed, WorkspaceInvite}, - utils::{require_admin, require_super_admin, Pagination}, - variables::ListableVariable, + utils::require_super_admin, }; use axum::{ body::StreamBody, extract::{Extension, Path, Query}, + headers, response::IntoResponse, routing::{delete, get, post}, Json, Router, }; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{Error, JsonResult, Result}, + flows::Flow, + scripts::{Schema, Script, ScriptLang}, + utils::{paginate, rd_string, require_admin, Pagination}, + variables::ListableVariable, +}; use hyper::{header, StatusCode}; use serde::{Deserialize, Serialize}; @@ -246,7 +250,7 @@ async fn list_workspaces_as_super_admin( ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; require_super_admin(&mut tx, email).await?; - let (per_page, offset) = crate::utils::paginate(pagination); + let (per_page, offset) = paginate(pagination); let workspaces = sqlx::query_as!( Workspace, @@ -309,7 +313,7 @@ async fn create_workspace( ) .execute(&mut tx) .await?; - let key = crate::utils::rd_string(64); + let key = rd_string(64); sqlx::query!( "INSERT INTO workspace_key (workspace_id, kind, key) diff --git a/backend/windmill-audit/Cargo.toml b/backend/windmill-audit/Cargo.toml new file mode 100644 index 0000000000..34696a2b5e --- /dev/null +++ b/backend/windmill-audit/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "windmill-audit" +version = "0.1.0" +edition = "2021" + +[lib] +name = "windmill_audit" +path = "./src/lib.rs" + +[dependencies] +serde.workspace = true +sql-builder.workspace = true +sqlx.workspace = true +chrono.workspace = true +serde_json.workspace = true +tracing.workspace = true +windmill-common = { workspace = true, features = ["axum"] } diff --git a/backend/src/audit.rs b/backend/windmill-audit/src/lib.rs similarity index 77% rename from backend/src/audit.rs rename to backend/windmill-audit/src/lib.rs index 4b2709e771..4d39f853fb 100644 --- a/backend/src/audit.rs +++ b/backend/windmill-audit/src/lib.rs @@ -10,28 +10,15 @@ use sql_builder::prelude::*; use std::collections::HashMap; -use crate::{ - db::UserDB, - error::{Error, JsonResult, Result}, - users::Authed, +use windmill_common::{ + error::{Error, Result}, utils::Pagination, }; -use axum::{ - extract::{Extension, Path, Query}, - routing::get, - Json, Router, -}; use serde::{Deserialize, Serialize}; use sql_builder::SqlBuilder; use sqlx::{FromRow, Postgres, Transaction}; -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_audit)) - .route("/get/:id", get(get_audit)) -} - #[derive(sqlx::Type, Serialize, Deserialize, Debug)] #[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")] pub enum ActionKind { @@ -99,14 +86,13 @@ pub struct ListAuditLogQuery { pub after: Option>, } -async fn list_audit( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, - Query(lq): Query, -) -> JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); +pub async fn list_audit( + mut tx: Transaction<'_, sqlx::Postgres>, + w_id: String, + pagination: Pagination, + lq: ListAuditLogQuery, +) -> Result> { + let (per_page, offset) = windmill_common::utils::paginate(pagination); let mut sqlb = SqlBuilder::select_from("audit") .field("*") @@ -136,25 +122,18 @@ async fn list_audit( } let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, AuditLog>(&sql) .fetch_all(&mut tx) .await?; - tx.commit().await?; - Ok(Json(rows)) + Ok(rows) } -async fn get_audit( - authed: Authed, - Extension(user_db): Extension, - Path(id): Path, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; +pub async fn get_audit(mut tx: Transaction<'_, sqlx::Postgres>, id: i32) -> Result { let audit_o = sqlx::query_as::<_, AuditLog>("SELECT * FROM audit WHERE id = $1") .bind(id) .fetch_optional(&mut tx) .await?; tx.commit().await?; - let audit = crate::utils::not_found_if_none(audit_o, "AuditLog", &id.to_string())?; - Ok(Json(audit)) + let audit = windmill_common::utils::not_found_if_none(audit_o, "AuditLog", &id.to_string())?; + Ok(audit) } diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml new file mode 100644 index 0000000000..3a5440270b --- /dev/null +++ b/backend/windmill-common/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "windmill-common" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[features] +default = [] +sqlx = ["dep:sqlx"] +hyper = ["dep:hyper"] +tokio = ["dep:tokio"] +axum = ["dep:axum", "dep:tracing"] +reqwest = ["dep:reqwest"] +prometheus = ["dep:tiny_http", "dep:prometheus"] +tracing_init = [ + "dep:console-subscriber", + "dep:tracing", + "dep:tracing-subscriber", +] + + +[lib] +name = "windmill_common" +path = "src/lib.rs" + +[dependencies] +hmac.workspace = true +sha2.workspace = true +thiserror.workspace = true +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +chrono.workspace = true +hex.workspace = true +rand.workspace = true +sqlx = { workspace = true, optional = true, features = ["postgres"] } +uuid.workspace = true +tiny_http = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } +axum = { workspace = true, optional = true } +hyper = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } +console-subscriber = { workspace = true, optional = true } +tracing-subscriber = { workspace = true, optional = true } diff --git a/backend/src/error.rs b/backend/windmill-common/src/error.rs similarity index 78% rename from backend/src/error.rs rename to backend/windmill-common/src/error.rs index 06cd3d43ed..6289c633df 100644 --- a/backend/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -6,17 +6,21 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(feature = "axum")] use axum::{ body::{self, BoxBody}, response::IntoResponse, Json, }; -use hyper::{Response, StatusCode}; + +#[cfg(feature = "sqlx")] use sqlx::migrate::MigrateError; use thiserror::Error; +#[cfg(feature = "tokio")] use tokio::io; pub type Result = std::result::Result; +#[cfg(feature = "axum")] pub type JsonResult = std::result::Result, Error>; #[derive(Debug, Error)] @@ -34,8 +38,10 @@ pub enum Error { #[error("{0}")] ExecutionErr(String), #[error("IO error: {0}")] + #[cfg(feature = "tokio")] IoErr(#[from] io::Error), #[error("Sql error: {0}")] + #[cfg(feature = "sqlx")] SqlErr(#[from] sqlx::Error), #[error("Bad request: {0}")] BadRequest(String), @@ -44,6 +50,7 @@ pub enum Error { #[error("Hexadecimal decoding error: {0}")] HexErr(#[from] hex::FromHexError), #[error("Migrating database: {0}")] + #[cfg(feature = "sqlx")] DatabaseMigration(#[from] MigrateError), #[error("Non-zero exit status: {0}")] ExitStatus(i32), @@ -62,18 +69,19 @@ pub fn to_anyhow(e: T) -> anyhow:: From::from(e) } +#[cfg(feature = "axum")] impl IntoResponse for Error { - fn into_response(self) -> Response { + fn into_response(self) -> axum::response::Response { let e = &self; let body = body::boxed(body::Full::from(e.to_string())); let status = match self { - Self::NotFound(_) => StatusCode::NOT_FOUND, - Self::NotAuthorized(_) => StatusCode::UNAUTHORIZED, - Self::SqlErr(_) | Self::BadRequest(_) => StatusCode::BAD_REQUEST, - _ => StatusCode::INTERNAL_SERVER_ERROR, + Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND, + Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED, + Self::SqlErr(_) | Self::BadRequest(_) => axum::http::StatusCode::BAD_REQUEST, + _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, }; tracing::error!(error = e.to_string()); - Response::builder() + axum::response::Response::builder() .header("Content-Type", "text/plain") .status(status) .body(body) diff --git a/backend/src/external_ip.rs b/backend/windmill-common/src/external_ip.rs similarity index 64% rename from backend/src/external_ip.rs rename to backend/windmill-common/src/external_ip.rs index df3edc2879..ce5b43f111 100644 --- a/backend/src/external_ip.rs +++ b/backend/windmill-common/src/external_ip.rs @@ -1,3 +1,11 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + //! Used to determine the internet address that connections from workers will appear to come from. //! //! For users writing scripts to access their infrastructure with firewalls requiring incoming diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs new file mode 100644 index 0000000000..93b0c4305c --- /dev/null +++ b/backend/windmill-common/src/flows.rs @@ -0,0 +1,220 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::{collections::HashMap, time::Duration}; + +use serde::{self, Deserialize, Serialize}; + +use crate::{ + more_serde::{default_id, default_true, is_default}, + scripts::{Schema, ScriptLang}, +}; + +#[derive(Serialize)] +#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))] +pub struct Flow { + pub workspace_id: String, + pub path: String, + pub summary: String, + pub description: String, + pub value: serde_json::Value, + pub edited_by: String, + pub edited_at: chrono::DateTime, + pub archived: bool, + pub schema: Option, + pub extra_perms: serde_json::Value, +} + +#[derive(Deserialize)] +#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))] +pub struct NewFlow { + pub path: String, + pub summary: String, + pub description: String, + pub value: serde_json::Value, + pub schema: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct FlowValue { + pub modules: Vec, + #[serde(default)] + pub failure_module: Option, + #[serde(default)] + #[serde(skip_serializing_if = "is_default")] + pub same_worker: bool, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct StopAfterIf { + pub expr: String, + pub skip_if_stopped: bool, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct Retry { + pub constant: ConstantDelay, + pub exponential: ExponentialDelay, +} + +impl Retry { + /// Takes the number of previous retries and returns the interval until the next retry if any. + /// + /// May return [`Duration::ZERO`] to retry immediately. + pub fn interval(&self, previous_attempts: u16) -> Option { + let Self { constant, exponential } = self; + + if previous_attempts < constant.attempts { + Some(Duration::from_secs(constant.seconds as u64)) + } else if previous_attempts - constant.attempts < exponential.attempts { + let exp = previous_attempts.saturating_add(1) as u32; + let secs = exponential.multiplier * exponential.seconds.saturating_pow(exp); + Some(Duration::from_secs(secs as u64)) + } else { + None + } + } + + pub fn has_attempts(&self) -> bool { + self.constant.attempts != 0 || self.exponential.attempts != 0 + } + + pub fn max_attempts(&self) -> u16 { + self.constant + .attempts + .saturating_add(self.exponential.attempts) + } + + pub fn max_interval(&self) -> Option { + self.max_attempts() + .checked_sub(1) + .and_then(|p| self.interval(p)) + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct ConstantDelay { + pub attempts: u16, + pub seconds: u16, +} + +/// multiplier * seconds ^ failures +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(default)] +pub struct ExponentialDelay { + pub attempts: u16, + pub multiplier: u16, + pub seconds: u16, +} + +impl Default for ExponentialDelay { + fn default() -> Self { + Self { attempts: 0, multiplier: 1, seconds: 0 } + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Suspend { + #[serde(skip_serializing_if = "Option::is_none")] + pub required_events: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowModule { + #[serde(default = "default_id")] + pub id: String, + #[serde(default)] + #[serde(alias = "input_transform")] + pub input_transforms: HashMap, + pub value: FlowModuleValue, + pub stop_after_if: Option, + pub summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub suspend: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sleep: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum InputTransform { + Static { value: serde_json::Value }, + Javascript { expr: String }, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BranchOneModules { + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub expr: String, + pub modules: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BranchAllModules { + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub modules: Vec, + #[serde(default = "default_true")] + pub skip_failure: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum FlowModuleValue { + Script { + #[serde(default)] + #[serde(alias = "input_transform")] + input_transforms: HashMap, + path: String, + }, + ForloopFlow { + iterator: InputTransform, + modules: Vec, + #[serde(default = "default_true")] + skip_failures: bool, + }, + BranchOne { + branches: Vec, + default: Vec, + }, + BranchAll { + branches: Vec, + }, + RawScript { + #[serde(default)] + #[serde(alias = "input_transform")] + input_transforms: HashMap, + content: String, + path: Option, + language: ScriptLang, + }, + Identity, +} + +#[derive(Deserialize)] +pub struct ListFlowQuery { + pub path_start: Option, + pub path_exact: Option, + pub edited_by: Option, + pub show_archived: Option, + pub order_by: Option, + pub order_desc: Option, +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs new file mode 100644 index 0000000000..7411c16024 --- /dev/null +++ b/backend/windmill-common/src/lib.rs @@ -0,0 +1,132 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::net::SocketAddr; + +pub mod error; +pub mod external_ip; +pub mod flows; +pub mod more_serde; +pub mod oauth2; +pub mod scripts; +pub mod users; +pub mod utils; +pub mod variables; +pub mod worker_flow; + +#[cfg(feature = "tracing_init")] +pub mod tracing_init; + +pub const DEFAULT_NUM_WORKERS: usize = 3; +pub const DEFAULT_TIMEOUT: i32 = 300; +pub const DEFAULT_SLEEP_QUEUE: u64 = 50; +pub const DEFAULT_MAX_CONNECTIONS: u32 = 100; + +#[cfg(feature = "tokio")] +pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> { + use std::io; + use tokio::signal::unix::SignalKind; + + async fn terminate() -> io::Result<()> { + tokio::signal::unix::signal(SignalKind::terminate())? + .recv() + .await; + Ok(()) + } + + tokio::select! { + _ = terminate() => {}, + _ = tokio::signal::ctrl_c() => {}, + } + println!("signal received, starting graceful shutdown"); + let _ = tx.send(()); + Ok(()) +} + +#[cfg(feature = "prometheus")] +pub async fn serve_metrics( + addr: SocketAddr, + rx: tokio::sync::broadcast::Receiver<()>, +) -> Result<(), anyhow::Error> { + use tokio::task::yield_now; + + let server = tiny_http::Server::http(addr).map_err(|e| anyhow::anyhow!(e.to_string()))?; + for request in server.incoming_requests() { + yield_now().await; + if !rx.is_empty() { + break; + } + let response = tiny_http::Response::from_string(metrics().await?); + let _ = request.respond(response); + } + Ok(()) +} + +#[cfg(feature = "prometheus")] +async fn metrics() -> Result { + let metric_families = prometheus::gather(); + Ok(prometheus::TextEncoder::new() + .encode_to_string(&metric_families) + .map_err(anyhow::Error::from)?) +} + +#[cfg(feature = "sqlx")] +pub async fn connect_db() -> anyhow::Result> { + use anyhow::Context; + use error::Error; + + let database_url = std::env::var("DATABASE_URL") + .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?; + + let max_connections = match std::env::var("DATABASE_CONNECTIONS") { + Ok(n) => n.parse::().context("invalid DATABASE_CONNECTIONS")?, + Err(_) => DEFAULT_MAX_CONNECTIONS, + }; + + Ok(connect(&database_url, max_connections).await?) +} + +#[cfg(feature = "sqlx")] +pub async fn connect( + database_url: &str, + max_connections: u32, +) -> Result, error::Error> { + use std::time::Duration; + + use crate::error::Error; + + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins + .connect(database_url) + .await + .map_err(|err| Error::ConnectingToDatabase(err.to_string())) +} + +// TODO: Move this elsewhere +pub async fn get_latest_hash_for_path<'c>( + db: &mut sqlx::Transaction<'c, sqlx::Postgres>, + w_id: &str, + script_path: &str, +) -> error::Result { + let script_hash_o = sqlx::query_scalar!( + "select hash from script where path = $1 AND (workspace_id = $2 OR workspace_id = \ + 'starter') AND + created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 OR \ + workspace_id = 'starter')) AND + deleted = false", + script_path, + w_id + ) + .fetch_optional(db) + .await?; + + let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?; + + Ok(scripts::ScriptHash(script_hash)) +} diff --git a/backend/src/more_serde.rs b/backend/windmill-common/src/more_serde.rs similarity index 52% rename from backend/src/more_serde.rs rename to backend/windmill-common/src/more_serde.rs index 32f6602e14..247558e20f 100644 --- a/backend/src/more_serde.rs +++ b/backend/windmill-common/src/more_serde.rs @@ -1,3 +1,11 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + //! helpers for serde + serde derive attributes use crate::utils::rd_string; diff --git a/backend/windmill-common/src/oauth2.rs b/backend/windmill-common/src/oauth2.rs new file mode 100644 index 0000000000..62a162fa37 --- /dev/null +++ b/backend/windmill-common/src/oauth2.rs @@ -0,0 +1,12 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use hmac::Hmac; +use sha2::Sha256; + +pub type HmacSha256 = Hmac; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs new file mode 100644 index 0000000000..62cf3c9851 --- /dev/null +++ b/backend/windmill-common/src/scripts.rs @@ -0,0 +1,257 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::{ + fmt::Display, + hash::{Hash, Hasher}, +}; + +use serde::de::Error as _; +use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; +use serde_json::to_string_pretty; + +use crate::utils::StripPath; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash)] +#[cfg_attr(feature = "sqlx", derive(sqlx::Type))] +#[cfg_attr( + feature = "sqlx", + sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase") +)] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum ScriptLang { + Deno, + Python3, + Go, +} + +impl ScriptLang { + pub fn as_str(&self) -> &'static str { + match self { + ScriptLang::Deno => "deno", + ScriptLang::Python3 => "python3", + ScriptLang::Go => "go", + } + } +} + +#[derive(PartialEq, Debug, Hash, Clone, Copy)] +#[cfg_attr(feature = "sqlx", derive(sqlx::Type))] +#[cfg_attr(feature = "sqlx", sqlx(transparent))] +pub struct ScriptHash(pub i64); + +#[derive(PartialEq)] +#[cfg_attr(feature = "sqlx", derive(sqlx::Type))] +#[cfg_attr(feature = "sqlx", sqlx(transparent))] +pub struct ScriptHashes(pub Vec); + +impl Display for ScriptHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", to_hex_string(&self.0)) + } +} +impl Serialize for ScriptHash { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + serializer.serialize_str(to_hex_string(&self.0).as_str()) + } +} +impl<'de> Deserialize<'de> for ScriptHash { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let i = to_i64(&s).map_err(|e| D::Error::custom(format!("{}", e)))?; + Ok(ScriptHash(i)) + } +} + +impl Serialize for ScriptHashes { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for element in &self.0 { + seq.serialize_element(&ScriptHash(*element))?; + } + seq.end() + } +} + +#[derive(Serialize, Deserialize, Debug, Hash)] +#[cfg_attr(feature = "sqlx", derive(sqlx::Type))] +#[cfg_attr( + feature = "sqlx", + sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase") +)] +#[serde(rename_all = "lowercase")] +pub enum ScriptKind { + Trigger, + Failure, + Script, + Approval, +} + +#[derive(Serialize)] +#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))] +pub struct Script { + pub workspace_id: String, + pub hash: ScriptHash, + pub path: String, + pub parent_hashes: Option, + pub summary: String, + pub description: String, + pub content: String, + pub created_by: String, + pub created_at: chrono::DateTime, + pub archived: bool, + pub schema: Option, + pub deleted: bool, + pub is_template: bool, + pub extra_perms: serde_json::Value, + pub lock: Option, + pub lock_error_logs: Option, + pub language: ScriptLang, + pub kind: ScriptKind, +} + +#[derive(Serialize, Deserialize, Debug)] +#[cfg_attr(feature = "sqlx", derive(sqlx::Type))] +#[cfg_attr(feature = "sqlx", sqlx)] +#[cfg_attr(feature = "sqlx", sqlx(transparent))] +#[serde(transparent)] +pub struct Schema(pub serde_json::Value); + +impl Hash for Schema { + fn hash(&self, state: &mut H) { + if let Ok(s) = to_string_pretty(&self.0) { + s.hash(state); + } + } +} + +#[derive(Serialize, Deserialize, Hash)] +pub struct NewScript { + pub path: String, + pub parent_hash: Option, + pub summary: String, + pub description: String, + pub content: String, + pub schema: Option, + pub is_template: Option, + pub lock: Option>, + pub language: ScriptLang, + pub kind: Option, +} + +#[derive(Deserialize)] +pub struct ListScriptQuery { + pub path_start: Option, + pub path_exact: Option, + pub created_by: Option, + pub first_parent_hash: Option, + pub last_parent_hash: Option, + pub parent_hash: Option, + pub show_archived: Option, + pub order_by: Option, + pub order_desc: Option, + pub is_template: Option, + pub kind: Option, +} + +pub fn to_i64(s: &str) -> crate::error::Result { + let v = hex::decode(s)?; + let nb: u64 = u64::from_be_bytes( + v[0..8] + .try_into() + .map_err(|_| hex::FromHexError::InvalidStringLength)?, + ); + Ok(nb as i64) +} + +pub fn to_hex_string(i: &i64) -> String { + hex::encode(i.to_be_bytes()) +} + +#[cfg(feature = "reqwest")] +pub async fn get_hub_script_by_path( + email: Option, + username: String, + path: StripPath, + http_client: reqwest::Client, + host: String, +) -> crate::error::Result { + use crate::{ + error::{to_anyhow, Error}, + utils::http_get_from_hub, + }; + + let path = path + .to_path() + .strip_prefix("hub/") + .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?; + + let content = http_get_from_hub( + http_client, + &format!("https://hub.windmill.dev/raw/{path}.ts"), + email, + username, + host, + true, + ) + .await? + .text() + .await + .map_err(to_anyhow)?; + Ok(content) +} + +#[cfg(feature = "reqwest")] +pub async fn get_full_hub_script_by_path( + email: Option, + username: String, + path: StripPath, + http_client: reqwest::Client, + host: String, +) -> crate::error::Result { + use crate::{ + error::{to_anyhow, Error}, + utils::http_get_from_hub, + }; + + let path = path + .to_path() + .strip_prefix("hub/") + .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?; + + let value = http_get_from_hub( + http_client, + &format!("https://hub.windmill.dev/raw2/{path}"), + email, + username, + host, + true, + ) + .await? + .json::() + .await + .map_err(to_anyhow)?; + Ok(value) +} + +#[derive(Deserialize, Serialize)] +pub struct HubScript { + pub content: String, + pub lockfile: Option, + pub language: ScriptLang, + pub schema: Option, +} diff --git a/backend/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs similarity index 65% rename from backend/src/tracing_init.rs rename to backend/windmill-common/src/tracing_init.rs index 82fedea45a..06806893b8 100644 --- a/backend/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -1,46 +1,18 @@ -use ::tracing::{field, Metadata, Span}; -use ::tracing_subscriber::{ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use tracing::Metadata; +use tracing_subscriber::{ filter::filter_fn, fmt::{format, Layer}, prelude::*, EnvFilter, }; -use hyper::Response; -use tower_http::trace::{MakeSpan, OnResponse}; - -#[derive(Clone)] -pub struct MyOnResponse {} - -impl OnResponse for MyOnResponse { - fn on_response( - self, - response: &Response, - latency: std::time::Duration, - _span: &tracing::Span, - ) { - tracing::info!( - latency = latency.as_millis(), - status = response.status().as_u16(), - "response" - ) - } -} - -#[derive(Clone)] -pub struct MyMakeSpan {} - -impl MakeSpan for MyMakeSpan { - fn make_span(&mut self, request: &hyper::Request) -> Span { - tracing::info_span!( - "request", - method = %request.method(), - uri = %request.uri(), - username = field::Empty, - workspace_id = field::Empty, - email = field::Empty, - ) - } -} fn json_layer() -> Layer> { tracing_subscriber::fmt::layer() diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs new file mode 100644 index 0000000000..9e8a070942 --- /dev/null +++ b/backend/windmill-common/src/users.rs @@ -0,0 +1,12 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub fn owner_to_token_owner(user: &str, is_group: bool) -> String { + let prefix = if is_group { 'g' } else { 'u' }; + format!("{}/{}", prefix, user) +} diff --git a/backend/src/utils.rs b/backend/windmill-common/src/utils.rs similarity index 79% rename from backend/src/utils.rs rename to backend/windmill-common/src/utils.rs index be2e6dce9e..0b1c1a0bd2 100644 --- a/backend/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -7,11 +7,9 @@ */ use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use reqwest::Response; use serde::Deserialize; -use sqlx::{Postgres, Transaction}; -use crate::error::{to_anyhow, Error, Result}; +use crate::error::{Error, Result}; pub const MAX_PER_PAGE: usize = 1000; pub const DEFAULT_PER_PAGE: usize = 100; @@ -34,26 +32,6 @@ impl StripPath { } } -pub async fn require_super_admin<'c>( - db: &mut Transaction<'c, Postgres>, - email: Option, -) -> Result<()> { - let is_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - email.as_ref() - ) - .fetch_one(db) - .await - .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?; - if !is_admin { - Err(Error::NotAuthorized( - "This endpoint require caller to be a super admin".to_owned(), - )) - } else { - Ok(()) - } -} - pub fn require_admin(is_admin: bool, username: &str) -> Result<()> { if !is_admin { Err(Error::NotAuthorized(format!( @@ -65,14 +43,6 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> { } } -pub fn rd_string(len: usize) -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(len) - .map(char::from) - .collect() -} - pub fn paginate(pagination: Pagination) -> (usize, usize) { let per_page = pagination .per_page @@ -83,8 +53,9 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) { (per_page, offset) } +#[cfg(feature = "sqlx")] pub async fn now_from_db<'c>( - db: &mut Transaction<'c, Postgres>, + db: &mut sqlx::Transaction<'c, sqlx::Postgres>, ) -> Result> { Ok(sqlx::query_scalar!("SELECT now()") .fetch_one(db) @@ -108,6 +79,7 @@ pub fn get_owner_from_path(path: &str) -> String { path.split('/').take(2).collect::>().join("/") } +#[cfg(feature = "reqwest")] pub async fn list_elems_from_hub( http_client: reqwest::Client, url: &str, @@ -119,10 +91,11 @@ pub async fn list_elems_from_hub( .await? .json::() .await - .map_err(to_anyhow)?; + .map_err(crate::error::to_anyhow)?; Ok(rows) } +#[cfg(feature = "reqwest")] pub async fn http_get_from_hub( http_client: reqwest::Client, url: &str, @@ -130,7 +103,7 @@ pub async fn http_get_from_hub( username: String, host: String, plain: bool, -) -> Result { +) -> Result { let response = http_client .get(url) .header( @@ -146,7 +119,15 @@ pub async fn http_get_from_hub( .header("X-hostname", host) .send() .await - .map_err(to_anyhow)?; + .map_err(crate::error::to_anyhow)?; Ok(response) } + +pub fn rd_string(len: usize) -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs new file mode 100644 index 0000000000..d103c685ed --- /dev/null +++ b/backend/windmill-common/src/variables.rs @@ -0,0 +1,118 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Clone)] + +pub struct ContextualVariable { + pub name: String, + pub value: String, + pub description: String, +} + +#[derive(Serialize, Deserialize)] +#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))] + +pub struct ListableVariable { + pub workspace_id: String, + pub path: String, + pub value: Option, + pub is_secret: bool, + pub description: String, + pub extra_perms: serde_json::Value, + pub account: Option, + pub is_oauth: bool, + pub is_expired: Option, +} + +#[derive(Deserialize)] +pub struct CreateVariable { + pub path: String, + pub value: String, + pub is_secret: bool, + pub description: String, + pub account: Option, + pub is_oauth: Option, +} + +pub fn get_reserved_variables( + w_id: &str, + token: &str, + email: &str, + username: &str, + job_id: &str, + permissioned_as: &str, + base_url: &str, + path: Option, + flow_id: Option, + flow_path: Option, + schedule_path: Option, +) -> [ContextualVariable; 11] { + [ + ContextualVariable { + name: "WM_WORKSPACE".to_string(), + value: w_id.to_string(), + description: "Workspace id of the current script".to_string(), + }, + ContextualVariable { + name: "WM_TOKEN".to_string(), + value: token.to_string(), + description: "Token ephemeral to the current script with equal permission to the \ + permission of the run (Usable as a bearer token)" + .to_string(), + }, + ContextualVariable { + name: "WM_EMAIL".to_string(), + value: email.to_string(), + description: "Email of the user that executed the current script".to_string(), + }, + ContextualVariable { + name: "WM_USERNAME".to_string(), + value: username.to_string(), + description: "Username of the user that executed the current script".to_string(), + }, + ContextualVariable { + name: "WM_BASE_URL".to_string(), + value: base_url.to_string(), + description: "base url of this instance".to_string(), + }, + ContextualVariable { + name: "WM_JOB_ID".to_string(), + value: job_id.to_string(), + description: "Job id of the current script".to_string(), + }, + ContextualVariable { + name: "WM_JOB_PATH".to_string(), + value: path.unwrap_or_else(|| "".to_string()), + description: "Path of the script or flow being run if any".to_string(), + }, + ContextualVariable { + name: "WM_FLOW_JOB_ID".to_string(), + value: flow_id.unwrap_or_else(|| "".to_string()), + description: "Job id of the encapsulating flow if the job is a flow step".to_string(), + }, + ContextualVariable { + name: "WM_FLOW_PATH".to_string(), + value: flow_path.unwrap_or_else(|| "".to_string()), + description: "Path of the encapsulating flow if the job is a flow step".to_string(), + }, + ContextualVariable { + name: "WM_SCHEDULE_PATH".to_string(), + value: schedule_path.unwrap_or_else(|| "".to_string()), + description: "Path of the schedule if the job of the step or encapsulating step has \ + been triggered by a schedule" + .to_string(), + }, + ContextualVariable { + name: "WM_PERMISSIONED_AS".to_string(), + value: permissioned_as.to_string(), + description: "Fully Qualified (u/g) owner name of executor of the job".to_string(), + }, + ] +} diff --git a/backend/windmill-common/src/worker_flow.rs b/backend/windmill-common/src/worker_flow.rs new file mode 100644 index 0000000000..c6d1f2b66d --- /dev/null +++ b/backend/windmill-common/src/worker_flow.rs @@ -0,0 +1,162 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{flows::FlowValue, more_serde::is_default}; + +const MINUTES: Duration = Duration::from_secs(60); +const HOURS: Duration = MINUTES.saturating_mul(60); + +pub const MAX_RETRY_ATTEMPTS: u16 = 1000; +pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); + +#[derive(Serialize, Deserialize, Debug)] +pub struct FlowStatus { + pub step: i32, + pub modules: Vec, + pub failure_module: FlowStatusModule, + #[serde(default)] + #[serde(skip_serializing_if = "is_default")] + pub retry: RetryStatus, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct RetryStatus { + pub fail_count: u16, + pub previous_result: Option, + pub failed_jobs: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Iterator { + pub index: usize, + pub itered: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BranchAllStatus { + pub branch: usize, + pub previous_result: serde_json::Value, + pub len: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum BranchChosen { + Default, + Branch { branch: usize }, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Approval { + pub resume_id: u16, + pub approver: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum FlowStatusModule { + WaitingForPriorSteps { + id: String, + }, + WaitingForEvents { + id: String, + count: u16, + job: Uuid, + }, + WaitingForExecutor { + id: String, + job: Uuid, + }, + InProgress { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + iterator: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branchall: Option, + }, + Success { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + approvers: Vec, + }, + Failure { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + }, +} + +impl FlowStatusModule { + pub fn job(&self) -> Option { + match self { + FlowStatusModule::WaitingForPriorSteps { .. } => None, + FlowStatusModule::WaitingForEvents { job, .. } => Some(*job), + FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job), + FlowStatusModule::InProgress { job, .. } => Some(*job), + FlowStatusModule::Success { job, .. } => Some(*job), + FlowStatusModule::Failure { job, .. } => Some(*job), + } + } + + pub fn id(&self) -> String { + match self { + FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(), + FlowStatusModule::WaitingForEvents { id, .. } => id.clone(), + FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(), + FlowStatusModule::InProgress { id, .. } => id.clone(), + FlowStatusModule::Success { id, .. } => id.clone(), + FlowStatusModule::Failure { id, .. } => id.clone(), + } + } +} + +impl FlowStatus { + pub fn new(f: &FlowValue) -> Self { + Self { + step: 0, + modules: f + .modules + .iter() + .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() }) + .collect(), + failure_module: FlowStatusModule::WaitingForPriorSteps { id: "failure".to_string() }, + retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] }, + } + } + + /// current module status ... excluding failure_module + pub fn current_step(&self) -> Option<&FlowStatusModule> { + let i = usize::try_from(self.step).ok()?; + self.modules.get(i) + } +} + +pub fn init_flow_status(f: &FlowValue) -> FlowStatus { + FlowStatus::new(f) +} diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml new file mode 100644 index 0000000000..69bac6cac5 --- /dev/null +++ b/backend/windmill-queue/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "windmill-queue" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_queue" +path = "src/lib.rs" + +[dependencies] +windmill-audit.workspace = true +windmill-common = { workspace = true, features = ["sqlx", "reqwest"] } +anyhow.workspace = true +hmac.workspace = true +sql-builder.workspace = true +sqlx.workspace = true +tracing.workspace = true +serde.workspace = true +serde_json.workspace = true +ulid.workspace = true +uuid.workspace = true +chrono.workspace = true +hex.workspace = true +reqwest.workspace = true +lazy_static.workspace = true +prometheus.workspace = true +cron.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs new file mode 100644 index 0000000000..65e5189667 --- /dev/null +++ b/backend/windmill-queue/src/jobs.rs @@ -0,0 +1,572 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::{collections::HashMap, str::FromStr}; + +use serde::{Deserialize, Serialize}; +use sqlx::{Pool, Postgres, Transaction}; +use tracing::instrument; +use ulid::Ulid; +use uuid::Uuid; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{self, to_anyhow, Error}, + flows::FlowValue, + scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang}, + utils::StripPath, + worker_flow::{init_flow_status, FlowStatus, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL}, +}; + +lazy_static::lazy_static! { + // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens. + static ref QUEUE_PUSH_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( + "queue_push_count", + "Total number of jobs pushed to the queue." + ) + .unwrap(); + static ref QUEUE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( + "queue_delete_count", + "Total number of jobs deleted from the queue." + ) + .unwrap(); + static ref QUEUE_PULL_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( + "queue_pull_count", + "Total number of jobs pulled from the queue." + ) + .unwrap(); +} + +const MAX_NB_OF_JOBS_IN_Q_PER_USER: i64 = 10; +const MAX_DURATION_LAST_1200: std::time::Duration = std::time::Duration::from_secs(900); + +pub async fn cancel_job<'c>( + username: &str, + reason: Option, + id: Uuid, + w_id: &str, + mut tx: Transaction<'c, Postgres>, +) -> error::Result<(Transaction<'c, Postgres>, Option)> { + let job_option = sqlx::query_scalar!( + "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 \ + AND workspace_id = $4 RETURNING id", + username, + reason, + id, + w_id + ) + .fetch_optional(&mut tx) + .await?; + let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default(); + while !jobs.is_empty() { + let p_job = jobs.pop(); + let new_jobs = sqlx::query_scalar!( + "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2 WHERE parent_job = $3 \ + AND workspace_id = $4 RETURNING id", + username, + reason, + p_job, + w_id + ) + .fetch_all(&mut tx) + .await?; + jobs.extend(new_jobs); + } + Ok((tx, job_option)) +} + +pub async fn pull(db: &Pool) -> windmill_common::error::Result> { + /* Jobs can be started if they: + * - haven't been started before, + * running = false + * - are flows with a step that needed resume, + * suspend_until is non-null + * and suspend = 0 when the resume messages are received + * or suspend_until <= now() if it has timed out */ + let job: Option = sqlx::query_as::<_, QueuedJob>( + "UPDATE queue + SET running = true + , started_at = coalesce(started_at, now()) + , last_ping = now() + , suspend_until = null + WHERE id = ( + SELECT id + FROM queue + WHERE ( running = false + AND scheduled_for <= now()) + OR (suspend_until IS NOT NULL + AND ( suspend <= 0 + OR suspend_until <= now())) + ORDER BY scheduled_for + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + RETURNING *", + ) + .fetch_optional(db) + .await?; + + if job.is_some() { + QUEUE_PULL_COUNT.inc(); + } + + Ok(job) +} + +pub async fn get_result_by_id( + db: Pool, + mut skip_direct: bool, + w_id: String, + flow_id: String, + node_id: String, +) -> error::Result { + let mut result_id: Option = None; + let mut parent_id = Uuid::from_str(&flow_id).ok(); + while result_id.is_none() && parent_id.is_some() { + if !skip_direct { + let r = sqlx::query!( + "SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 ", + parent_id.unwrap(), + w_id, + ) + .fetch_optional(&db) + .await?; + if let Some(r) = r { + let value = r + .flow_status + .as_ref() + .ok_or_else(|| Error::InternalErr(format!("requiring a flow status value")))? + .to_owned(); + parent_id = r.parent_job; + let status_o = serde_json::from_value::(value).ok(); + result_id = status_o.and_then(|status| { + status + .modules + .iter() + .find(|m| m.id() == node_id) + .and_then(|m| m.job()) + }); + } else { + parent_id = None; + } + } else { + let q_parent = sqlx::query_scalar!( + "SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2", + parent_id.unwrap(), + w_id, + ) + .fetch_optional(&db) + .await? + .flatten(); + parent_id = q_parent; + skip_direct = false + } + } + let result_id = windmill_common::utils::not_found_if_none( + result_id, + "Flow result by id", + format!("{}, {}", flow_id, node_id), + )?; + let value = sqlx::query_scalar!( + "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", + result_id, + w_id, + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(serde_json::Value::Null); + Ok(value) +} + +#[instrument(level = "trace", skip_all)] +pub async fn delete_job( + db: &Pool, + w_id: &str, + job_id: Uuid, +) -> windmill_common::error::Result<()> { + QUEUE_DELETE_COUNT.inc(); + let job_removed = sqlx::query_scalar!( + "DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1", + w_id, + job_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))? + .unwrap_or(0) + == 1; + tracing::debug!("Job {job_id} deletion was achieved with success: {job_removed}"); + Ok(()) +} + +pub async fn get_queued_job<'c>( + id: Uuid, + w_id: &str, + tx: &mut Transaction<'c, Postgres>, +) -> error::Result> { + let r = sqlx::query_as::<_, QueuedJob>( + "SELECT * + FROM queue WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .fetch_optional(tx) + .await?; + Ok(r) +} + +#[instrument(level = "trace", skip_all)] +pub async fn push<'c>( + mut tx: Transaction<'c, Postgres>, + workspace_id: &str, + job_payload: JobPayload, + args: Option>, + user: &str, + permissioned_as: String, + scheduled_for_o: Option>, + schedule_path: Option, + parent_job: Option, + is_flow_step: bool, + mut same_worker: bool, +) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { + let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now); + let args_json = args.map(serde_json::Value::Object); + let job_id: Uuid = Ulid::new().into(); + + let premium_workspace = + sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id) + .fetch_one(&mut tx) + .await + .map_err(|e| { + Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}")) + })?; + + if !premium_workspace && std::env::var("CLOUD_HOSTED").is_ok() { + let rate_limiting_queue = sqlx::query_scalar!( + "SELECT COUNT(id) FROM queue WHERE permissioned_as = $1 AND workspace_id = $2", + permissioned_as, + workspace_id + ) + .fetch_one(&mut tx) + .await?; + + if let Some(nb_jobs) = rate_limiting_queue { + if nb_jobs > MAX_NB_OF_JOBS_IN_Q_PER_USER { + return Err(error::Error::ExecutionErr(format!( + "You have exceeded the number of authorized elements of queue at any given \ + time: {}", + MAX_NB_OF_JOBS_IN_Q_PER_USER + ))); + } + } + + let rate_limiting_duration_ms = sqlx::query_scalar!( + " + SELECT SUM(duration_ms) + FROM completed_job + WHERE permissioned_as = $1 + AND created_at > NOW() - INTERVAL '1200 seconds' + AND workspace_id = $2", + permissioned_as, + workspace_id + ) + .fetch_one(&mut tx) + .await?; + + if let Some(sum_duration_ms) = rate_limiting_duration_ms { + if sum_duration_ms as u128 > MAX_DURATION_LAST_1200.as_millis() { + return Err(error::Error::ExecutionErr(format!( + "You have exceeded the scripts cumulative duration limit over the last 20m \ + which is: {} seconds", + MAX_DURATION_LAST_1200.as_secs() + ))); + } + } + } + + let (script_hash, script_path, raw_code, job_kind, raw_flow, language) = match job_payload { + JobPayload::ScriptHash { hash, path } => { + let language = sqlx::query_scalar!( + "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND \ + (workspace_id = $2 OR workspace_id = 'starter')", + hash.0, + workspace_id + ) + .fetch_one(&mut tx) + .await + .map_err(|e| { + Error::InternalErr(format!( + "fetching language for hash {hash} in {workspace_id}: {e}" + )) + })?; + ( + Some(hash.0), + Some(path), + None, + JobKind::Script, + None, + Some(language), + ) + } + JobPayload::ScriptHub { path } => { + let email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + user, + workspace_id + ) + .fetch_optional(&mut tx) + .await?; + let script = get_hub_script(path.clone(), email, user).await?; + ( + None, + Some(path), + Some(script.content.clone()), + JobKind::Script_Hub, + None, + Some(script.language.clone()), + ) + } + JobPayload::Code(RawCode { content, path, language }) => ( + None, + path, + Some(content), + JobKind::Preview, + None, + Some(language), + ), + JobPayload::Dependencies { hash, dependencies, language } => ( + Some(hash.0), + None, + Some(dependencies), + JobKind::Dependencies, + None, + Some(language), + ), + JobPayload::RawFlow { value, path } => { + (None, path, None, JobKind::FlowPreview, Some(value), None) + } + JobPayload::Flow(flow) => { + let value_json = sqlx::query_scalar!( + "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \ + 'starter')", + flow, + workspace_id + ) + .fetch_optional(&mut tx) + .await? + .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?; + let value = serde_json::from_value::(value_json).map_err(|err| { + Error::InternalErr(format!( + "could not convert json to flow for {flow}: {err:?}" + )) + })?; + (None, Some(flow), None, JobKind::Flow, Some(value), None) + } + JobPayload::Identity => (None, None, None, JobKind::Identity, None, None), + }; + + let is_running = same_worker; + if let Some(flow) = raw_flow.as_ref() { + same_worker = same_worker || flow.same_worker; + + for module in flow.modules.iter() { + if let Some(retry) = &module.retry { + if retry.max_attempts() > MAX_RETRY_ATTEMPTS { + Err(Error::BadRequest(format!( + "retry attempts exceeds the maximum of {MAX_RETRY_ATTEMPTS}" + )))? + } + + if matches!(retry.max_interval(), Some(interval) if interval > MAX_RETRY_INTERVAL) { + let max = MAX_RETRY_INTERVAL.as_secs(); + Err(Error::BadRequest(format!( + "retry interval exceeds the maximum of {max} seconds" + )))? + } + } + } + } + + let flow_status = raw_flow.as_ref().map(init_flow_status); + let uuid = sqlx::query_scalar!( + "INSERT INTO queue + (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, + script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, \ + flow_status, is_flow_step, language, started_at, same_worker) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, CASE WHEN $3 THEN now() END, $18) \ + RETURNING id", + workspace_id, + job_id, + is_running, + parent_job, + user, + permissioned_as, + scheduled_for, + script_hash, + script_path.clone(), + raw_code, + args_json, + job_kind: JobKind, + schedule_path, + raw_flow.map(|f| serde_json::json!(f)), + flow_status.map(|f| serde_json::json!(f)), + is_flow_step, + language: ScriptLang, + same_worker + ) + .fetch_one(&mut tx) + .await + .map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id}: {e}")))?; + // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction. + QUEUE_PUSH_COUNT.inc(); + + { + let uuid_string = job_id.to_string(); + let uuid_str = uuid_string.as_str(); + let mut hm = HashMap::from([("uuid", uuid_str), ("permissioned_as", &permissioned_as)]); + + let s: String; + let operation_name = match job_kind { + JobKind::Preview => "jobs.run.preview", + JobKind::Script => { + s = ScriptHash(script_hash.unwrap()).to_string(); + hm.insert("hash", s.as_str()); + "jobs.run.script" + } + JobKind::Flow => "jobs.run.flow", + JobKind::FlowPreview => "jobs.run.flow_preview", + JobKind::Script_Hub => "jobs.run.script_hub", + JobKind::Dependencies => "jobs.run.dependencies", + JobKind::Identity => "jobs.run.identity", + }; + + audit_log( + &mut tx, + &user, + operation_name, + ActionKind::Execute, + workspace_id, + script_path.as_ref().map(|x| x.as_str()), + Some(hm), + ) + .await?; + } + Ok((uuid, tx)) +} + +pub fn canceled_job_to_result(job: &QueuedJob) -> String { + let reason = job + .canceled_reason + .as_deref() + .unwrap_or_else(|| "no reason given"); + let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown"); + format!("Job canceled: {reason} by {canceler}") +} + +pub async fn get_hub_script( + path: String, + email: Option, + user: &str, +) -> error::Result { + get_full_hub_script_by_path( + email, + user.to_string(), + StripPath(path), + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .build() + .map_err(to_anyhow)?, + std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string()), + ) + .await + .map(|e| e) +} + +#[derive(Debug, sqlx::FromRow, Serialize, Clone)] +pub struct QueuedJob { + pub workspace_id: String, + pub id: Uuid, + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub scheduled_for: chrono::DateTime, + pub running: bool, + pub script_hash: Option, + pub script_path: Option, + pub args: Option, + pub logs: Option, + pub raw_code: Option, + pub canceled: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub last_ping: Option>, + pub job_kind: JobKind, + pub schedule_path: Option, + pub permissioned_as: String, + pub flow_status: Option, + pub raw_flow: Option, + pub is_flow_step: bool, + pub language: Option, + pub same_worker: bool, +} + +impl QueuedJob { + pub fn script_path(&self) -> &str { + self.script_path + .as_ref() + .map(String::as_str) + .unwrap_or("NO_FLOW_PATH") + } +} + +impl QueuedJob { + pub fn parse_raw_flow(&self) -> Option { + self.raw_flow + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + } + + pub fn parse_flow_status(&self) -> Option { + self.flow_status + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] +#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase"))] +pub enum JobKind { + Script, + #[allow(non_camel_case_types)] + Script_Hub, + Preview, + Dependencies, + Flow, + FlowPreview, + Identity, +} + +#[derive(Debug, Clone)] +pub enum JobPayload { + ScriptHub { path: String }, + ScriptHash { hash: ScriptHash, path: String }, + Code(RawCode), + Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang }, + Flow(String), + RawFlow { value: FlowValue, path: Option }, + Identity, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct RawCode { + pub content: String, + pub path: Option, + pub language: ScriptLang, +} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs new file mode 100644 index 0000000000..b47a3c3238 --- /dev/null +++ b/backend/windmill-queue/src/lib.rs @@ -0,0 +1,12 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +mod jobs; +pub mod schedule; + +pub use jobs::*; diff --git a/backend/src/schedule.rs b/backend/windmill-queue/src/schedule.rs similarity index 74% rename from backend/src/schedule.rs rename to backend/windmill-queue/src/schedule.rs index 13a8278f1f..5edfe84ec9 100644 --- a/backend/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -8,39 +8,16 @@ use std::str::FromStr; -use crate::{ - audit::{audit_log, ActionKind}, - db::{UserDB, DB}, - error::{self, Error, JsonResult, Result}, - jobs::{self, push, JobPayload}, - users::Authed, - utils::{get_owner_from_path, now_from_db, Pagination, StripPath}, -}; -use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, -}; - use chrono::{DateTime, Duration, FixedOffset}; use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; use sqlx::{query_scalar, FromRow, Postgres, Transaction}; +use windmill_audit::{audit_log, ActionKind}; +use windmill_common::{ + error::{self, Error, Result}, + utils::{get_owner_from_path, not_found_if_none, now_from_db, paginate, Pagination, StripPath}, +}; -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_schedule)) - .route("/get/*path", get(get_schedule)) - .route("/exists/*path", get(exists_schedule)) - .route("/create", post(create_schedule)) - .route("/update/*path", post(edit_schedule)) - .route("/delete/*path", delete(delete_schedule)) - .route("/setenabled/*path", post(set_enabled)) -} - -pub fn global_service() -> Router { - Router::new().route("/preview", post(preview_schedule)) -} +use crate::{push, JobPayload}; #[derive(FromRow, Serialize, Deserialize, Debug)] pub struct Schedule { @@ -97,10 +74,10 @@ pub async fn push_scheduled_job<'c>( return Ok(tx); } - let mut args: Option> = None; + let mut args: Option> = None; if let Some(args_v) = schedule.args { - if let Value::Object(args_m) = args_v { + if let serde_json::Value::Object(args_m) = args_v { args = Some(args_m) } else { return Err(error::Error::ExecutionErr( @@ -113,7 +90,7 @@ pub async fn push_scheduled_job<'c>( JobPayload::Flow(schedule.script_path) } else { JobPayload::ScriptHash { - hash: jobs::get_latest_hash_for_path( + hash: windmill_common::get_latest_hash_for_path( &mut tx, &schedule.workspace_id, &schedule.script_path, @@ -140,15 +117,13 @@ pub async fn push_scheduled_job<'c>( Ok(tx) } -async fn create_schedule( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(ns): Json, +pub async fn create_schedule( + mut tx: Transaction<'_, Postgres>, + w_id: String, + ns: NewSchedule, + username: &str, ) -> Result { cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?; let schedule = sqlx::query_as!( @@ -159,7 +134,7 @@ async fn create_schedule( ns.path, ns.schedule, ns.offset, - &authed.username, + username, ns.script_path, ns.is_flow, ns.args, @@ -171,7 +146,7 @@ async fn create_schedule( audit_log( &mut tx, - &authed.username, + username, "schedule.create", ActionKind::Create, &w_id, @@ -241,18 +216,17 @@ async fn clear_schedule<'c>(db: &mut Transaction<'c, Postgres>, path: &str) -> R Ok(()) } -async fn edit_schedule( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(es): Json, +pub async fn edit_schedule( + mut tx: Transaction<'_, Postgres>, + w_id: String, + path: StripPath, + es: EditSchedule, + username: &String, ) -> Result { let path = path.to_path(); cron::Schedule::from_str(&es.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - check_flow_conflict(&mut tx, &w_id, &path, es.is_flow, &es.script_path).await?; clear_schedule(&mut tx, path).await?; @@ -277,7 +251,7 @@ async fn edit_schedule( audit_log( &mut tx, - &authed.username, + username, "schedule.edit", ActionKind::Update, &w_id, @@ -294,19 +268,15 @@ async fn edit_schedule( ) .await?; - tx.commit().await?; Ok(path.to_string()) } -async fn list_schedule( - authed: Authed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = crate::utils::paginate(pagination); - let mut tx = user_db.begin(&authed).await?; - +pub async fn list_schedule( + mut tx: Transaction<'_, Postgres>, + w_id: String, + pagination: Pagination, +) -> Result> { + let (per_page, offset) = paginate(pagination); let rows = sqlx::query_as!( Schedule, "SELECT * FROM schedule WHERE workspace_id = $1 ORDER BY edited_at desc LIMIT $2 OFFSET $3", @@ -317,7 +287,7 @@ async fn list_schedule( .fetch_all(&mut tx) .await?; tx.commit().await?; - Ok(Json(rows)) + Ok(rows) } pub async fn get_schedule_opt<'c>( @@ -335,24 +305,12 @@ pub async fn get_schedule_opt<'c>( .await?; Ok(schedule_opt) } -async fn get_schedule( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let mut tx = user_db.begin(&authed).await?; - let schedule_o = get_schedule_opt(&mut tx, &w_id, path).await?; - let schedule = crate::utils::not_found_if_none(schedule_o, "Schedule", path)?; - tx.commit().await?; - Ok(Json(schedule)) -} - -async fn exists_schedule( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { +pub async fn exists_schedule( + tx: &mut Transaction<'_, Postgres>, + w_id: String, + path: StripPath, +) -> Result { let path = path.to_path(); let exists = sqlx::query_scalar!( @@ -360,11 +318,11 @@ async fn exists_schedule( path, w_id ) - .fetch_one(&db) + .fetch_one(tx) .await? .unwrap_or(false); - Ok(Json(exists)) + Ok(exists) } #[derive(Deserialize)] @@ -373,9 +331,9 @@ pub struct PreviewPayload { pub offset: Option, } -pub async fn preview_schedule( - Json(PreviewPayload { schedule, offset }): Json, -) -> JsonResult>> { +pub fn preview_schedule( + PreviewPayload { schedule, offset }: PreviewPayload, +) -> Result>> { let schedule = cron::Schedule::from_str(&schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?; let upcoming: Vec> = schedule @@ -383,7 +341,7 @@ pub async fn preview_schedule( .take(10) .map(|x| x.into()) .collect(); - Ok(Json(upcoming)) + Ok(upcoming) } fn get_offset(offset: Option) -> FixedOffset { @@ -396,14 +354,13 @@ pub struct SetEnabled { } pub async fn set_enabled( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(SetEnabled { enabled }): Json, + mut tx: Transaction<'_, Postgres>, + w_id: String, + path: StripPath, + SetEnabled { enabled }: SetEnabled, + username: &str, ) -> Result { let path = path.to_path(); - let mut tx = user_db.begin(&authed).await?; - let schedule_o = sqlx::query_as!( Schedule, "UPDATE schedule SET enabled = $1 WHERE path = $2 AND workspace_id = $3 RETURNING *", @@ -414,7 +371,7 @@ pub async fn set_enabled( .fetch_optional(&mut tx) .await?; - let schedule = crate::utils::not_found_if_none(schedule_o, "Schedule", path)?; + let schedule = not_found_if_none(schedule_o, "Schedule", path)?; clear_schedule(&mut tx, path).await?; @@ -423,7 +380,7 @@ pub async fn set_enabled( } audit_log( &mut tx, - &authed.username, + username, "schedule.setenabled", ActionKind::Update, &w_id, @@ -438,13 +395,13 @@ pub async fn set_enabled( )) } -async fn delete_schedule( - authed: Authed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, +pub async fn delete_schedule( + mut tx: Transaction<'_, Postgres>, + w_id: String, + path: StripPath, + username: &str, ) -> Result { let path = path.to_path(); - let mut tx = user_db.begin(&authed).await?; sqlx::query!( "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2", @@ -456,7 +413,7 @@ async fn delete_schedule( audit_log( &mut tx, - &authed.username, + username, "schedule.delete", ActionKind::Delete, &w_id, diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml new file mode 100644 index 0000000000..1f18f5ae85 --- /dev/null +++ b/backend/windmill-worker/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "windmill-worker" +version.workspace = true +authors.workspace = true +edition.workspace = true +default-run = "worker" + +[[bin]] +name = "worker" +path = "./src/main.rs" + +[dependencies] +windmill-queue.workspace = true +windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker. +windmill-common = { workspace = true, features = [ + "tokio", + "sqlx", + "prometheus", + "tracing_init", +] } +windmill-api-client.workspace = true +windmill-parser.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-go.workspace = true +windmill-parser-py.workspace = true +sqlx.workspace = true +uuid.workspace = true +tracing.workspace = true +tokio.workspace = true +serde.workspace = true +serde_json.workspace = true +futures.workspace = true +async-recursion.workspace = true +anyhow.workspace = true +itertools.workspace = true +regex.workspace = true +prometheus.workspace = true +lazy_static.workspace = true +chrono.workspace = true +dotenv.workspace = true +rand.workspace = true # TODO: Remove. only used by token creation hack. +deno_core.workspace = true diff --git a/backend/windmill-worker/README.md b/backend/windmill-worker/README.md new file mode 100644 index 0000000000..67592475a5 --- /dev/null +++ b/backend/windmill-worker/README.md @@ -0,0 +1,5 @@ +# Windmill Worker + +The worker. Used to process and execute flows & jobs. + +This crate exposes both a library as well as a binary target. diff --git a/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto similarity index 100% rename from nsjail/download.py.config.proto rename to backend/windmill-worker/nsjail/download.py.config.proto diff --git a/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh similarity index 100% rename from nsjail/download_deps.py.sh rename to backend/windmill-worker/nsjail/download_deps.py.sh diff --git a/nsjail/run.deno.config.proto b/backend/windmill-worker/nsjail/run.deno.config.proto similarity index 100% rename from nsjail/run.deno.config.proto rename to backend/windmill-worker/nsjail/run.deno.config.proto diff --git a/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto similarity index 100% rename from nsjail/run.go.config.proto rename to backend/windmill-worker/nsjail/run.go.config.proto diff --git a/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto similarity index 100% rename from nsjail/run.python3.config.proto rename to backend/windmill-worker/nsjail/run.python3.config.proto diff --git a/backend/windmill-worker/src/jobs.rs b/backend/windmill-worker/src/jobs.rs new file mode 100644 index 0000000000..42084331b4 --- /dev/null +++ b/backend/windmill-worker/src/jobs.rs @@ -0,0 +1,175 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use sqlx::{Pool, Postgres, Transaction}; +use tracing::instrument; +use uuid::Uuid; +use windmill_common::error::Error; +use windmill_queue::{delete_job, JobKind, QueuedJob}; + +#[instrument(level = "trace", skip_all)] +pub async fn add_completed_job_error( + db: &Pool, + client: &windmill_api_client::Client, + queued_job: &QueuedJob, + logs: String, + e: E, + metrics: Option, +) -> Result<(Uuid, serde_json::Map), Error> { + metrics.map(|m| m.worker_execution_failed.inc()); + let mut output_map = serde_json::Map::new(); + output_map.insert( + "error".to_string(), + serde_json::Value::String(e.to_string()), + ); + let a = add_completed_job( + db, + client, + &queued_job, + false, + false, + serde_json::Value::Object(output_map.clone()), + logs, + ) + .await?; + Ok((a, output_map)) +} + +#[instrument(level = "trace", skip_all)] +pub async fn add_completed_job( + db: &Pool, + client: &windmill_api_client::Client, + queued_job: &QueuedJob, + success: bool, + skipped: bool, + result: serde_json::Value, + logs: String, +) -> Result { + let mut tx = db.begin().await?; + let job_id = queued_job.id.clone(); + sqlx::query!( + "INSERT INTO completed_job AS cj + ( workspace_id + , id + , parent_job + , created_by + , created_at + , started_at + , duration_ms + , success + , script_hash + , script_path + , args + , result + , logs + , raw_code + , canceled + , canceled_by + , canceled_reason + , job_kind + , schedule_path + , permissioned_as + , flow_status + , raw_flow + , is_flow_step + , is_skipped + , language ) + VALUES ($1, $2, $3, $4, $5, $6, EXTRACT(milliseconds FROM (now() - $6)), $7, $8, $9,\ + $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) + ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)", + queued_job.workspace_id, + queued_job.id, + queued_job.parent_job, + queued_job.created_by, + queued_job.created_at, + queued_job.started_at, + success, + queued_job.script_hash.map(|x| x.0), + queued_job.script_path, + queued_job.args, + result, + logs, + queued_job.raw_code, + queued_job.canceled, + queued_job.canceled_by, + queued_job.canceled_reason, + queued_job.job_kind: JobKind, + queued_job.schedule_path, + queued_job.permissioned_as, + queued_job.flow_status, + queued_job.raw_flow, + queued_job.is_flow_step, + skipped, + queued_job.language: ScriptLang, + ) + .execute(&mut tx) + .await + .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?; + let _ = delete_job(db, &queued_job.workspace_id, job_id).await?; + if !queued_job.is_flow_step + && queued_job.job_kind != JobKind::Flow + && queued_job.job_kind != JobKind::FlowPreview + && queued_job.schedule_path.is_some() + && queued_job.script_path.is_some() + { + tx = schedule_again_if_scheduled( + tx, + client, + queued_job.schedule_path.as_ref().unwrap(), + queued_job.script_path.as_ref().unwrap(), + &queued_job.workspace_id, + ) + .await?; + } + tx.commit().await?; + tracing::debug!("Added completed job {}", queued_job.id); + Ok(queued_job.id) +} + +#[instrument(level = "trace", skip_all)] +pub async fn schedule_again_if_scheduled<'c>( + mut tx: Transaction<'c, Postgres>, + client: &windmill_api_client::Client, + schedule_path: &str, + script_path: &str, + w_id: &str, +) -> windmill_common::error::Result> { + let schedule = client + .get_schedule(w_id, schedule_path) + .await + .map_err(|_| { + Error::InternalErr(format!( + "Could not find schedule {:?} for workspace {}", + schedule_path, w_id + )) + })? + .into_inner(); + if schedule.enabled && script_path == schedule.script_path { + tx = windmill_queue::schedule::push_scheduled_job( + tx, + windmill_queue::schedule::Schedule { + workspace_id: w_id.to_owned(), + path: schedule.path, + edited_by: schedule.edited_by, + edited_at: schedule.edited_at, + schedule: schedule.schedule, + offset_: schedule.offset as _, + enabled: schedule.enabled, + script_path: schedule.script_path, + is_flow: schedule.is_flow, + args: schedule + .args + .and_then(|e| serde_json::to_value(e).map_or(None, |v| Some(v))), + extra_perms: serde_json::to_value(schedule.extra_perms).expect("hashmap -> json"), + }, + ) + .await?; + } + + Ok(tx) +} diff --git a/backend/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs similarity index 82% rename from backend/src/js_eval.rs rename to backend/windmill-worker/src/js_eval.rs index bf3d87f990..21ff4915f6 100644 --- a/backend/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -14,8 +14,7 @@ use regex::Regex; use serde_json::Value; use tokio::{sync::oneshot, time::timeout}; use uuid::Uuid; - -use crate::{client, error::Error}; +use windmill_common::error::Error; pub struct EvalCreds { pub workspace: String, @@ -136,25 +135,6 @@ fn add_closing_bracket(s: &str) -> String { s } -pub fn eval_sync(code: &str) -> Result { - let mut context = JsRuntime::new(RuntimeOptions::default()); - let code = format!("let x = {}; x", code); - let res = context.execute_script("", &code); - match res { - Ok(global) => { - let scope = &mut context.handle_scope(); - let local = v8::Local::new(scope, global); - let deserialized_value = serde_v8::from_v8::(scope, local); - - match deserialized_value { - Ok(value) => Ok(value), - Err(err) => Err(format!("Cannot deserialize value: {:?}", err)), - } - } - Err(err) => Err(format!("Evaling error: {:?}", err)), - } -} - const SPLIT_PAT: &str = ";\n"; async fn eval( context: &mut JsRuntime, @@ -279,34 +259,30 @@ async function resource(path) {{ // Ok(path) // } +// TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client? #[op] async fn op_variable(args: Vec) -> Result { let workspace = &args[0]; let path = &args[1]; let token = &args[2]; let base_url = &args[3]; - client::get_variable(workspace, path, token, &base_url).await + let client = windmill_api_client::create_client(base_url, token.clone()); + let result = client.get_variable(workspace, path, None).await?; + Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned())) } #[op] -async fn op_get_result(args: Vec) -> Result, anyhow::Error> { +async fn op_get_result( + args: Vec, +) -> Result { let workspace = &args[0]; let id = &args[1]; let token = &args[2]; let base_url = &args[3]; - let client = reqwest::Client::new(); - let result = client - .get(format!( - "{base_url}/api/w/{workspace}/jobs/completed/get_result/{id}" - )) - .bearer_auth(token) - .send() - .await - .map_err(|e| anyhow::anyhow!("error getting result for {id}: {}", e))? - .json::>() - .await - .map_err(|e| anyhow::anyhow!("error getting result for {id}: {}", e))?; - Ok(result) + let client = windmill_api_client::create_client(base_url, token.clone()); + let result = client.get_completed_job(workspace, &id.parse()?).await?; + // TODO: verify this works. Previously this returned Option, now it's statically typed. + Ok(result.into_inner()) } #[op] @@ -317,29 +293,27 @@ async fn op_get_id(args: Vec) -> Result, anyho let base_url = &args[3]; let node_id = &args[4]; - let client = reqwest::Client::new(); + let client = windmill_api_client::create_client(base_url, token.clone()); let result = client - .get(format!( - "{base_url}/api/w/{workspace}/jobs/result_by_id/{flow_job_id}/{node_id}?skip_direct=true" - )) - .bearer_auth(token) - .send() + .result_by_id(workspace, flow_job_id, node_id, Some(true)) .await - .map_err(|e| anyhow::anyhow!("error getting result for flow {flow_job_id} and node {node_id}: {}", e))? - .json::>() - .await - .map_err(|e| anyhow::anyhow!("error getting result for flow {flow_job_id} and node {node_id}: {}", e))?; + .map_or(None, |e| Some(e.into_inner())); Ok(result) } #[op] -async fn op_resource(args: Vec) -> Result, anyhow::Error> { +async fn op_resource( + args: Vec, +) -> Result { let workspace = &args[0]; let path = &args[1]; let token = &args[2]; let base_url = &args[3]; - client::get_resource(workspace, path, token, &base_url).await + let client = windmill_api_client::create_client(base_url, token.clone()); + let result = client.get_resource(workspace, path).await?; + // TODO: verify this works. Previously this returned Option, now it's statically typed. + Ok(result.into_inner()) } #[cfg(test)] diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs new file mode 100644 index 0000000000..a9abf4728f --- /dev/null +++ b/backend/windmill-worker/src/lib.rs @@ -0,0 +1,6 @@ +mod jobs; +mod js_eval; +mod worker; +mod worker_flow; + +pub use worker::*; diff --git a/backend/windmill-worker/src/main.rs b/backend/windmill-worker/src/main.rs new file mode 100644 index 0000000000..81bf2bc5ef --- /dev/null +++ b/backend/windmill-worker/src/main.rs @@ -0,0 +1,133 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::{net::SocketAddr, time::Duration}; + +use anyhow::Context; +use sqlx::{postgres::PgPoolOptions, Pool, Postgres}; +use windmill_common::{ + error::{self, Error}, + utils::rd_string, +}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // dotenv().ok(); + + windmill_common::tracing_init::initialize_tracing(); + + let db = async { + let database_url = std::env::var("DATABASE_URL") + .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?; + + let max_connections = match std::env::var("DATABASE_CONNECTIONS") { + Ok(n) => n.parse::().context("invalid DATABASE_CONNECTIONS")?, + Err(_) => 10, + }; + + Ok::, error::Error>( + PgPoolOptions::new() + .max_connections(max_connections) + .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins + .connect(&database_url) + .await + .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?, + ) + } + .await?; + + let metrics_addr: Option = std::env::var("METRICS_ADDR") + .ok() + .map(|s| { + s.parse::() + .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001)))) + .or_else(|_| s.parse::().map(Some)) + }) + .transpose()? + .flatten(); + + let (tx, rx) = tokio::sync::broadcast::channel::<()>(3); + let shutdown_signal = windmill_common::shutdown_signal(tx); + + let base_internal_url = + std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string()); + + let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()); + + let timeout = std::env::var("TIMEOUT") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(windmill_common::DEFAULT_TIMEOUT); + + let workers_f = async { + let sleep_queue = std::env::var("SLEEP_QUEUE") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE); + let disable_nuser = std::env::var("DISABLE_NUSER") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + let disable_nsjail = std::env::var("DISABLE_NSJAIL") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + let keep_job_dir = std::env::var("KEEP_JOB_DIR") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + + tracing::info!( + "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \ + {base_url}, SLEEP_QUEUE: {sleep_queue}, TIMEOUT: \ + {timeout}, KEEP_JOB_DIR: {keep_job_dir}" + ); + let instance_name = rd_string(5); + + let ip = windmill_common::external_ip::get_ip() + .await + .unwrap_or_else(|e| { + tracing::warn!(error = e.to_string(), "failed to get external IP"); + "unretrievable IP".to_string() + }); + let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5)); + windmill_worker::run_worker( + &db.clone(), + timeout, + &instance_name, + worker_name, + 1, + 1, + &ip, + sleep_queue, + windmill_worker::WorkerConfig { + disable_nsjail, + disable_nuser, + base_internal_url, + base_url, + keep_job_dir, + }, + rx.resubscribe(), + ) + .await; + Ok(()) as anyhow::Result<()> + }; + + let metrics_f = async { + match metrics_addr { + Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe()) + .await + .map_err(anyhow::Error::from), + None => Ok(()), + } + }; + + futures::try_join!(shutdown_signal, workers_f, metrics_f)?; + + Ok(()) +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs new file mode 100644 index 0000000000..e91e41f508 --- /dev/null +++ b/backend/windmill-worker/src/worker.rs @@ -0,0 +1,2221 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use itertools::Itertools; +use sqlx::{Pool, Postgres, Transaction}; +use std::{borrow::Borrow, collections::HashMap, io, panic, process::Stdio, time::Duration}; +use tracing::{trace_span, Instrument}; +use uuid::Uuid; +use windmill_common::{ + error::{self, to_anyhow, Error}, + scripts::{ScriptHash, ScriptLang}, + variables, +}; +use windmill_queue::{ + canceled_job_to_result, get_hub_script, get_queued_job, pull, JobKind, QueuedJob, +}; + +use serde_json::{json, Map, Value}; + +use tokio::{ + fs::{metadata, symlink, DirBuilder, File}, + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, + process::{Child, Command}, + sync::{ + mpsc::{self, Sender}, + oneshot, watch, + }, + time::{interval, sleep, Instant, MissedTickBehavior}, +}; + +use futures::{ + future::{self, ready, FutureExt}, + stream::{self, StreamExt}, +}; + +use async_recursion::async_recursion; + +use crate::{ + jobs::{add_completed_job, add_completed_job_error}, + worker_flow::{ + handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, + }, +}; + +#[tracing::instrument(level = "trace", skip_all)] +pub async fn create_token_for_owner<'c>( + mut tx: Transaction<'c, Postgres>, + w_id: &str, + owner: &str, + label: &str, + expires_in: i32, + username: &str, +) -> error::Result<(Transaction<'c, Postgres>, String)> { + // TODO: Bad implementation. We should not have access to this DB here. + use rand::prelude::*; + let token: String = rand::thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(30) + .map(char::from) + .collect(); + let is_super_admin = username.contains('@') + && sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + owner.split_once('/').map(|x| x.1).unwrap_or("") + ) + .fetch_optional(&mut tx) + .await? + .unwrap_or(false); + + let expiration = sqlx::query_scalar!( + "INSERT INTO token + (workspace_id, token, owner, label, expiration, super_admin) + VALUES ($1, $2, $3, $4, now() + ($5 || ' seconds')::interval, $6) RETURNING expiration", + &w_id, + token, + owner, + label, + expires_in.to_string(), + is_super_admin + ) + .fetch_one(&mut tx) + .await?; + + let mut truncated_token = token[..10].to_owned(); + truncated_token.push_str("*****"); + + windmill_audit::audit_log( + &mut tx, + &username, + "users.token.create", + windmill_audit::ActionKind::Create, + w_id, + Some(&truncated_token), + Some( + [ + Some(("label", label)), + expiration + .map(|x| x.to_string()) + .as_ref() + .map(|exp| ("expiration", &exp[..])), + ] + .into_iter() + .flatten() + .collect(), + ), + ) + .await?; + Ok((tx, token)) +} + +const TMP_DIR: &str = "/tmp/windmill"; +const PIP_SUPERCACHE_DIR: &str = "/tmp/windmill/cache/pip_permanent"; +const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip"; +const DENO_CACHE_DIR: &str = "/tmp/windmill/cache/deno"; +const GO_CACHE_DIR: &str = "/tmp/windmill/cache/go"; +const NUM_SECS_ENV_CHECK: u64 = 15; +const DEFAULT_HEAVY_DEPS: [&str; 18] = [ + "numpy", + "pandas", + "anyio", + "attrs", + "certifi", + "h11", + "httpcore", + "httpx", + "idna", + "python-dateutil", + "rfc3986", + "six", + "sniffio", + "windmill-api", + "wmill", + "psycopg2-binary", + "matplotlib", + "seaborn", +]; + +const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); +const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); +const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); + +const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto"); + +const NSJAIL_CONFIG_RUN_DENO_CONTENT: &str = include_str!("../nsjail/run.deno.config.proto"); +const MAX_LOG_SIZE: u32 = 200000; +const GO_REQ_SPLITTER: &str = "//go.sum"; + +#[derive(Clone)] +pub struct Metrics { + pub worker_execution_failed: prometheus::IntCounter, +} + +#[derive(Clone, Debug)] +pub struct WorkerConfig { + pub base_internal_url: String, + pub base_url: String, + pub disable_nuser: bool, + pub disable_nsjail: bool, + pub keep_job_dir: bool, +} + +lazy_static::lazy_static! { + static ref WORKER_STARTED: prometheus::IntGauge = prometheus::register_int_gauge!( + "worker_started", + "Total number of workers started." + ) + .unwrap(); + static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( + "queue_zombie_restart_count", + "Total number of jobs restarted due to ping timeout." + ) + .unwrap(); + static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!( + "queue_zombie_delete_count", + "Total number of jobs deleted due to their ping timing out in an unrecoverable state." + ) + .unwrap(); + static ref WORKER_UPTIME_OPTS: prometheus::Opts = prometheus::opts!( + "worker_uptime", + "Total number of milliseconds since the worker has started" + ); +} + +#[tracing::instrument(level = "trace")] +pub async fn run_worker( + db: &Pool, + timeout: i32, + worker_instance: &str, + worker_name: String, + i_worker: u64, + num_workers: u64, + ip: &str, + sleep_queue: u64, + worker_config: WorkerConfig, + mut rx: tokio::sync::broadcast::Receiver<()>, +) { + let start_time = Instant::now(); + + let worker_dir = format!("{TMP_DIR}/{worker_name}"); + tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir"); + + for x in [ + &worker_dir, + PIP_SUPERCACHE_DIR, + PIP_CACHE_DIR, + DENO_CACHE_DIR, + GO_CACHE_DIR, + ] { + DirBuilder::new() + .recursive(true) + .create(x) + .await + .expect("could not create initial worker dir"); + } + + let _ = write_file( + &worker_dir, + "download_deps.py.sh", + INCLUDE_DEPS_PY_SH_CONTENT, + ) + .await; + + let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_ENV_CHECK + 1); + + insert_initial_ping(worker_instance, &worker_name, ip, db).await; + + let uptime_metric = prometheus::register_int_counter!(WORKER_UPTIME_OPTS + .clone() + .const_label("name", &worker_name)) + .unwrap(); + uptime_metric.inc_by( + ((Instant::now() - start_time).as_millis() - uptime_metric.get() as u128) + .try_into() + .unwrap(), + ); + + let worker_execution_duration = prometheus::register_histogram_vec!( + prometheus::HistogramOpts::new( + "worker_execution_duration", + "Duration between receiving a job and completing it", + ) + .const_label("name", &worker_name), + &["workspace_id", "language"], + ) + .expect("register prometheus metric"); + + let worker_execution_failed = prometheus::register_int_counter_vec!( + prometheus::Opts::new("worker_execution_failed", "Number of failed jobs",) + .const_label("name", &worker_name), + &["workspace_id", "language"], + ) + .expect("register prometheus metric"); + + let worker_execution_count = prometheus::register_int_counter_vec!( + prometheus::Opts::new("worker_execution_count", "Number of executed jobs",) + .const_label("name", &worker_name), + &["workspace_id", "language"], + ) + .expect("register prometheus metric"); + + let mut jobs_executed = 0; + + let deno_path = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); + let go_path = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string()); + let python_path = + std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); + let python_heavy_deps = std::env::var("PYTHON_HEAVY_DEPS") + .map(|x| x.split(',').map(|x| x.to_string()).collect::>()) + .unwrap_or_else(|_| vec![]); + let nsjail_path = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); + let path_env = std::env::var("PATH").unwrap_or_else(|_| String::new()); + let gopath_env = std::env::var("GOPATH").unwrap_or_else(|_| String::new()); + let home_env = std::env::var("HOME").unwrap_or_else(|_| String::new()); + let pip_index_url = std::env::var("PIP_INDEX_URL").ok(); + let pip_extra_index_url = std::env::var("PIP_EXTRA_INDEX_URL").ok(); + let pip_trusted_host = std::env::var("PIP_TRUSTED_HOST").ok(); + let envs = Envs { + deno_path, + go_path, + python_path, + python_heavy_deps, + nsjail_path, + path_env, + gopath_env, + home_env, + pip_index_url, + pip_extra_index_url, + pip_trusted_host, + }; + WORKER_STARTED.inc(); + + let (same_worker_tx, mut same_worker_rx) = mpsc::channel::(5); + + loop { + uptime_metric.inc_by( + ((Instant::now() - start_time).as_millis() - uptime_metric.get() as u128) + .try_into() + .unwrap(), + ); + let do_break = async { + if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK { + sqlx::query!( + "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2", + jobs_executed, + &worker_name + ) + .execute(db) + .await + .expect("update worker ping"); + + last_ping = Instant::now(); + } + + let (do_break, next_job) = async { + tokio::select! { + biased; + _ = rx.recv() => { + println!("received killpill for worker {}", i_worker); + (true, Ok(None)) + }, + Some(job_id) = same_worker_rx.recv() => { + (false, sqlx::query_as::<_, QueuedJob>("SELECT * FROM queue WHERE id = $1") + .bind(job_id) + .fetch_optional(db) + .await + .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()))) + }, + job = pull(&db) => (false, job), + } + }.instrument(trace_span!("worker_get_next_job")).await; + if do_break { + return true; + } + + match next_job { + Ok(Some(job)) => { + let label_values = [ + &job.workspace_id, + job.language.as_ref().map(|l| l.as_str()).unwrap_or(""), + ]; + + let _timer = worker_execution_duration + .with_label_values(label_values.as_slice()) + .start_timer(); + + jobs_executed += 1; + worker_execution_count + .with_label_values(label_values.as_slice()) + .inc(); + + let metrics = Metrics { + worker_execution_failed: worker_execution_failed + .with_label_values(label_values.as_slice()), + }; + + tracing::info!(worker = %worker_name, id = %job.id, "fetched job {}", job.id); + + let job_dir = format!("{worker_dir}/{}", job.id); + + DirBuilder::new() + .create(&job_dir) + .await + .expect("could not create job dir"); + + let same_worker = job.same_worker; + let is_flow = job.job_kind == JobKind::Flow || job.job_kind == JobKind::FlowPreview; + + if is_flow && same_worker { + let target = &format!("{job_dir}/shared"); + if let Some(parent_flow) = job.parent_job { + let parent_shared_dir = format!("{worker_dir}/{parent_flow}/shared"); + if metadata(&parent_shared_dir).await.is_err() { + DirBuilder::new() + .recursive(true) + .create(&parent_shared_dir) + .await + .expect("could not create parent shared dir"); + } + symlink(&parent_shared_dir, target) + .await + .expect("could not symlink target"); + } else { + DirBuilder::new() + .create(target) + .await + .expect("could not create shared dir"); + } + } + let tx = db.begin().await.expect("could not start token transaction"); + let (tx, token) = create_token_for_owner( + tx, + &job.workspace_id, + &job.permissioned_as, + "ephemeral-script", + timeout * 2, + &job.created_by, + ) + .await.expect("could not create job token"); + tx.commit().await.expect("could not commit job token"); + let job_client = windmill_api_client::create_client(&worker_config.base_url, token.clone()); + + if let Some(err) = handle_queued_job( + job.clone(), + db, + &job_client, + token, + timeout, + &worker_name, + &worker_dir, + &job_dir, + &worker_config, + metrics.clone(), + &envs, + same_worker_tx.clone(), + &worker_config.base_internal_url, + ) + .await + .err() + { + handle_job_error( + db, + &job_client, + job, + err, + Some(metrics), + false, + same_worker_tx.clone(), + &worker_dir, + !worker_config.keep_job_dir, + &worker_config.base_internal_url, + ) + .await; + }; + + if !worker_config.keep_job_dir && !(is_flow && same_worker) { + let _ = tokio::fs::remove_dir_all(job_dir).await; + } + } + Ok(None) => { + tokio::time::sleep(Duration::from_millis(sleep_queue * num_workers)).await + } + Err(err) => { + tracing::error!(worker = %worker_name, "run_worker: pulling jobs: {}", err); + } + }; + + false + } + .instrument(trace_span!("worker_loop_iteration")) + .await; + if do_break { + break; + } + } +} + +async fn handle_job_error( + db: &Pool, + client: &windmill_api_client::Client, + job: QueuedJob, + err: Error, + metrics: Option, + unrecoverable: bool, + same_worker_tx: Sender, + worker_dir: &str, + keep_job_dir: bool, + base_internal_url: &str, +) { + let m = add_completed_job_error( + db, + client, + &job, + "Unexpected error during job execution:\n".to_string(), + &err, + metrics.clone(), + ) + .await + .map(|(_, m)| m) + .unwrap_or_else(|_| Map::new()); + + if let Some(parent_job_id) = job.parent_job { + let updated_flow = update_flow_status_after_job_completion( + db, + client, + &job, + false, + serde_json::Value::Object(m), + metrics.clone(), + unrecoverable, + same_worker_tx, + worker_dir, + keep_job_dir, + base_internal_url, + ) + .await; + if let Err(err) = updated_flow { + if let Ok(mut tx) = db.begin().await { + if let Ok(Some(parent_job)) = + get_queued_job(parent_job_id, &job.workspace_id, &mut tx).await + { + let _ = add_completed_job_error( + db, + client, + &parent_job, + format!("Unexpected error during flow job error handling:\n{err}"), + err, + metrics, + ) + .await; + } + } + } + } + tracing::error!(job_id = %job.id, err = err.alt(), "error handling job: {} {} {}", job.id, job.workspace_id, job.created_by); +} + +async fn insert_initial_ping( + worker_instance: &str, + worker_name: &str, + ip: &str, + db: &Pool, +) { + sqlx::query!( + "INSERT INTO worker_ping (worker_instance, worker, ip) VALUES ($1, $2, $3)", + worker_instance, + worker_name, + ip + ) + .execute(db) + .await + .expect("insert worker_ping initial value"); +} + +struct Envs { + deno_path: String, + go_path: String, + python_path: String, + python_heavy_deps: Vec, + nsjail_path: String, + path_env: String, + gopath_env: String, + home_env: String, + pip_index_url: Option, + pip_extra_index_url: Option, + pip_trusted_host: Option, +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_queued_job( + job: QueuedJob, + db: &sqlx::Pool, + client: &windmill_api_client::Client, + token: String, + timeout: i32, + worker_name: &str, + worker_dir: &str, + job_dir: &str, + worker_config: &WorkerConfig, + metrics: Metrics, + envs: &Envs, + same_worker_tx: Sender, + base_internal_url: &str, +) -> windmill_common::error::Result<()> { + if job.canceled { + return Err(Error::ExecutionErr(canceled_job_to_result(&job)))?; + } + match job.job_kind { + JobKind::FlowPreview | JobKind::Flow => { + let args = job.args.clone().unwrap_or(Value::Null); + handle_flow( + &job, + db, + client, + args, + same_worker_tx, + worker_dir, + base_internal_url, + ) + .await?; + } + _ => { + let mut logs = "".to_string(); + + if job.is_flow_step { + update_flow_status_in_progress( + db, + &job.workspace_id, + job.parent_job + .ok_or_else(|| Error::InternalErr(format!("expected parent job")))?, + job.id, + ) + .await?; + } + + tracing::info!( + worker = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "handling job {}", + job.id + ); + + logs.push_str(&format!("job {} on worker {}\n", &job.id, &worker_name)); + let result = match job.job_kind { + JobKind::Dependencies => { + handle_dependency_job(&job, &mut logs, job_dir, db, timeout, &envs).await + } + JobKind::Identity => Ok(job.args.clone().unwrap_or_else(|| Value::Null)), + _ => { + handle_code_execution_job( + &job, + db, + client, + token, + job_dir, + worker_dir, + &mut logs, + timeout, + worker_config, + envs, + ) + .await + } + }; + + match result { + Ok(r) => { + add_completed_job(db, client, &job, true, false, r.clone(), logs).await?; + if job.is_flow_step { + update_flow_status_after_job_completion( + db, + client, + &job, + true, + r, + Some(metrics.clone()), + false, + same_worker_tx.clone(), + worker_dir, + worker_config.keep_job_dir, + &worker_config.base_internal_url, + ) + .await?; + } + } + Err(e) => { + let error_message = match e { + Error::ExitStatus(_) => { + let last_10_log_lines = logs + .lines() + .skip(logs.lines().count().max(10) - 10) + .join("\n") + .to_string() + .replace("\n\n", "\n"); + + let log_lines = last_10_log_lines + .split("CODE EXECUTION ---") + .last() + .unwrap_or(&logs); + format!("Error during execution of the script:\n{}", log_lines) + } + err @ _ => format!("error before termination: {err:#?}"), + }; + + let (_, output_map) = add_completed_job_error( + db, + client, + &job, + logs, + error_message, + Some(metrics.clone()), + ) + .await?; + if job.is_flow_step { + update_flow_status_after_job_completion( + db, + client, + &job, + false, + serde_json::Value::Object(output_map), + Some(metrics), + false, + same_worker_tx, + worker_dir, + worker_config.keep_job_dir, + &worker_config.base_internal_url, + ) + .await?; + } + } + }; + } + } + Ok(()) +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn write_file(dir: &str, path: &str, content: &str) -> error::Result { + let path = format!("{}/{}", dir, path); + let mut file = File::create(&path).await?; + file.write_all(content.as_bytes()).await?; + file.flush().await?; + Ok(file) +} + +#[async_recursion] +async fn transform_json_value( + client: &windmill_api_client::Client, + workspace: &str, + v: Value, +) -> error::Result { + match v { + Value::String(y) if y.starts_with("$var:") => { + let path = y.strip_prefix("$var:").unwrap(); + let v = client + .get_variable(workspace, path, None) + .await + .map_err(to_anyhow) + .map(|v| v.into_inner())? + .value + .map_or_else( + || Err(Error::NotFound(format!("Variable not found at {path}"))), + |e| Ok(e), + )?; + Ok(Value::String(v)) + } + Value::String(y) if y.starts_with("$res:") => { + let path = y.strip_prefix("$res:").unwrap(); + if path.split("/").count() < 2 { + return Err(Error::InternalErr( + format!("invalid resource path: {path}",), + )); + } + let v = client + .get_resource(workspace, path) + .await + .map_err(to_anyhow)?; + transform_json_value( + client, + workspace, + serde_json::to_value(v.into_inner()).map_err(to_anyhow)?, + ) + .await + } + Value::Object(mut m) => { + for (a, b) in m.clone().into_iter() { + m.insert(a, transform_json_value(client, workspace, b).await?); + } + Ok(Value::Object(m)) + } + a @ _ => Ok(a), + } +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_code_execution_job( + job: &QueuedJob, + db: &sqlx::Pool, + client: &windmill_api_client::Client, + token: String, + job_dir: &str, + worker_dir: &str, + logs: &mut String, + timeout: i32, + worker_config: &WorkerConfig, + envs: &Envs, +) -> error::Result { + let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview) + || (matches!(job.job_kind, JobKind::Script_Hub) && job.language == Some(ScriptLang::Deno)) + { + let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); + (code, None, job.language.to_owned()) + } else if matches!(job.job_kind, JobKind::Script_Hub) { + let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); + let script = get_hub_script( + job.script_path + .clone() + .unwrap_or_else(|| "missing script path".to_string()), + None, + &job.created_by, + ) + .await?; + (code, script.lockfile, job.language.to_owned()) + } else { + sqlx::query_as::<_, (String, Option, Option)>( + "SELECT content, lock, language FROM script WHERE hash = $1 AND (workspace_id = $2 OR \ + workspace_id = 'starter')", + ) + .bind(&job.script_hash.unwrap_or(ScriptHash(0)).0) + .bind(&job.workspace_id) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))? + }; + let worker_name = worker_dir.split("/").last().unwrap_or("unknown"); + let lang_str = job + .language + .as_ref() + .map(|x| format!("{x:?}")) + .unwrap_or_else(|| "NO_LANG".to_string()); + + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "started {} job {}", + &lang_str, + job.id + ); + + let shared_mount = if job.same_worker { + format!( + r#" +mount {{ + src: "{worker_dir}/{}/shared" + dst: "/shared" + is_bind: true + rw: true +}} + "#, + job.parent_job.ok_or(Error::ExecutionErr( + "no parent job, required for same worker job".to_string() + ))?, + ) + } else { + "".to_string() + }; + let result: error::Result = match language { + None => { + return Err(Error::ExecutionErr( + "Require language to be not null".to_string(), + ))?; + } + Some(ScriptLang::Python3) => { + handle_python_job( + worker_config, + envs, + requirements_o, + job_dir, + worker_dir, + worker_name, + job, + logs, + db, + client, + token, + timeout, + &inner_content, + &shared_mount, + ) + .await + } + Some(ScriptLang::Deno) => { + handle_deno_job( + worker_config, + envs, + logs, + job, + db, + client, + token, + job_dir, + &inner_content, + timeout, + &shared_mount, + ) + .await + } + Some(ScriptLang::Go) => { + handle_go_job( + worker_config, + envs, + logs, + job, + db, + client, + token, + &inner_content, + timeout, + job_dir, + requirements_o, + &shared_mount, + ) + .await + } + }; + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + is_ok = result.is_ok(), + "finished {} job {}", + &lang_str, + job.id + ); + result +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_go_job( + WorkerConfig { base_internal_url, disable_nuser, disable_nsjail, base_url, .. }: &WorkerConfig, + Envs { nsjail_path, go_path, path_env, gopath_env, home_env, .. }: &Envs, + logs: &mut String, + job: &QueuedJob, + db: &sqlx::Pool, + client: &windmill_api_client::Client, + token: String, + inner_content: &str, + timeout: i32, + job_dir: &str, + requirements_o: Option, + shared_mount: &str, +) -> Result { + //go does not like executing modules at temp root + let job_dir = &format!("{job_dir}/go"); + if let Some(requirements) = requirements_o { + gen_go_mymod(inner_content, job_dir).await?; + let (md, sum) = requirements + .split_once(GO_REQ_SPLITTER) + .ok_or(Error::ExecutionErr( + "Invalid requirement file, missing splitter".to_string(), + ))?; + write_file(job_dir, "go.mod", md).await?; + write_file(job_dir, "go.sum", sum).await?; + } else { + logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n"); + set_logs(logs, job.id, db).await; + + install_go_dependencies( + &job.id, + inner_content, + logs, + job_dir, + db, + timeout, + go_path, + true, + ) + .await?; + } + + logs.push_str("\n\n--- GO CODE EXECUTION ---\n"); + set_logs(logs, job.id, db).await; + create_args_and_out_file(client, job, job_dir).await?; + { + let sig = windmill_parser_go::parse_go_sig(&inner_content)?; + drop(inner_content); + + const WRAPPER_CONTENT: &str = r#"package main + +import ( + "encoding/json" + "os" + "fmt" + "mymod/inner" +) + +func main() {{ + + dat, err := os.ReadFile("args.json") + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + + var req inner.Req + + if err := json.Unmarshal(dat, &req); err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + + res, err := inner.Run(req) + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + res_json, err := json.Marshal(res) + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend) + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + _, err = f.WriteString(string(res_json)) + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} +}}"#; + + write_file(job_dir, "main.go", WRAPPER_CONTENT).await?; + + { + let spread = &sig + .args + .clone() + .into_iter() + .map(|x| format!("req.{}", capitalize(&x.name))) + .join(", "); + let req_body = &sig + .args + .into_iter() + .map(|x| { + format!( + "{} {} `json:\"{}\"`", + capitalize(&x.name), + windmill_parser_go::otyp_to_string(x.otyp), + x.name + ) + }) + .join("\n"); + let runner_content: String = format!( + r#"package inner +type Req struct {{ + {req_body} +}} + +func Run(req Req) (interface{{}}, error){{ + return main({spread}) +}} + +"#, + ); + write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?; + } + } + let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + + let child = if !disable_nsjail { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_GO_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", GO_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()) + .replace("{SHARED_MOUNT}", shared_mount), + ) + .await?; + + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("GOMEMLIMIT", "2000MiB") + .args(vec![ + "--config", + "run.config.proto", + "--", + go_path, + "run", + "main.go", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(go_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("GOPATH", gopath_env) + .env("HOME", home_env) + .args(vec!["run", "main.go"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + handle_child(&job.id, db, logs, timeout, child).await?; + read_result(job_dir).await +} + +fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_deno_job( + WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, + Envs { nsjail_path, deno_path, path_env, .. }: &Envs, + logs: &mut String, + job: &QueuedJob, + db: &sqlx::Pool, + client: &windmill_api_client::Client, + token: String, + job_dir: &str, + inner_content: &String, + timeout: i32, + shared_mount: &str, +) -> error::Result { + logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); + set_logs(logs, job.id, db).await; + let _ = write_file(job_dir, "inner.ts", inner_content).await?; + let sig = trace_span!("parse_deno_signature") + .in_scope(|| windmill_parser_ts::parse_deno_signature(inner_content))?; + create_args_and_out_file(client, job, job_dir).await?; + let spread = sig.args.into_iter().map(|x| x.name).join(","); + let wrapper_content: String = format!( + r#" +import {{ main }} from "./inner.ts"; + +const args = await Deno.readTextFile("args.json") + .then(JSON.parse) + .then(({{ {spread} }}) => [ {spread} ]) + +async function run() {{ + let res: any = await main(...args); + const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); + await Deno.writeTextFile("result.json", res_json); + Deno.exit(0); +}} +run(); +"#, + ); + write_file(job_dir, "main.ts", &wrapper_content).await?; + let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + + let hostname_base = base_url.split("://").last().unwrap_or("localhost"); + let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost"); + let deno_auth_tokens = format!("{token}@{hostname_base};{token}@{hostname_internal}"); + let child = async { + Ok(if !disable_nsjail { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_DENO_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", DENO_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()) + .replace("{SHARED_MOUNT}", shared_mount), + ) + .await?; + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("DENO_AUTH_TOKENS", deno_auth_tokens) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "--config", + "run.config.proto", + "--", + deno_path, + "run", + "--unstable", + "--v8-flags=--max-heap-size=2048", + "-A", + "/tmp/main.ts", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(deno_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("DENO_AUTH_TOKENS", deno_auth_tokens) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "run", + "--unstable", + "--v8-flags=--max-heap-size=2048", + "-A", + "main.ts", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }) as error::Result<_> + } + .instrument(trace_span!("create_deno_jail")) + .await?; + handle_child(&job.id, db, logs, timeout, child).await?; + read_result(job_dir).await +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn create_args_and_out_file( + client: &windmill_api_client::Client, + job: &QueuedJob, + job_dir: &str, +) -> Result<(), Error> { + let args = if let Some(args) = &job.args { + Some(transform_json_value(client, &job.workspace_id, args.clone()).await?) + } else { + None + }; + let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; + write_file(job_dir, "args.json", &ser_args).await?; + write_file(job_dir, "result.json", "").await?; + Ok(()) +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_python_job( + WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, + envs @ Envs { + nsjail_path, + python_path, + python_heavy_deps, + path_env, + pip_extra_index_url, + pip_index_url, + pip_trusted_host, + .. + }: &Envs, + requirements_o: Option, + job_dir: &str, + worker_dir: &str, + worker_name: &str, + job: &QueuedJob, + logs: &mut String, + db: &sqlx::Pool, + client: &windmill_api_client::Client, + token: String, + timeout: i32, + inner_content: &String, + shared_mount: &str, +) -> error::Result { + create_dependencies_dir(job_dir).await; + + let mut additional_python_paths: Vec = vec![]; + + let requirements = match requirements_o { + Some(r) => r, + None => { + let requirements = windmill_parser_py::parse_python_imports(&inner_content)?.join("\n"); + if requirements.is_empty() { + "".to_string() + } else { + pip_compile(job, &requirements, logs, job_dir, envs, db, timeout) + .await? + .map_err(|e| { + Error::ExecutionErr(format!("pip compile failed: {}", e.to_string())) + })? + } + } + }; + + if requirements.len() > 0 { + if !disable_nsjail { + let _ = write_file( + job_dir, + "download.config.proto", + &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{WORKER_DIR}", &worker_dir) + .replace("{CACHE_DIR}", PIP_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), + ) + .await?; + } + + let mut heavy_deps = DEFAULT_HEAVY_DEPS + .iter() + .map(|s| s.to_string()) + .collect::>(); + heavy_deps.extend(python_heavy_deps.into_iter().map(|s| s.to_string())); + + let (heavy, regular): (Vec<&str>, Vec<&str>) = requirements + .split("\n") + .partition(|d| heavy_deps.iter().any(|hd| d.starts_with(hd))); + + let _ = write_file(job_dir, "requirements.txt", ®ular.join("\n")).await?; + + let mut vars = vec![]; + if let Some(url) = pip_extra_index_url { + vars.push(("EXTRA_INDEX_URL", url)); + } + if let Some(url) = pip_index_url { + vars.push(("INDEX_URL", url)); + } + if let Some(host) = pip_trusted_host { + vars.push(("TRUSTED_HOST", host)); + } + + if heavy.len() > 0 { + logs.push_str(&format!( + "\nheavy deps detected, using supercache for: {heavy:?}" + )); + additional_python_paths = + handle_python_heavy_reqs(python_path, heavy, vars.clone(), job, logs, db, timeout) + .await?; + } + + if regular.len() > 0 { + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "started setup python dependencies" + ); + + let child = if !disable_nsjail { + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(vars) + .args(vec!["--config", "download.config.proto"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + let mut args = vec![ + "-m", + "pip", + "install", + "--no-deps", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + "./dependencies", + "-r", + "./requirements.txt", + ]; + if let Some(url) = pip_extra_index_url { + args.extend(["--extra-index-url", url]); + } + if let Some(url) = pip_index_url { + args.extend(["--index-url", url]); + } + if let Some(host) = pip_trusted_host { + args.extend(["--trusted-host", host]); + } + Command::new(python_path) + .current_dir(job_dir) + .env_clear() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + + logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n"); + let child = handle_child(&job.id, db, logs, timeout, child).await; + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + is_ok = child.is_ok(), + "finished setting up python dependencies {}", + job.id + ); + child?; + } else { + logs.push_str("\nskipping pip install since not needed"); + }; + } + logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n"); + + set_logs(logs, job.id, db).await; + + let _ = write_file(job_dir, "inner.py", inner_content).await?; + + let sig = windmill_parser_py::parse_python_signature(inner_content)?; + let transforms = sig + .args + .into_iter() + .map(|x| match x.typ { + windmill_parser::Typ::Bytes => { + format!( + "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ + kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", + x.name, x.name, x.name, x.name + ) + } + windmill_parser::Typ::Datetime => { + format!( + "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ + kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], \ + '%Y-%m-%dT%H:%M')\n", + x.name, x.name, x.name, x.name + ) + } + _ => "".to_string(), + }) + .collect::>() + .join(""); + create_args_and_out_file(client, job, job_dir).await?; + + let wrapper_content: String = format!( + r#" +import json +import base64 +from datetime import datetime + +inner_script = __import__("inner") + +with open("args.json") as f: + kwargs = json.load(f, strict=False) +for k, v in list(kwargs.items()): + if v == '': + del kwargs[k] +{transforms} +res = inner_script.main(**kwargs) +res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '') +with open("result.json", 'w') as f: + f.write(res_json) +"#, + ); + write_file(job_dir, "main.py", &wrapper_content).await?; + + let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; + let additional_python_paths_folders = additional_python_paths + .iter() + .map(|x| format!(":{x}")) + .join(""); + if !disable_nsjail { + let shared_deps = additional_python_paths + .into_iter() + .map(|pp| { + format!( + r#" +mount {{ + src: "{pp}" + dst: "{pp}" + is_bind: true + rw: false +}} + "# + ) + }) + .join("\n"); + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) + .replace( + "{ADDITIONAL_PYTHON_PATHS}", + additional_python_paths_folders.as_str(), + ), + ) + .await?; + } else { + reserved_variables.insert( + "PYTHONPATH".to_string(), + format!("{job_dir}/dependencies{additional_python_paths_folders}"), + ); + } + + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "started python code execution {}", + job.id + ); + let child = if !disable_nsjail { + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + // inject PYTHONPATH here - for some reason I had to do it in nsjail conf + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "--config", + "run.config.proto", + "--", + python_path, + "-u", + "/tmp/main.py", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(python_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec!["-u", "main.py"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + + handle_child(&job.id, db, logs, timeout, child).await?; + read_result(job_dir).await +} + +async fn create_dependencies_dir(job_dir: &str) { + DirBuilder::new() + .recursive(true) + .create(&format!("{job_dir}/dependencies")) + .await + .expect("could not create dependencies dir"); +} + +async fn read_result(job_dir: &str) -> error::Result { + let mut file = File::open(format!("{job_dir}/result.json")).await?; + let mut content = "".to_string(); + file.read_to_string(&mut content).await?; + serde_json::from_str(&content) + .map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}"))) +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_dependency_job( + job: &QueuedJob, + logs: &mut String, + job_dir: &str, + db: &sqlx::Pool, + timeout: i32, + envs: &Envs, +) -> error::Result { + let content: Result = match job.language { + Some(ScriptLang::Python3) => { + create_dependencies_dir(job_dir).await; + let requirements = &job + .raw_code + .as_ref() + .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))? + .clone(); + pip_compile(job, requirements, logs, job_dir, envs, db, timeout).await? + } + Some(ScriptLang::Go) => { + let requirements = job + .raw_code + .as_ref() + .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; + install_go_dependencies( + &job.id, + &requirements, + logs, + job_dir, + db, + timeout, + &envs.go_path, + false, + ) + .await + .map_err(|e| e.to_string()) + } + _ => Err("Language incompatible with dep job".to_string()), + }; + + match content { + Ok(content) => { + sqlx::query!( + "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", + &content, + &job.script_hash.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + Ok(json!({ "success": "Successful lock file generation", "lock": content })) + } + Err(error) => { + sqlx::query!( + "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", + &format!("{logs}\n{error}"), + &job.script_hash.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + Err(Error::ExecutionErr(format!("Error locking file: {error}")))? + } + } +} + +async fn pip_compile( + job: &QueuedJob, + requirements: &str, + logs: &mut String, + job_dir: &str, + Envs { pip_extra_index_url, pip_index_url, pip_trusted_host, .. }: &Envs, + db: &Pool, + timeout: i32, +) -> Result, Error> { + logs.push_str(&format!("content of requirements:\n{}\n", requirements)); + let file = "requirements.in"; + write_file(job_dir, file, &requirements).await?; + let mut args = vec!["-q", "--no-header", file]; + if let Some(url) = pip_extra_index_url { + args.extend(["--extra-index-url", url]); + } + if let Some(url) = pip_index_url { + args.extend(["--index-url", url]); + } + if let Some(host) = pip_trusted_host { + args.extend(["--trusted-host", host]); + } + let child = Command::new("pip-compile") + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + handle_child(&job.id, db, logs, timeout, child) + .await + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + let path_lock = format!("{job_dir}/requirements.txt"); + let mut file = File::open(path_lock).await?; + let mut req_content = "".to_string(); + file.read_to_string(&mut req_content).await?; + Ok(Ok(req_content + .lines() + .filter(|x| !x.trim_start().starts_with('#')) + .map(|x| x.to_string()) + .collect::>() + .join("\n"))) +} + +async fn install_go_dependencies( + job_id: &Uuid, + code: &str, + logs: &mut String, + job_dir: &str, + db: &sqlx::Pool, + timeout: i32, + go_path: &str, + preview: bool, +) -> error::Result { + gen_go_mymod(code, job_dir).await?; + let child = Command::new("go") + .current_dir(job_dir) + .args(vec!["mod", "init", "mymod"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + handle_child(job_id, db, logs, timeout, child).await?; + + let child = Command::new(go_path) + .current_dir(job_dir) + .env("GOMEMLIMIT", "2000MiB") + .args(vec!["mod", "tidy"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + handle_child(job_id, db, logs, timeout, child) + .await + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + + if preview { + Ok(String::new()) + } else { + let mut req_content = "".to_string(); + + let mut file = File::open(format!("{job_dir}/go.mod")).await?; + file.read_to_string(&mut req_content).await?; + + req_content.push_str(&format!("\n{GO_REQ_SPLITTER}\n")); + + if let Ok(mut file) = File::open(format!("{job_dir}/go.sum")).await { + file.read_to_string(&mut req_content).await?; + } + + Ok(req_content) + } +} + +async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> { + let code = if code.trim_start().starts_with("package") { + code.to_string() + } else { + format!("package inner; {code}") + }; + + let mymod_dir = format!("{job_dir}/inner"); + DirBuilder::new() + .recursive(true) + .create(&mymod_dir) + .await + .expect("could not create go's mymod dir"); + + write_file(&mymod_dir, "inner_main.go", &code).await?; + + Ok(()) +} + +// TODO: this really shouldn't be here +pub async fn get_email_from_username( + username: &String, + db: &Pool, +) -> error::Result> { + let email = sqlx::query_scalar!("SELECT email FROM usr WHERE username = $1", username) + .fetch_optional(db) + .await?; + Ok(email) +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn get_reserved_variables( + job: &QueuedJob, + token: &str, + base_url: &str, + db: &sqlx::Pool, +) -> Result, Error> { + let flow_path = if let Some(uuid) = job.parent_job { + sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid) + .fetch_optional(db) + .await? + .flatten() + } else { + None + }; + + let variables = variables::get_reserved_variables( + &job.workspace_id, + token, + &get_email_from_username(&job.created_by, db) + .await? + .unwrap_or_else(|| "nosuitable@email.xyz".to_string()), + &job.created_by, + &job.id.to_string(), + &job.permissioned_as, + base_url, + job.script_path.clone(), + job.parent_job.map(|x| x.to_string()), + flow_path, + job.schedule_path.clone(), + ); + Ok(variables + .into_iter() + .map(|rv| (rv.name, rv.value)) + .collect()) +} + +/// - wait until child exits and return with exit status +/// - read lines from stdout and stderr and append them to the "queue"."logs" +/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) +/// - update the `last_line` and `logs` strings with the program output +/// - update "queue"."last_ping" every five seconds +/// - kill process if we exceed timeout or "queue"."canceled" is set +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_child( + job_id: &Uuid, + db: &Pool, + logs: &mut String, + timeout: i32, + mut child: Child, +) -> error::Result<()> { + let timeout = Duration::from_secs(u64::try_from(timeout).expect("invalid timeout")); + let ping_interval = Duration::from_secs(5); + let cancel_check_interval = Duration::from_millis(500); + let write_logs_delay = Duration::from_millis(500); + + let (set_too_many_logs, mut too_many_logs) = watch::channel::(false); + + let output = child_joined_output_stream(&mut child); + let job_id = job_id.clone(); + + let (tx, mut rx) = mpsc::channel::<()>(1); + + /* the cancellation future is polled on by `wait_on_child` while + * waiting for the child to exit normally */ + let cancel_check = async { + let db = db.clone(); + + let mut interval = interval(cancel_check_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + loop { + tokio::select!( + _ = rx.recv() => break, + _ = interval.tick() => { + if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job_id) + .fetch_optional(&db) + .await + .map(|v| Some(true) == v) + .unwrap_or_else(|err| { + tracing::error!(%job_id, %err, "error checking cancelation for job {job_id}: {err}"); + false + }) + { + break; + } + }, + ); + } + }; + + #[derive(PartialEq, Debug)] + enum KillReason { + TooManyLogs, + Timeout, + Cancelled, + } + /* a future that completes when the child process exits */ + let wait_on_child = async { + let db = db.clone(); + + let kill_reason = tokio::select! { + biased; + result = child.wait() => return result.map(Ok), + Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs, + _ = cancel_check => KillReason::Cancelled, + _ = sleep(timeout) => KillReason::Timeout, + }; + tx.send(()).await.expect("rx should never be dropped"); + drop(tx); + + let set_reason = async { + if kill_reason == KillReason::Timeout { + if let Err(err) = sqlx::query( + r#" + UPDATE queue + SET canceled = true + , canceled_by = 'timeout', + , canceled_reason = $1 + WHERE id = $2 + r"#, + ) + .bind(format!("duration > {}", timeout.as_secs())) + .bind(job_id) + .execute(&db) + .await + { + tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}"); + } + } + }; + + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, child.kill()).await; + kill.map(|()| Err(kill_reason)) + }; + + /* a future that reads output from the child and appends to the database */ + let lines = async move { + /* log_remaining is zero when output limit was reached */ + let mut log_remaining = (MAX_LOG_SIZE as usize).saturating_sub(logs.chars().count()); + let mut result = io::Result::Ok(()); + let mut output = output; + /* `do_write` resolves the task, but does not contain the Result. + * It's useful to know if the task completed. */ + let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle(); + + while let Some(line) = output.by_ref().next().await { + let do_write_ = do_write.shared(); + + let mut read_lines = stream::once(async { line }) + .chain(output.by_ref()) + /* after receiving a line, continue until some delay has passed + * _and_ the previous database write is complete */ + .take_until(future::join(sleep(write_logs_delay), do_write_.clone())) + .boxed(); + + /* Read up until an error is encountered, + * handle log lines first and then the error... */ + let mut joined = String::new(); + + while let Some(line) = read_lines.next().await { + match line { + Ok(_) if log_remaining == 0 => (), + Ok(line) => { + append_with_limit(&mut joined, &line, &mut log_remaining); + + if log_remaining == 0 { + tracing::info!(%job_id, "Too many logs lines for job {job_id}"); + let _ = set_too_many_logs.send(true); + joined.push_str(&format!( + "Job logs or result reached character limit of {MAX_LOG_SIZE}; killing job." + )); + /* stop reading and drop our streams fairly quickly */ + break; + } + } + Err(err) => { + result = Err(err); + break; + } + } + } + + logs.push_str(&joined); + + /* Ensure the last flush completed before starting a new one. + * + * This shouldn't pause since `take_until()` reads lines until `do_write` + * resolves. We only stop reading lines before `take_until()` resolves if we reach + * EOF or a read error. In those cases, waiting on a database query to complete is + * fine because we're done. */ + + if let Some(Ok(p)) = do_write_ + .then(|()| write_result) + .await + .err() + .map(|err| err.try_into_panic()) + { + panic::resume_unwind(p); + } + + (do_write, write_result) = + tokio::spawn(append_logs(job_id, joined, db.clone())).remote_handle(); + + if let Err(err) = result { + tracing::error!(%job_id, %err, "error reading output for job {job_id}: {err}"); + break; + } + + if *set_too_many_logs.borrow() { + break; + } + } + + /* drop our end of the pipe */ + drop(output); + + if let Some(Ok(p)) = do_write + .then(|()| write_result) + .await + .err() + .map(|err| err.try_into_panic()) + { + panic::resume_unwind(p); + } + }.instrument(trace_span!("child_lines")); + + /* a stream updating "queue"."last_ping" at an interval */ + + let (kill_tx, mut kill_rx) = oneshot::channel::<()>(); + + let mut interval = interval(ping_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let db1 = db.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(err) = sqlx::query!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id) + .execute(&db1) + .await + { + tracing::error!(%job_id, %err, "error setting last ping for job {job_id}: {err}"); + }; + }, + _ = (&mut kill_rx) => return, + } + } + }); + let (wait_result, _) = tokio::join!(wait_on_child, lines); + kill_tx.send(()).expect("send should always work"); + + match wait_result { + _ if *too_many_logs.borrow() => Err(Error::ExecutionErr( + "logs or result reached limit".to_string(), + )), + Ok(Ok(status)) => { + if status.success() { + Ok(()) + } else if let Some(code) = status.code() { + Err(error::Error::ExitStatus(code)) + } else { + Err(error::Error::ExecutionErr( + "process terminated by signal".to_string(), + )) + } + } + Ok(Err(kill_reason)) => Err(Error::ExecutionErr(format!( + "job process killed because {kill_reason:#?}" + ))), + Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))), + } +} + +/// takes stdout and stderr from Child, panics if either are not present +/// +/// builds a stream joining both stdout and stderr each read line by line +fn child_joined_output_stream( + child: &mut Child, +) -> impl stream::FusedStream> { + let stderr = child + .stderr + .take() + .expect("child did not have a handle to stdout"); + + let stdout = child + .stdout + .take() + .expect("child did not have a handle to stdout"); + + let stdout = BufReader::new(stdout).lines(); + let stderr = BufReader::new(stderr).lines(); + stream::select(lines_to_stream(stderr), lines_to_stream(stdout)) +} + +fn lines_to_stream( + mut lines: tokio::io::Lines, +) -> impl futures::Stream> { + stream::poll_fn(move |cx| { + std::pin::Pin::new(&mut lines) + .poll_next_line(cx) + .map(|result| result.transpose()) + }) +} + +// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields, +// so this pushes \n to thd destination string in each call +fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { + if *limit > 0 { + dst.push('\n'); + } + *limit -= 1; + + let src_len = src.chars().count(); + if src_len <= *limit { + dst.push_str(&src); + *limit -= src_len; + } else { + let byte_pos = src + .char_indices() + .skip(*limit) + .next() + .map(|(byte_pos, _)| byte_pos) + .unwrap_or(0); + dst.push_str(&src[0..byte_pos]); + *limit = 0; + } +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn set_logs(logs: &str, id: uuid::Uuid, db: &Pool) { + if sqlx::query!( + "UPDATE queue SET logs = $1 WHERE id = $2", + logs.to_owned(), + id + ) + .execute(db) + .await + .is_err() + { + tracing::error!(%id, "error updating logs for id {id}") + }; +} + +/* TODO retry this? */ +#[tracing::instrument(level = "trace", skip_all)] +async fn append_logs(job_id: uuid::Uuid, logs: impl AsRef, db: impl Borrow>) { + if logs.as_ref().is_empty() { + return; + } + + if let Err(err) = sqlx::query!( + "UPDATE queue SET logs = concat(logs, $1::text) WHERE id = $2", + logs.as_ref(), + job_id, + ) + .execute(db.borrow()) + .await + { + tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}"); + } +} + +pub async fn handle_zombie_jobs_periodically( + db: &Pool, + timeout: i32, + base_url: &str, + mut rx: tokio::sync::broadcast::Receiver<()>, +) { + loop { + handle_zombie_jobs(db, timeout, base_url).await; + + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(60)) => (), + _ = rx.recv() => { + println!("received killpill for monitor job"); + break; + } + } + } +} + +async fn handle_zombie_jobs(db: &Pool, timeout: i32, base_url: &str) { + let restarted = sqlx::query!( + "UPDATE queue SET running = false WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = false RETURNING id, workspace_id, last_ping", + (timeout * 5).to_string(), + JobKind::Flow: JobKind, + ) + .fetch_all(db) + .await + .ok() + .unwrap_or_else(|| vec![]); + + QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _); + for r in restarted { + tracing::info!( + "restarted zombie job {} {} {}", + r.id, + r.workspace_id, + r.last_ping + ); + } + + let timeouts = sqlx::query_as::<_, QueuedJob>( + "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND same_worker = true", + ) + .bind((timeout * 5).to_string()) + .bind(JobKind::Flow) + .fetch_all(db) + .await + .ok() + .unwrap_or_else(|| vec![]); + + QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _); + for job in timeouts { + tracing::info!( + "timedouts zombie same_worker job {} {}", + job.id, + job.workspace_id, + ); + + // since the job is unrecoverable, the same worker queue should never be sent anything + let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1); + + let tx = db.begin().await.expect("could not start token transaction"); + let (tx, token) = create_token_for_owner( + tx, + &job.workspace_id, + &job.permissioned_as, + "ephemeral-zombie-jobs", + timeout * 2, + &job.created_by, + ) + .await + .expect("could not create job token"); + tx.commit().await.expect("could not commit job token"); + let client = windmill_api_client::create_client(base_url, token.clone()); + + let _ = handle_job_error( + db, + &client, + job, + error::Error::ExecutionErr("Same worker job timed out".to_string()), + None, + true, + same_worker_tx_never_used, + "", + true, + &std::env::var("BASE_INTERNAL_URL") + .unwrap_or_else(|_| "http://localhost:8000".to_string()), + ) + .await; + } +} + +async fn handle_python_heavy_reqs( + python_path: &String, + heavy_requirements: Vec<&str>, + vars: Vec<(&str, &String)>, + job: &QueuedJob, + logs: &mut String, + db: &sqlx::Pool, + timeout: i32, +) -> error::Result> { + let mut req_paths: Vec = vec![]; + for req in heavy_requirements { + // todo: handle many reqs + let venv_p = format!("{PIP_SUPERCACHE_DIR}/{req}"); + if metadata(&venv_p).await.is_ok() { + tracing::info!("already exists: {:?}", &venv_p); + req_paths.push(venv_p); + continue; + } + + logs.push_str("\n--- PIP SUPERCACHE INSTALL ---\n"); + logs.push_str(&format!("\nthe heavy dependency {req} is being installed for the first time.\nIt will take a bit longer but further execution will be much faster!")); + + logs.push_str("pip install\n"); + let child = Command::new(python_path) + .env_clear() + .envs(vars.clone()) + .args(vec![ + "-m", + "pip", + "install", + &req, + "-I", + "--no-deps", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + venv_p.as_str(), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + handle_child(&job.id, db, logs, timeout, child).await?; + + req_paths.push(venv_p); + } + Ok(req_paths) +} diff --git a/backend/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs similarity index 80% rename from backend/src/worker_flow.rs rename to backend/windmill-worker/src/worker_flow.rs index 531651b928..eecd787288 100644 --- a/backend/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1,178 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + use std::collections::HashMap; use std::time::Duration; -use crate::{ - db::DB, - error::{self, Error}, - flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend}, - jobs::{ - add_completed_job, add_completed_job_error, canceled_job_to_result, get_queued_job, push, - schedule_again_if_scheduled, script_path_to_payload, JobPayload, QueuedJob, RawCode, - }, - js_eval::{eval_timeout, EvalCreds, IdContext}, - more_serde::is_default, - users::create_token_for_owner, - worker, -}; +use crate::jobs::{add_completed_job, add_completed_job_error, schedule_again_if_scheduled}; +use crate::js_eval::{eval_timeout, EvalCreds, IdContext}; +use crate::worker; use anyhow::Context; use async_recursion::async_recursion; use futures::TryStreamExt; -use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; use tokio::sync::mpsc::Sender; use tracing::instrument; use uuid::Uuid; - -const MINUTES: Duration = Duration::from_secs(60); -const HOURS: Duration = MINUTES.saturating_mul(60); - -pub const MAX_RETRY_ATTEMPTS: u16 = 1000; -pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); - -#[derive(Serialize, Deserialize, Debug)] -pub struct FlowStatus { - pub step: i32, - pub modules: Vec, - pub failure_module: FlowStatusModule, - #[serde(default)] - #[serde(skip_serializing_if = "is_default")] - pub retry: RetryStatus, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct RetryStatus { - pub fail_count: u16, - pub previous_result: Option, - pub failed_jobs: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Iterator { - pub index: usize, - pub itered: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct BranchAllStatus { - pub branch: usize, - pub previous_result: serde_json::Value, - pub len: usize, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum BranchChosen { - Default, - Branch { branch: usize }, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Approval { - pub resume_id: u16, - pub approver: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "type")] -pub enum FlowStatusModule { - WaitingForPriorSteps { - id: String, +use windmill_common::{ + error::{self, to_anyhow, Error}, + flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend}, + worker_flow::{ + Approval, BranchAllStatus, BranchChosen, FlowStatus, FlowStatusModule, RetryStatus, + MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL, }, - WaitingForEvents { - id: String, - count: u16, - job: Uuid, - }, - WaitingForExecutor { - id: String, - job: Uuid, - }, - InProgress { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - iterator: Option, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branchall: Option, - }, - Success { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - approvers: Vec, - }, - Failure { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - }, -} +}; -impl FlowStatusModule { - pub fn job(&self) -> Option { - match self { - FlowStatusModule::WaitingForPriorSteps { .. } => None, - FlowStatusModule::WaitingForEvents { job, .. } => Some(*job), - FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job), - FlowStatusModule::InProgress { job, .. } => Some(*job), - FlowStatusModule::Success { job, .. } => Some(*job), - FlowStatusModule::Failure { job, .. } => Some(*job), - } - } +type DB = sqlx::Pool; - pub fn id(&self) -> String { - match self { - FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(), - FlowStatusModule::WaitingForEvents { id, .. } => id.clone(), - FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(), - FlowStatusModule::InProgress { id, .. } => id.clone(), - FlowStatusModule::Success { id, .. } => id.clone(), - FlowStatusModule::Failure { id, .. } => id.clone(), - } - } -} - -impl FlowStatus { - pub fn new(f: &FlowValue) -> Self { - Self { - step: 0, - modules: f - .modules - .iter() - .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() }) - .collect(), - failure_module: FlowStatusModule::WaitingForPriorSteps { id: "failure".to_string() }, - retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] }, - } - } - - /// current module status ... excluding failure_module - pub fn current_step(&self) -> Option<&FlowStatusModule> { - let i = usize::try_from(self.step).ok()?; - self.modules.get(i) - } -} +use windmill_queue::{ + canceled_job_to_result, get_queued_job, push, JobPayload, QueuedJob, RawCode, +}; #[async_recursion] #[instrument(level = "trace", skip_all)] pub async fn update_flow_status_after_job_completion( db: &DB, + client: &windmill_api_client::Client, job: &QueuedJob, success: bool, result: serde_json::Value, @@ -241,9 +107,10 @@ pub async fn update_flow_status_after_job_completion( let skip_failure = skip_branch_failure || skip_loop_failures; let (step_counter, new_status) = match module_status { - FlowStatusModule::InProgress { iterator: Some(Iterator { index, itered, .. }), .. } - if (*index + 1 < itered.len() && (success || skip_loop_failures)) => - { + FlowStatusModule::InProgress { + iterator: Some(windmill_common::worker_flow::Iterator { index, itered, .. }), + .. + } if (*index + 1 < itered.len() && (success || skip_loop_failures)) => { (old_status.step, module_status.clone()) } FlowStatusModule::InProgress { @@ -389,6 +256,7 @@ pub async fn update_flow_status_after_job_completion( { tx = schedule_again_if_scheduled( tx, + client, flow_job.schedule_path.as_ref().unwrap(), flow_job.script_path.as_ref().unwrap(), &w_id, @@ -411,6 +279,7 @@ pub async fn update_flow_status_after_job_completion( if flow_job.canceled { add_completed_job_error( db, + client, &flow_job, logs, &canceled_job_to_result(&flow_job), @@ -420,6 +289,7 @@ pub async fn update_flow_status_after_job_completion( } else { add_completed_job( db, + client, &flow_job, success, stop_early && skip_if_stop_early.unwrap_or(false), @@ -433,6 +303,7 @@ pub async fn update_flow_status_after_job_completion( match handle_flow( &flow_job, db, + client, result.clone(), same_worker_tx.clone(), worker_dir, @@ -443,6 +314,7 @@ pub async fn update_flow_status_after_job_completion( Err(err) => { let _ = add_completed_job_error( db, + client, &flow_job, "Unexpected error during flow chaining:\n".to_string(), err, @@ -463,6 +335,7 @@ pub async fn update_flow_status_after_job_completion( if flow_job.parent_job.is_some() { return Ok(update_flow_status_after_job_completion( db, + client, &flow_job, success, result, @@ -573,10 +446,6 @@ async fn compute_bool_from_expr( } } -pub fn init_flow_status(f: &FlowValue) -> FlowStatus { - FlowStatus::new(f) -} - pub async fn update_flow_status_in_progress( db: &DB, w_id: &str, @@ -694,6 +563,7 @@ fn flatten_previous_result(last_result: serde_json::Value) -> serde_json::Value pub async fn handle_flow( flow_job: &QueuedJob, db: &sqlx::Pool, + client: &windmill_api_client::Client, last_result: serde_json::Value, same_worker_tx: Sender, worker_dir: &str, @@ -710,6 +580,7 @@ pub async fn handle_flow( let fake_job = QueuedJob { parent_job: Some(flow_job.id), ..flow_job.clone() }; update_flow_status_after_job_completion( db, + client, &fake_job, true, serde_json::json!({}), @@ -734,6 +605,7 @@ pub async fn handle_flow( status, flow, db, + client, last_result, same_worker_tx, base_internal_url, @@ -749,10 +621,11 @@ async fn push_next_flow_job( mut status: FlowStatus, flow: FlowValue, db: &sqlx::Pool, + client: &windmill_api_client::Client, mut last_result: serde_json::Value, same_worker_tx: Sender, base_internal_url: &str, -) -> anyhow::Result<()> { +) -> error::Result<()> { /* `mut` because reassigned on FlowStatusModule::Failure when failure_module is Some */ let mut i = usize::try_from(status.step) .with_context(|| format!("invalid module index {}", status.step))?; @@ -898,7 +771,7 @@ async fn push_next_flow_job( ) .bind(json!(FlowStatusModule::WaitingForEvents { id: status_module.id(), count: required_events, job: last })) .bind((required_events - resume_messages.len() as u16) as i32) - .bind(suspend.timeout.map(|t| Duration::from_secs(t.into())).unwrap_or_else(|| 30 * MINUTES)) + .bind(Duration::from_secs(suspend.timeout.map(|t| t.into()).unwrap_or_else(|| 30 * 60))) .bind(flow_job.id) .execute(&mut tx) .await?; @@ -915,7 +788,9 @@ async fn push_next_flow_job( let logs = "Timed out waiting to be resumed".to_string(); let result = json!({ "error": logs }); let _uuid = - add_completed_job(db, &flow_job, success, skipped, result, logs).await?; + add_completed_job(db, client, &flow_job, success, skipped, result, logs) + .await?; + return Ok(()); } } @@ -1024,12 +899,14 @@ async fn push_next_flow_job( _ => (), } - let mut transform_context: Option<(String, Vec, IdContext)> = None; + let mut transform_context: Option = None; let mut args = match &module.value { FlowModuleValue::Script { input_transforms, .. } | FlowModuleValue::RawScript { input_transforms, .. } => { - transform_context = - Some(get_transform_context(&db, &flow_job, &status, &flow.modules).await?); + let tx = db.begin().await?; + let (tx, ctx) = get_transform_context(tx, &flow_job, &status, &flow.modules).await?; + transform_context = Some(ctx); + tx.commit().await?; let (token, steps, by_id) = transform_context.as_ref().unwrap(); transform_input( &flow_job.args, @@ -1071,11 +948,12 @@ async fn push_next_flow_job( } }; - let next_flow_transform = compute_next_flow_transform( + let tx = db.begin().await?; + let (tx, next_flow_transform) = compute_next_flow_transform( flow_job, &flow, transform_context, - &db, + tx, &module, &status, &status_module, @@ -1083,6 +961,7 @@ async fn push_next_flow_job( base_internal_url, ) .await?; + tx.commit().await?; let (job_payload, next_status) = match next_flow_transform { NextFlowTransform::Continue(job_payload, next_state) => (job_payload, next_state), @@ -1093,6 +972,7 @@ async fn push_next_flow_job( &flow_job.id, flow.clone(), &db, + client, FlowStatusModule::Success { id: status_module.id(), job: flow_job.id, @@ -1154,7 +1034,7 @@ async fn push_next_flow_job( FlowStatusModule::InProgress { job: uuid, - iterator: Some(Iterator { index, itered }), + iterator: Some(windmill_common::worker_flow::Iterator { index, itered }), flow_jobs: Some(flow_jobs), branch_chosen: None, branchall: None, @@ -1206,7 +1086,7 @@ async fn push_next_flow_job( tx.commit().await?; if continue_on_same_worker { - same_worker_tx.send(uuid).await?; + same_worker_tx.send(uuid).await.map_err(to_anyhow)?; } return Ok(()); } @@ -1217,11 +1097,12 @@ async fn jump_to_next_step( job_id: &Uuid, flow: FlowValue, db: &DB, + client: &windmill_api_client::Client, status_module: FlowStatusModule, last_result: serde_json::Value, same_worker_tx: Sender, base_internal_url: &str, -) -> anyhow::Result<()> { +) -> error::Result<()> { let mut tx = db.begin().await?; let next_step = i @@ -1258,6 +1139,7 @@ async fn jump_to_next_step( new_status, flow, db, + client, last_result, same_worker_tx, base_internal_url, @@ -1267,7 +1149,8 @@ async fn jump_to_next_step( let success = true; let skipped = false; let logs = "Forloop completed without iteration".to_string(); - let _uuid = add_completed_job(db, &new_job, success, skipped, json!([]), logs).await?; + let _uuid = + add_completed_job(db, client, &new_job, success, skipped, json!([]), logs).await?; return Ok(()); } } @@ -1303,38 +1186,62 @@ enum NextFlowTransform { Continue(JobPayload, NextStatus), } -async fn compute_next_flow_transform( +// a similar function exists on the backend +// TODO: rewrite this to use an endpoint in the backend directly, instead of checking for hub itself, and then using the API +async fn script_path_to_payload<'c>( + script_path: &str, + db: &mut sqlx::Transaction<'c, sqlx::Postgres>, + w_id: &String, +) -> Result { + let job_payload = if script_path.starts_with("hub/") { + JobPayload::ScriptHub { path: script_path.to_owned() } + } else { + let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?; + JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() } + }; + Ok(job_payload) +} + +type TransformContext = (String, Vec, IdContext); + +async fn compute_next_flow_transform<'c>( flow_job: &QueuedJob, flow: &FlowValue, - transform_context: Option<(String, Vec, IdContext)>, - db: &DB, + transform_context: Option, + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, module: &FlowModule, status: &FlowStatus, status_module: &FlowStatusModule, last_result: serde_json::Value, base_internal_url: &str, -) -> error::Result { +) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, NextFlowTransform)> { match &module.value { - FlowModuleValue::Identity => Ok(NextFlowTransform::Continue( - JobPayload::Identity, - NextStatus::NextStep, - )), - FlowModuleValue::Script { path: script_path, .. } => Ok(NextFlowTransform::Continue( - script_path_to_payload(script_path, &mut db.begin().await?, &flow_job.workspace_id) - .await?, - NextStatus::NextStep, + FlowModuleValue::Identity => Ok(( + tx, + NextFlowTransform::Continue(JobPayload::Identity, NextStatus::NextStep), )), + FlowModuleValue::Script { path: script_path, .. } => { + let payload = + script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?; + Ok(( + tx, + NextFlowTransform::Continue(payload, NextStatus::NextStep), + )) + } FlowModuleValue::RawScript { path, content, language, .. } => { let path = path .clone() .or_else(|| Some(format!("{}/{}", flow_job.script_path(), status.step))); - Ok(NextFlowTransform::Continue( - JobPayload::Code(RawCode { - path, - content: content.clone(), - language: language.clone(), - }), - NextStatus::NextStep, + Ok(( + tx, + NextFlowTransform::Continue( + JobPayload::Code(RawCode { + path, + content: content.clone(), + language: language.clone(), + }), + NextStatus::NextStep, + ), )) } /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */ @@ -1346,31 +1253,31 @@ async fn compute_next_flow_transform( let (token, steps, by_id) = if let Some(x) = transform_context { x } else { - get_transform_context(&db, &flow_job, &status, &flow.modules).await? + let (tx_new, res) = + get_transform_context(tx, &flow_job, &status, &flow.modules).await?; + tx = tx_new; + res }; /* Iterator is an InputTransform, evaluate it into an array. */ - let itered = iterator - .clone() - .evaluate_with( - || { - vec![ - ("result".to_string(), last_result.clone()), - ("previous_result".to_string(), last_result.clone()), - ] - }, - token, - flow_job.workspace_id.clone(), - steps, - Some(by_id), - base_internal_url, - ) - .await? - .into_array() - .map_err(|not_array| { - Error::ExecutionErr(format!( - "Expected an array value, found: {not_array}" - )) - })?; + let itered = evaluate_with( + iterator.clone(), + || { + vec![ + ("result".to_string(), last_result.clone()), + ("previous_result".to_string(), last_result.clone()), + ] + }, + token, + flow_job.workspace_id.clone(), + steps, + Some(by_id), + base_internal_url, + ) + .await? + .into_array() + .map_err(|not_array| { + Error::ExecutionErr(format!("Expected an array value, found: {not_array}")) + })?; if let Some(first) = itered.first() { new_args.insert("iter".to_string(), json!({ "index": 0, "value": first })); @@ -1387,7 +1294,7 @@ async fn compute_next_flow_transform( } FlowStatusModule::InProgress { - iterator: Some(Iterator { itered, index }), + iterator: Some(windmill_common::worker_flow::Iterator { itered, index }), flow_jobs: Some(flow_jobs), .. } => { @@ -1418,17 +1325,20 @@ async fn compute_next_flow_transform( }; match next_loop_status { - LoopStatus::EmptyIterator => Ok(NextFlowTransform::EmptyInnerFlows), - LoopStatus::NextIteration(ns) => Ok(NextFlowTransform::Continue( - JobPayload::RawFlow { - value: FlowValue { - modules: (*modules).clone(), - failure_module: flow.failure_module.clone(), - same_worker: flow.same_worker, + LoopStatus::EmptyIterator => Ok((tx, NextFlowTransform::EmptyInnerFlows)), + LoopStatus::NextIteration(ns) => Ok(( + tx, + NextFlowTransform::Continue( + JobPayload::RawFlow { + value: FlowValue { + modules: (*modules).clone(), + failure_module: flow.failure_module.clone(), + same_worker: flow.same_worker, + }, + path: Some(format!("{}/loop-{}", flow_job.script_path(), status.step)), }, - path: Some(format!("{}/loop-{}", flow_job.script_path(), status.step)), - }, - NextStatus::NextLoopIteration(ns), + NextStatus::NextLoopIteration(ns), + ), )), } } @@ -1469,27 +1379,30 @@ async fn compute_next_flow_transform( default.clone() }; - Ok(NextFlowTransform::Continue( - JobPayload::RawFlow { - value: FlowValue { - modules, - failure_module: flow.failure_module.clone(), - same_worker: flow.same_worker, + Ok(( + tx, + NextFlowTransform::Continue( + JobPayload::RawFlow { + value: FlowValue { + modules, + failure_module: flow.failure_module.clone(), + same_worker: flow.same_worker, + }, + path: Some(format!( + "{}/branchone-{}", + flow_job.script_path(), + status.step + )), }, - path: Some(format!( - "{}/branchone-{}", - flow_job.script_path(), - status.step - )), - }, - NextStatus::BranchChosen(branch), + NextStatus::BranchChosen(branch), + ), )) } FlowModuleValue::BranchAll { branches, .. } => { let (status, flow_jobs) = match status_module { FlowStatusModule::WaitingForPriorSteps { .. } => { if branches.is_empty() { - return Ok(NextFlowTransform::EmptyInnerFlows); + return Ok((tx, NextFlowTransform::EmptyInnerFlows)); } else { ( BranchAllStatus { @@ -1528,33 +1441,36 @@ async fn compute_next_flow_transform( )) })?; - Ok(NextFlowTransform::Continue( - JobPayload::RawFlow { - value: FlowValue { - modules, - failure_module: flow.failure_module.clone(), - same_worker: flow.same_worker, + Ok(( + tx, + NextFlowTransform::Continue( + JobPayload::RawFlow { + value: FlowValue { + modules, + failure_module: flow.failure_module.clone(), + same_worker: flow.same_worker, + }, + path: Some(format!( + "{}/branchall-{}", + flow_job.script_path(), + status.branch + )), }, - path: Some(format!( - "{}/branchall-{}", - flow_job.script_path(), - status.branch - )), - }, - NextStatus::NextBranchStep(NextBranch { status, flow_jobs }), + NextStatus::NextBranchStep(NextBranch { status, flow_jobs }), + ), )) } } } -async fn get_transform_context( - db: &DB, +async fn get_transform_context<'c>( + tx: sqlx::Transaction<'c, sqlx::Postgres>, flow_job: &QueuedJob, status: &FlowStatus, modules: &Vec, -) -> error::Result<(String, Vec, IdContext)> { - let new_token = create_token_for_owner( - db, +) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, TransformContext)> { + let (tx, new_token) = crate::create_token_for_owner( + tx, &flow_job.workspace_id, &flow_job.permissioned_as, "transform-input", @@ -1573,53 +1489,36 @@ async fn get_transform_context( .zip(new_steps.clone()) .collect(); - Ok((new_token, new_steps, IdContext(flow_job.id, id_map))) + Ok((tx, (new_token, new_steps, IdContext(flow_job.id, id_map)))) } -impl InputTransform { - async fn evaluate_with( - self, - vars: F, - token: String, - workspace: String, - steps: Vec, - by_id: Option, - base_internal_url: &str, - ) -> anyhow::Result - where - F: FnOnce() -> Vec<(String, Value)>, - { - match self { - InputTransform::Static { value } => Ok(value), - InputTransform::Javascript { expr } => { - eval_timeout( - expr, - vars(), - Some(EvalCreds { workspace, token }), - steps, - by_id, - base_internal_url.to_string(), - ) - .await - } +async fn evaluate_with( + transform: InputTransform, + vars: F, + token: String, + workspace: String, + steps: Vec, + by_id: Option, + base_internal_url: &str, +) -> anyhow::Result +where + F: FnOnce() -> Vec<(String, serde_json::Value)>, +{ + match transform { + InputTransform::Static { value } => Ok(value), + InputTransform::Javascript { expr } => { + eval_timeout( + expr, + vars(), + Some(EvalCreds { workspace, token }), + steps, + by_id, + base_internal_url.to_string(), + ) + .await } } } - -impl QueuedJob { - pub fn parse_raw_flow(&self) -> Option { - self.raw_flow - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - } - - pub fn parse_flow_status(&self) -> Option { - self.flow_status - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - } -} - trait IntoArray: Sized { fn into_array(self) -> Result, Self>; } diff --git a/deno-client/build.sh b/deno-client/build.sh index 3f725028b8..f2c8849bb1 100755 --- a/deno-client/build.sh +++ b/deno-client/build.sh @@ -2,7 +2,7 @@ set -e npm ci --ignore-scripts -npx --yes openapi-typescript-codegen --input ../backend/openapi.yaml \ +npx --yes openapi-typescript-codegen --input ../backend/windmill-api/openapi.yaml \ --output ./src --useOptions \ && sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/core/request.ts npx --yes denoify diff --git a/frontend/package.json b/frontend/package.json index 39de04de23..8f718f015b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .", "format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. .", "package": "svelte-kit package && cd package && rm README.md && rm README_DEV.md && sed -i -e 's/windmill/windmill-components/g' package.json", - "generate-backend-client": "openapi --input ../backend/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts", + "generate-backend-client": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts", "pretest": "tsc --incremental -p tests/tsconfig.json", "test": "playwright test --config=tests-out/playwright.config.js" }, diff --git a/go-client/build.sh b/go-client/build.sh index 93bf127801..d9505b6d88 100755 --- a/go-client/build.sh +++ b/go-client/build.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -cp ../backend/openapi.yaml openapi.yaml +cp ../backend/windmill-api/openapi.yaml openapi.yaml sed -z 's/ extra_params:\n additionalProperties:\n type: string/ extra_params: {}/' openapi.yaml > openapi1.yaml sed -z 's/ enum: \[script, failure, trigger, command\]//' openapi1.yaml > openapi2.yaml diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 77a660ee6d..67ed6e61d3 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -12,6 +12,8 @@ info: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html +paths: {} + externalDocs: description: documentation portal url: https://docs.windmill.dev diff --git a/python-client/build.sh b/python-client/build.sh index 6c93443076..fc3bc71625 100755 --- a/python-client/build.sh +++ b/python-client/build.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -cp ../backend/openapi.yaml openapi.yaml +cp ../backend/windmill-api/openapi.yaml openapi.yaml npx @redocly/openapi-cli@latest bundle openapi.yaml > openapi-bundled.yaml